PapaTutuWawa/gpu-screen-recorder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Is this the final solution to the audio crackling problem? increase pts by number of samples and add dummy audio frames between

Browse Source
This commit is contained in:
dec05eba 2022-09-20 20:11:56 +02:00
parent a668cac2bb
commit f810161203
2 changed files with 53 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/main.cpp
View File

@ -483,7 +483,7 @@ static AVCodecContext* create_audio_codec_context(AVFormatContext *av_format_con
#endif
codec_context->time_base.num = 1;
codec_context->time_base.den = AV_TIME_BASE;
codec_context->time_base.den = codec_context->sample_rate;
codec_context->framerate.num = fps;
codec_context->framerate.den = 1;
@ -1462,11 +1462,12 @@ int main(int argc, char **argv) {
av_opt_set_sample_fmt(swr, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
swr_init(swr);
int64_t pts = 0;
const double target_audio_hz = 1.0 / (double)audio_track.codec_context->sample_rate;
while(running) {
void *sound_buffer;
int sound_buffer_size = sound_device_read_next_chunk(&audio_track.sound_device, &sound_buffer);
if(sound_buffer_size < 0)
sound_buffer = empty_audio;
int ret = av_frame_make_writable(audio_track.frame);
if (ret < 0) {
@ -1474,15 +1475,42 @@ int main(int argc, char **argv) {
break;
}
// TODO: Instead of converting audio, get float audio from alsa. Or does alsa do conversion internally to get this format?
swr_convert(swr, &audio_track.frame->data[0], audio_track.frame->nb_samples, (const uint8_t**)&sound_buffer, audio_track.sound_device.frames);
audio_track.frame->pts = (clock_get_monotonic_seconds() - start_time_pts) * AV_TIME_BASE;
const double this_audio_frame_time = clock_get_monotonic_seconds();
const int64_t expected_frames = std::round((this_audio_frame_time - start_time_pts) / target_audio_hz);
const int64_t num_missing_frames = std::max(0L, (expected_frames - pts) / audio_track.frame->nb_samples);
// Jesus is there a better way to do this? I JUST WANT TO KEEP VIDEO AND AUDIO SYNCED HOLY FUCK I WANT TO KILL MYSELF NOW.
// THIS PIECE OF SHIT WANTS EMPTY FRAMES OTHERWISE VIDEO PLAYS TOO FAST TO KEEP UP WITH AUDIO OR THE AUDIO PLAYS TOO EARLY.
// BUT WE CANT USE DELAYS TO GIVE DUMMY DATA BECAUSE PULSEAUDIO MIGHT GIVE AUDIO A BIG DELAYED!!!
if(num_missing_frames >= 5) {
// TODO:
//audio_track.frame->data[0] = empty_audio;
swr_convert(swr, &audio_track.frame->data[0], audio_track.frame->nb_samples, (const uint8_t**)&empty_audio, audio_track.sound_device.frames);
// TODO: Check if duplicate frame can be saved just by writing it with a different pts instead of sending it again
for(int i = 0; i < num_missing_frames; ++i) {
audio_track.frame->pts = pts;
pts += audio_track.frame->nb_samples;
ret = avcodec_send_frame(audio_track.codec_context, audio_track.frame);
if(ret >= 0){
receive_frames(audio_track.codec_context, audio_track.stream_index, audio_track.stream, audio_track.frame, av_format_context, record_start_time, frame_data_queue, replay_buffer_size_secs, frames_erased, *write_output_mutex);
} else {
fprintf(stderr, "Failed to encode audio!\n");
}
}
}
ret = avcodec_send_frame(audio_track.codec_context, audio_track.frame);
if(ret >= 0){
receive_frames(audio_track.codec_context, audio_track.stream_index, audio_track.stream, audio_track.frame, av_format_context, record_start_time, frame_data_queue, replay_buffer_size_secs, frames_erased, *write_output_mutex);
} else {
fprintf(stderr, "Failed to encode audio!\n");
if(sound_buffer_size >= 0) {
// TODO: Instead of converting audio, get float audio from alsa. Or does alsa do conversion internally to get this format?
swr_convert(swr, &audio_track.frame->data[0], audio_track.frame->nb_samples, (const uint8_t**)&sound_buffer, audio_track.sound_device.frames);
audio_track.frame->pts = pts;
pts += audio_track.frame->nb_samples;
ret = avcodec_send_frame(audio_track.codec_context, audio_track.frame);
if(ret >= 0){
receive_frames(audio_track.codec_context, audio_track.stream_index, audio_track.stream, audio_track.frame, av_format_context, record_start_time, frame_data_queue, replay_buffer_size_secs, frames_erased, *write_output_mutex);
} else {
fprintf(stderr, "Failed to encode audio!\n");
}
}
}

81
src/sound.cpp
View File

@ -20,7 +20,6 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <cmath>
#include <pulse/pulseaudio.h>
@ -28,29 +27,6 @@
#include <pulse/xmalloc.h>
#include <pulse/error.h>
#define CHECK_DEAD_GOTO(p, rerror, label) \
do { \
if (!(p)->context || !PA_CONTEXT_IS_GOOD(pa_context_get_state((p)->context)) || \
!(p)->stream || !PA_STREAM_IS_GOOD(pa_stream_get_state((p)->stream))) { \
if (((p)->context && pa_context_get_state((p)->context) == PA_CONTEXT_FAILED) || \
((p)->stream && pa_stream_get_state((p)->stream) == PA_STREAM_FAILED)) { \
if (rerror) \
*(rerror) = pa_context_errno((p)->context); \
} else \
if (rerror) \
*(rerror) = PA_ERR_BADSTATE; \
goto label; \
} \
} while(false);
static double clock_get_monotonic_seconds() {
struct timespec ts;
ts.tv_sec = 0;
ts.tv_nsec = 0;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec * 0.000000001;
}
static int sound_device_index = 0;
struct pa_handle {
@ -154,59 +130,30 @@ fail:
return NULL;
}
static void pa_sound_device_mainloop_timed(pa_handle *p, int64_t timeout_ms) {
const double start_time = clock_get_monotonic_seconds();
while((clock_get_monotonic_seconds() - start_time) * 1000.0 < timeout_ms) {
pa_mainloop_prepare(p->mainloop, 1 * 1000);
pa_mainloop_poll(p->mainloop);
pa_mainloop_dispatch(p->mainloop);
}
}
// Returns a negative value on failure. Always blocks a time specified matching the sampling rate of the audio.
// Returns a negative value on failure or if no data is available at the moment
static int pa_sound_device_read(pa_handle *p, void *data, size_t length) {
assert(p);
int r = 0;
int *rerror = &r;
bool retry = true;
pa_mainloop_iterate(p->mainloop, 0, NULL);
const int64_t timeout_ms = std::round((1000.0 / (double)pa_stream_get_sample_spec(p->stream)->rate) * 1000.0);
pa_mainloop_prepare(p->mainloop, timeout_ms * 1000);
pa_mainloop_poll(p->mainloop);
pa_mainloop_dispatch(p->mainloop);
CHECK_DEAD_GOTO(p, rerror, fail);
if(pa_stream_readable_size(p->stream) < length)
return -1;
while(true) {
if(pa_stream_readable_size(p->stream) < length) {
if(!retry)
break;
int r = pa_stream_peek(p->stream, &p->read_data, &p->read_length);
if(r != 0)
return -1;
retry = false;
pa_sound_device_mainloop_timed(p, timeout_ms);
continue;
}
r = pa_stream_peek(p->stream, &p->read_data, &p->read_length);
if(r != 0) {
if(retry)
pa_sound_device_mainloop_timed(p, timeout_ms);
return -1;
}
if(p->read_length < length || !p->read_data) {
pa_stream_drop(p->stream);
if(retry)
pa_sound_device_mainloop_timed(p, timeout_ms);
return -1;
}
memcpy(data, p->read_data, length);
if(p->read_length < length || !p->read_data) {
pa_stream_drop(p->stream);
return 0;
return -1;
}
fail:
return -1;
memcpy(data, p->read_data, length);
pa_stream_drop(p->stream);
return 0;
}
int sound_device_get_by_name(SoundDevice *device, const char *name, unsigned int num_channels, unsigned int period_frame_size) {