最近在研究ffmpeg，想做个视频聊天demo.
研究了雷博的博客，视频单独推流没问题，音频单独推流也没得问题。但是同步的时候问题来了。也不知道自己错在哪里
新手，没有C币 只能厚颜向大家提问了。
while (1) {
//Wait
SDL_WaitEvent(&event);
if (event.type == SFM_REFRESH_EVENT)
{
av_init_packet(dec_packet);
//Get an AVPacket
if (av_compare_ts(cur_pts_v, ifmt_ctx_video->streams[in_videoindex]->time_base, cur_pts_a, ifmt_ctx_audio->streams[in_audioindex]->time_base) < 0)
{
got_frame = -1;
got_packet = -1;
if (av_read_frame(ifmt_ctx_video, dec_packet) >= 0)
{
if (dec_packet->stream_index == in_videoindex)
{
pFrame_vedio = av_frame_alloc();
if (!pFrame_vedio) {
printf("alloc pFrame Failed.\n");
return -1;
}
//ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture, packet);
ret = avcodec_send_packet(inCodecCtx_v, dec_packet);
got_frame = avcodec_receive_frame(inCodecCtx_v, pFrame_vedio);
if (ret < 0)
{
av_frame_free(&pFrame_vedio);
printf("Decode Error.\n");
return -1;
}
if (got_frame == 0)
{
//转码成YUV格式
sws_scale(img_convert_ctx, (const unsigned char* const*)pFrame_vedio->data, pFrame_vedio->linesize, 0, inCodecCtx_v->height, pFrameYUV->data, pFrameYUV->linesize);
sws_scale(img_convert_ctx_SDL, (const unsigned char* const*)pFrame_vedio->data, pFrame_vedio->linesize, 0, inCodecCtx_v->height, pFrameYUV_SDL->data, pFrameYUV_SDL->linesize);
//初始化封装包
enc_packet.data = NULL;
enc_packet.size = 0;
av_init_packet(&enc_packet);
//编码
//ret = avcodec_encode_video2(oCodecCtx, &enc_packet, pFrameYUV, &got_encpicture);
ret = avcodec_send_frame(outCodecCtx_v, pFrameYUV);
//FIX : non-strictly-monotonic PTS
AVRational time_base_in = { 1, AV_TIME_BASE }; //{ 1, 1000000 };
AVRational time_base_conert = outCodecCtx_v->time_base;//{ 1, 1000 };
pFrameYUV->pts = av_rescale_q(dec_packet->pts, time_base_in, time_base_conert);
if (ret < 0)
{
av_frame_free(&pFrame_vedio);
printf("Encode Error.\n");
return -1;
}
got_packet = avcodec_receive_packet(outCodecCtx_v, &enc_packet);
if (got_packet == 0)
{
enc_packet.stream_index = out_video_st->index;
//FIX：No PTS (Example: Raw H.264)
//Simple Write PTS
if (enc_packet.pts == AV_NOPTS_VALUE)
{
//Write PTS
AVRational time_base1 = ofmt_ctx->streams[out_videoindex]->time_base;
//Duration between 2 frames (us)
int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(ofmt_ctx->streams[out_videoindex]->r_frame_rate);
//Parameters
enc_packet.pts = (double)(frame_index*calc_duration) / (double)(av_q2d(time_base1)*AV_TIME_BASE);
enc_packet.dts = enc_packet.pts;
enc_packet.duration = (double)calc_duration / (double)(av_q2d(time_base1)*AV_TIME_BASE);
}
if (dec_packet->stream_index == in_videoindex)
{
//Delay
AVRational time_base = { 1, AV_TIME_BASE };//{ 1, 1000 };
AVRational time_base_q = ofmt_ctx->streams[out_videoindex]->time_base;
int64_t pts_time = av_rescale_q(dec_packet->dts, time_base, time_base_q);
int64_t now_time = av_rescale_q(av_gettime(), time_base, time_base_q);
if (pts_time > now_time)
av_usleep(pts_time - now_time);
}
//Write PTS
AVRational time_base = ofmt_ctx->streams[out_videoindex]->time_base;//{ 1, 1000 };
AVRational r_framerate1 = ifmt_ctx_video->streams[in_videoindex]->r_frame_rate;// { 50, 2 };
AVRational time_base_q = { 1, AV_TIME_BASE };
//Duration between 2 frames (us)
int64_t calc_duration = (double)(AV_TIME_BASE)*(1 / av_q2d(r_framerate1));
enc_packet.pts = av_rescale_q(frame_index*calc_duration, time_base_q, time_base);
enc_packet.dts = enc_packet.pts;
enc_packet.duration = av_rescale_q(calc_duration, time_base_q, time_base); //(double)(calc_duration)*(double)(av_q2d(time_base_q)) / (double)(av_q2d(time_base));
enc_packet.pos = -1;
//Print to Screen
if (enc_packet.stream_index == in_videoindex) {
frame_index++;
printf("%8d Send video frames to output URL\n", frame_index);
}
cur_pts_v = enc_packet.pts;
//写到输出上下文
ret = av_interleaved_write_frame(ofmt_ctx, &enc_packet);
}
av_packet_unref(&enc_packet);
}
av_frame_free(&pFrame_vedio);
}
av_packet_unref(dec_packet);
#if USE_SDL
//int y_size = iCodecCtx->width * iCodecCtx->height;
////SDL---------------------------
//// 设置纹理数据
//pFrameYUV_SDL->data[0] = out_buffer; // Y
//pFrameYUV_SDL->data[1] = out_buffer + y_size; // U
//pFrameYUV_SDL->data[2] = out_buffer + y_size * 3 / 2; // V
#if 0
SDL_UpdateTexture(sdlTexture, NULL, pFrameYUV->data[0], pFrameYUV->linesize[0]);
#else
SDL_UpdateYUVTexture(sdlTexture, &rect,
pFrameYUV_SDL->data[0], pFrameYUV_SDL->linesize[0],
pFrameYUV_SDL->data[1], pFrameYUV_SDL->linesize[1],
pFrameYUV_SDL->data[2], pFrameYUV_SDL->linesize[2]);
#endif
// 清理渲染器
SDL_RenderClear(sdlRenderer);
// 将纹理数据copy到渲染器
//将sdlRect区域的图像显示到dstsdlRect区域
//SDL_RenderCopy( sdlRenderer, sdlTexture, &sdlRect, &sdlRect );
SDL_RenderCopy(sdlRenderer, sdlTexture, NULL, &rect);
// 显示
SDL_RenderPresent(sdlRenderer);
//SDL End-----------------------
#endif
}
}
else
{
const int output_frame_size = outCodecCtx_a->frame_size;
//int finished = 0;
av_init_packet(dec_packet);
dec_packet->data = NULL;
dec_packet->size = 0;
got_frame = -1;
got_packet = -1;
if (av_read_frame(ifmt_ctx_audio, dec_packet) >= 0)
{
if (dec_packet->stream_index == in_audioindex)
{
pFrame_audio = av_frame_alloc();
if (!pFrame_audio)
{
printf("alloc pFrame Failed.\n");
return -1;
}
if (dec_packet->stream_index >= stream_mapping_size ||
stream_mapping[dec_packet->stream_index] < 0) {
av_packet_unref(dec_packet);
continue;
}
//ret = avcodec_decode_audio4(inCodecCtx_a, pFrame, &got_picture, packet);
int ret = avcodec_send_packet(inCodecCtx_a, dec_packet);
got_frame = avcodec_receive_frame(inCodecCtx_a, pFrame_audio);
if (ret < 0)
{
av_frame_free(&pFrame_audio);
printf("Decode Error.\n");
return -1;
}
if (got_frame == 0)
{
//写入fifo
while (true)
{
int fifo_size = av_audio_fifo_size(fifo);
if (fifo_size >= output_frame_size)
break;
uint8_t **converted_input_samples;
converted_input_samples = (uint8_t **)calloc(outCodecCtx_a->channels,
sizeof(converted_input_samples));
ret = av_samples_alloc(converted_input_samples, NULL,
outCodecCtx_a->channels,
pFrame_audio->nb_samples,
outCodecCtx_a->sample_fmt, 0);
//swr_convert(au_convert_ctx, pFrameMP3->data, pFrameMP3->nb_samples, (const uint8_t**)m_ain, pFrame_audio->nb_samples);
int conver_num = swr_convert(au_convert_ctx,
converted_input_samples, pFrame_audio->nb_samples,
(const uint8_t**)pFrame_audio->extended_data, pFrame_audio->nb_samples);
ret = av_audio_fifo_realloc(fifo, av_audio_fifo_size(fifo) + pFrame_audio->nb_samples);
ret = av_audio_fifo_write(fifo, (void **)converted_input_samples, conver_num);
}
av_init_packet(&enc_packet);
enc_packet.data = NULL;
enc_packet.size = 0;
float sumlen = av_audio_fifo_size(fifo);
int num = 0;
//从fifo读出
while (true)
{
int fifo_size = av_audio_fifo_size(fifo);
if (fifo_size < output_frame_size)
break;
const int frame_size = FFMIN(av_audio_fifo_size(fifo),
outCodecCtx_a->frame_size);
AVFrame *covert_frame;
covert_frame = av_frame_alloc();
covert_frame->nb_samples = output_frame_size;
covert_frame->channel_layout = outCodecCtx_a->channel_layout;
covert_frame->format = outCodecCtx_a->sample_fmt;
covert_frame->sample_rate = outCodecCtx_a->sample_rate;
ret = av_frame_get_buffer(covert_frame, 0);
ret = av_audio_fifo_read(fifo, (void **)covert_frame->data, frame_size);
if (covert_frame)
{
covert_frame->pts = audio_pts;
audio_pts += covert_frame->nb_samples;
}
ret = avcodec_send_frame(outCodecCtx_a, covert_frame);
got_packet = avcodec_receive_packet(outCodecCtx_a, &enc_packet);
if (got_packet == 0)
{
frame_index_a++;
AVRational timebase = {1,AV_TIME_BASE };
int64_t cal_duration = AV_TIME_BASE *(float)covert_frame->nb_samples / (float)covert_frame->sample_rate;
enc_packet.pts = av_rescale_q_rnd(audio_pts, outCodecCtx_a->time_base,
ofmt_ctx->streams[out_audio_st->index]->time_base,
(AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
enc_packet.dts = enc_packet.pts;
enc_packet.duration = av_rescale_q_rnd(cal_duration, timebase,
ofmt_ctx->streams[out_audio_st->index]->time_base,
(AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
//int64_t cal_duration = AV_TIME_BASE *(float)covert_frame->nb_samples / (float)covert_frame->sample_rate;
//enc_packet.pos = -1;
//enc_packet.pts = frame_index_a*cal_duration;
//enc_packet.dts = enc_packet.pts;
//enc_packet.duration = cal_duration;
AVRational time_base = ofmt_ctx->streams[out_audioindex]->time_base;//{ 1, 1000 };
AVRational time_base_q = { 1, AV_TIME_BASE };
//Delay
int64_t pts_time = av_rescale_q(enc_packet.pts, time_base, time_base_q);
int64_t now_time = av_gettime() - start_time;
if (pts_time > now_time)
av_usleep(pts_time - now_time);
//Print to Screen
if (enc_packet.stream_index == in_audioindex)
{
printf("Send %8d audio frames to output URL\n", frame_index_a);
}
cur_pts_a = enc_packet.pts;
ret = av_interleaved_write_frame(ofmt_ctx, &enc_packet);
}
av_frame_free(&covert_frame);
}
av_packet_unref(&enc_packet);
av_packet_unref(dec_packet);
av_frame_free(&pFrame_audio);
}
}
}
av_packet_unref(dec_packet);
}
}
else if (event.type == SDL_QUIT)
{
thread_exit = 1;
}
else if(event.type == SFM_BREAK_EVENT)
{
break;
}
}
