前言
我是一名打算长期走音视频路线的Android开发者。从此系列文章开始，记录我的音视频开发学习之路
ijkplayer播放器系列文章列表：
理解ijkplayer（一）：开始
理解ijkplayer（二）项目结构分析
理解ijkplayer（三）从Java层开始初始化
理解ijkplayer（四）拉流
理解ijkplayer（五）解码、播放
理解ijkplayer（六）从ijkplayer看ffmpeg源码
理解ijkplayer（七）动态切换分辨率

1 解码线程

简略版代码：

解码线程位于：strem_component_open()中，简略版如下：

  1. static int stream_component_open(FFPlayer *ffp, int stream_index)
  2. {
  3.       AVCodecContext *avctx;//解码器上下文
  4.         AVCodec *codec = NULL;//解码器
  5.     //找到解码器
  6.     codec = avcodec_find_decoder(avctx->codec_id);
  8.       switch (avctx->codec_type) {
  9.     case AVMEDIA_TYPE_AUDIO:
  10.         ret = audio_open(ffp, channel_layout, nb_channels, sample_rate, &is->audio_tgt);
  11.            //decoder初始化
  12.         decoder_init(&is->auddec, avctx, &is->audioq, is->continue_read_thread);
  13.                 //decoder启动，启动audio_thread线程
  14.         if ((ret = decoder_start(&is->auddec, audio_thread, ffp, "ff_audio_dec")) < 0)
  15.             goto out;
  16.         break;
  17.     case AVMEDIA_TYPE_VIDEO:
  18.         //decoder初始化
  19.         decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread);
  20.         ffp->node_vdec = ffpipeline_open_video_decoder(ffp->pipeline, ffp);
  21.         if (!ffp->node_vdec)
  22.           goto fail;
  23.         //解码器开始
  24.         if ((ret = decoder_start(&is->viddec, video_thread, ffp, "ff_video_dec")) < 0)
  25.           goto out;
  26.         break;
  27.       case AVMEDIA_TYPE_SUBTITLE:
  28.         //decoder初始化
  29.         decoder_init(&is->subdec, avctx, &is->subtitleq, is->continue_read_thread);
  30.         //解码器开始
  31.         if ((ret = decoder_start(&is->subdec, subtitle_thread, ffp, "ff_subtitle_dec")) < 0)
  32.             goto out;
  33.         break;
  34. }

完整版代码：

  1. /* open a given stream. Return 0 if OK */
  2. static int stream_component_open(FFPlayer *ffp, int stream_index)
  3. {
  4.     VideoState *is = ffp->is;
  5.     AVFormatContext *ic = is->ic;
  6.     AVCodecContext *avctx;//解码器上下文
  7.     AVCodec *codec = NULL;//解码器
  8.     const char *forced_codec_name = NULL;
  9.     AVDictionary *opts = NULL;
  10.     AVDictionaryEntry *t = NULL;
  11.     int sample_rate, nb_channels;
  12.     int64_t channel_layout;
  13.     int ret = 0;
  14.     int stream_lowres = ffp->lowres;
  16.     if (stream_index < 0 || stream_index >= ic->nb_streams)
  17.         return -1;
  18.     avctx = avcodec_alloc_context3(NULL);
  19.     if (!avctx)
  20.         return AVERROR(ENOMEM);
  21.     //将AVCodecParameters中的变量赋值给AVCodecContext
  22.     ret = avcodec_parameters_to_context(avctx, ic->streams[stream_index]->codecpar);
  23.     if (ret < 0)
  24.         goto fail;
  25.     av_codec_set_pkt_timebase(avctx, ic->streams[stream_index]->time_base);
  26.     //找到解码器
  27.     codec = avcodec_find_decoder(avctx->codec_id);
  29.     switch (avctx->codec_type) {
  30.         case AVMEDIA_TYPE_AUDIO   : is->last_audio_stream    = stream_index; forced_codec_name = ffp->audio_codec_name; break;
  31.         case AVMEDIA_TYPE_SUBTITLE: is->last_subtitle_stream = stream_index; forced_codec_name = ffp->subtitle_codec_name; break;
  32.         case AVMEDIA_TYPE_VIDEO   : is->last_video_stream    = stream_index; forced_codec_name = ffp->video_codec_name; break;
  33.         default: break;
  34.     }
  35.     if (forced_codec_name)
  36.         codec = avcodec_find_decoder_by_name(forced_codec_name);
  37.     if (!codec) {
  38.         if (forced_codec_name) av_log(NULL, AV_LOG_WARNING,
  39.                                       "No codec could be found with name '%s'\n", forced_codec_name);
  40.         else                   av_log(NULL, AV_LOG_WARNING,
  41.                                       "No codec could be found with id %d\n", avctx->codec_id);
  42.         ret = AVERROR(EINVAL);
  43.         goto fail;
  44.     }
  46.     avctx->codec_id = codec->id;
  47.     if(stream_lowres > av_codec_get_max_lowres(codec)){
  48.         av_log(avctx, AV_LOG_WARNING, "The maximum value for lowres supported by the decoder is %d\n",
  49.                 av_codec_get_max_lowres(codec));
  50.         stream_lowres = av_codec_get_max_lowres(codec);
  51.     }
  52.     av_codec_set_lowres(avctx, stream_lowres);
  54. #if FF_API_EMU_EDGE
  55.     if(stream_lowres) avctx->flags |= CODEC_FLAG_EMU_EDGE;
  56. #endif
  57.     if (ffp->fast)
  58.         avctx->flags2 |= AV_CODEC_FLAG2_FAST;
  59. #if FF_API_EMU_EDGE
  60.     if(codec->capabilities & AV_CODEC_CAP_DR1)
  61.         avctx->flags |= CODEC_FLAG_EMU_EDGE;
  62. #endif
  64.     opts = filter_codec_opts(ffp->codec_opts, avctx->codec_id, ic, ic->streams[stream_index], codec);
  65.     if (!av_dict_get(opts, "threads", NULL, 0))
  66.         av_dict_set(&opts, "threads", "auto", 0);
  67.     if (stream_lowres)
  68.         av_dict_set_int(&opts, "lowres", stream_lowres, 0);
  69.     if (avctx->codec_type == AVMEDIA_TYPE_VIDEO || avctx->codec_type == AVMEDIA_TYPE_AUDIO)
  70.         av_dict_set(&opts, "refcounted_frames", "1", 0);
  71.     if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) {
  72.         goto fail;
  73.     }
  74.     if ((t = av_dict_get(opts, "", NULL, AV_DICT_IGNORE_SUFFIX))) {
  75.         av_log(NULL, AV_LOG_ERROR, "Option %s not found.\n", t->key);
  76. #ifdef FFP_MERGE
  77.         ret =  AVERROR_OPTION_NOT_FOUND;
  78.         goto fail;
  79. #endif
  80.     }
  82.     is->eof = 0;
  83.     ic->streams[stream_index]->discard = AVDISCARD_DEFAULT;
  84.     switch (avctx->codec_type) {
  85.     case AVMEDIA_TYPE_AUDIO:
  86. #if CONFIG_AVFILTER
  87.         {
  88.             AVFilterContext *sink;
  90.             is->audio_filter_src.freq           = avctx->sample_rate;
  91.             is->audio_filter_src.channels       = avctx->channels;
  92.             is->audio_filter_src.channel_layout = get_valid_channel_layout(avctx->channel_layout, avctx->channels);
  93.             is->audio_filter_src.fmt            = avctx->sample_fmt;
  94.             SDL_LockMutex(ffp->af_mutex);
  95.             if ((ret = configure_audio_filters(ffp, ffp->afilters, 0)) < 0) {
  96.                 SDL_UnlockMutex(ffp->af_mutex);
  97.                 goto fail;
  98.             }
  99.             ffp->af_changed = 0;
  100.             SDL_UnlockMutex(ffp->af_mutex);
  101.             sink = is->out_audio_filter;
  102.             sample_rate    = av_buffersink_get_sample_rate(sink);
  103.             nb_channels    = av_buffersink_get_channels(sink);
  104.             channel_layout = av_buffersink_get_channel_layout(sink);
  105.         }
  106. #else
  107.         sample_rate    = avctx->sample_rate;
  108.         nb_channels    = avctx->channels;
  109.         channel_layout = avctx->channel_layout;
  110. #endif
  112.         /* prepare audio output */
  113.         //audio_open方法是在做什么？
  114.         if ((ret = audio_open(ffp, channel_layout, nb_channels, sample_rate, &is->audio_tgt)) < 0)
  115.             goto fail;
  116.         ffp_set_audio_codec_info(ffp, AVCODEC_MODULE_NAME, avcodec_get_name(avctx->codec_id));
  117.         is->audio_hw_buf_size = ret;
  118.         is->audio_src = is->audio_tgt;
  119.         is->audio_buf_size  = 0;
  120.         is->audio_buf_index = 0;
  122.         /* init averaging filter */
  123.         is->audio_diff_avg_coef  = exp(log(0.01) / AUDIO_DIFF_AVG_NB);
  124.         is->audio_diff_avg_count = 0;
  125.         /* since we do not have a precise anough audio FIFO fullness,
  126.            we correct audio sync only if larger than this threshold */
  127.         is->audio_diff_threshold = 2.0 * is->audio_hw_buf_size / is->audio_tgt.bytes_per_sec;
  129.         is->audio_stream = stream_index;
  130.         is->audio_st = ic->streams[stream_index];
  131.         //decoder初始化
  132.         decoder_init(&is->auddec, avctx, &is->audioq, is->continue_read_thread);
  133.         if ((is->ic->iformat->flags & (AVFMT_NOBINSEARCH | AVFMT_NOGENSEARCH | AVFMT_NO_BYTE_SEEK)) && !is->ic->iformat->read_seek) {
  134.             is->auddec.start_pts = is->audio_st->start_time;
  135.             is->auddec.start_pts_tb = is->audio_st->time_base;
  136.         }
  137.         //decoder启动，启动audio_thread线程
  138.         if ((ret = decoder_start(&is->auddec, audio_thread, ffp, "ff_audio_dec")) < 0)
  139.             goto out;
  140.         SDL_AoutPauseAudio(ffp->aout, 0);
  141.         break;
  142.     case AVMEDIA_TYPE_VIDEO:
  143.         is->video_stream = stream_index;
  144.         is->video_st = ic->streams[stream_index];
  145.         //async_init_decoder是一个option，默认是0
  146.         if (ffp->async_init_decoder) {
  147.             while (!is->initialized_decoder) {
  148.                 SDL_Delay(5);
  149.             }
  150.             if (ffp->node_vdec) {
  151.                 is->viddec.avctx = avctx;
  152.                 ret = ffpipeline_config_video_decoder(ffp->pipeline, ffp);
  153.             }
  154.             if (ret || !ffp->node_vdec) {
  155.                 decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread);
  156.                 ffp->node_vdec = ffpipeline_open_video_decoder(ffp->pipeline, ffp);
  157.                 if (!ffp->node_vdec)
  158.                     goto fail;
  159.             }
  160.         } else {
  161.             //decoder初始化
  162.             decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread);
  163.             ffp->node_vdec = ffpipeline_open_video_decoder(ffp->pipeline, ffp);
  164.             if (!ffp->node_vdec)
  165.                 goto fail;
  166.         }
  167.         //解码器开始
  168.         if ((ret = decoder_start(&is->viddec, video_thread, ffp, "ff_video_dec")) < 0)
  169.             goto out;
  171.         is->queue_attachments_req = 1;
  173.         if (ffp->max_fps >= 0) {
  174.             if(is->video_st->avg_frame_rate.den && is->video_st->avg_frame_rate.num) {
  175.                 double fps = av_q2d(is->video_st->avg_frame_rate);
  176.                 SDL_ProfilerReset(&is->viddec.decode_profiler, fps + 0.5);
  177.                 if (fps > ffp->max_fps && fps < 130.0) {
  178.                     is->is_video_high_fps = 1;
  179.                     av_log(ffp, AV_LOG_WARNING, "fps: %lf (too high)\n", fps);
  180.                 } else {
  181.                     av_log(ffp, AV_LOG_WARNING, "fps: %lf (normal)\n", fps);
  182.                 }
  183.             }
  184.             if(is->video_st->r_frame_rate.den && is->video_st->r_frame_rate.num) {
  185.                 double tbr = av_q2d(is->video_st->r_frame_rate);
  186.                 if (tbr > ffp->max_fps && tbr < 130.0) {
  187.                     is->is_video_high_fps = 1;
  188.                     av_log(ffp, AV_LOG_WARNING, "fps: %lf (too high)\n", tbr);
  189.                 } else {
  190.                     av_log(ffp, AV_LOG_WARNING, "fps: %lf (normal)\n", tbr);
  191.                 }
  192.             }
  193.         }
  195.         if (is->is_video_high_fps) {
  196.             avctx->skip_frame       = FFMAX(avctx->skip_frame, AVDISCARD_NONREF);
  197.             avctx->skip_loop_filter = FFMAX(avctx->skip_loop_filter, AVDISCARD_NONREF);
  198.             avctx->skip_idct        = FFMAX(avctx->skip_loop_filter, AVDISCARD_NONREF);
  199.         }
  201.         break;
  202.     case AVMEDIA_TYPE_SUBTITLE:
  203.         if (!ffp->subtitle) break;
  205.         is->subtitle_stream = stream_index;
  206.         is->subtitle_st = ic->streams[stream_index];
  208.         ffp_set_subtitle_codec_info(ffp, AVCODEC_MODULE_NAME, avcodec_get_name(avctx->codec_id));
  210.         decoder_init(&is->subdec, avctx, &is->subtitleq, is->continue_read_thread);
  211.         if ((ret = decoder_start(&is->subdec, subtitle_thread, ffp, "ff_subtitle_dec")) < 0)
  212.             goto out;
  213.         break;
  214.     default:
  215.         break;
  216.     }
  217.     goto out;
  219. fail:
  220.     avcodec_free_context(&avctx);
  221. out:
  222.     av_dict_free(&opts);
  224.     return ret;
  225. }

小结：

  1. 找到解码器
  2. 初始化解码器
  3. 分别启动audio_thread，video_thread和subtitle_thread这3条解码线程，内部开始不断解码。

那么以下3节则逐个分析这3条解码线程

2 字幕解码线程subtitle_thread

由于字幕解码线程最简单，所以先来看看他是如何工作的，对剩下的两个解码线程就更好理解了。

  1. static int subtitle_thread(void *arg)
  2. {
  3.     FFPlayer *ffp = arg;
  4.     VideoState *is = ffp->is;
  5.     Frame *sp;
  6.     int got_subtitle;
  7.     double pts;
  9.     for (;;) {
  10.         //阻塞方法，阻塞，直到能取出windex（写下标）下标下的Frame
  11.         if (!(sp = frame_queue_peek_writable(&is->subpq)))
  12.             return 0;
  13.         //解码，填充Frame中的字幕数据
  14.         if ((got_subtitle = decoder_decode_frame(ffp, &is->subdec, NULL, &sp->sub)) < 0)
  15.             break;
  17.         pts = 0;
  18. #ifdef FFP_MERGE
  19.         if (got_subtitle && sp->sub.format == 0) {
  20. #else
  21.         if (got_subtitle) {
  22. #endif
  23.             if (sp->sub.pts != AV_NOPTS_VALUE)
  24.                 pts = sp->sub.pts / (double)AV_TIME_BASE;
  25.             sp->pts = pts;
  26.             sp->serial = is->subdec.pkt_serial;
  27.             sp->width = is->subdec.avctx->width;
  28.             sp->height = is->subdec.avctx->height;
  29.             sp->uploaded = 0;
  31.             /* now we can update the picture count */
  32.             //后移字幕FrameQueue的windex
  33.             frame_queue_push(&is->subpq);
  34. #ifdef FFP_MERGE
  35.         } else if (got_subtitle) {
  36.             avsubtitle_free(&sp->sub);
  37. #endif
  38.         }
  39.     }
  40.     return 0;
  41. }

解码后的数据sp要保存，留着待会渲染，也就是要入队，那么由于FrameQueue是数组的特殊性，因此入队的操作不需要新建的frame数据作为参数，只需要确保数组中的write index的数据正确填充，然后将write index后移一个位置，就称为入队成功了：

  1. static void  frame_queue_push(FrameQueue *f)
  2. {
  3.     //当使用数组作为队列的时候，只需要移动数组中的下标到有效下标，就表示入队了，并不需要外部再传一个参数进来。
  4.     //如果到了尾下标，则windex回到起点。这是用数组作为循环队列的必要操作。
  5.     if (++f->windex == f->max_size)
  6.         f->windex = 0;
  7.     SDL_LockMutex(f->mutex);
  8.     f->size++;
  9.     SDL_CondSignal(f->cond);
  10.     SDL_UnlockMutex(f->mutex);
  11. }

那么接着来看：decoder_decode_frame（）

注意：本文基于0.8.0的ijkplayer，这个函数和以前的ijkplayer的解码逻辑和调用的ffmpeg的函数都有些区别。我看到0.8.0的版本的decoder_decode_frame（）函数的逻辑是在0.8.7的时候修改并上线的。

2.1 decoder_decode_frame()，since version 0.8.7

先看本文基于的0.8.0的ijkplayer的函数：

  1. static int decoder_decode_frame(FFPlayer *ffp, Decoder *d, AVFrame *frame, AVSubtitle *sub) {
  2.     int ret = AVERROR(EAGAIN);
  4.     for (;;) {
  5.         AVPacket pkt;
  7.         if (d->queue->serial == d->pkt_serial) {
  8.             do {
  9.                 if (d->queue->abort_request)
  10.                     return -1;
  12.                 switch (d->avctx->codec_type) {
  13.                     case AVMEDIA_TYPE_VIDEO:
  14.                         //从解码器中接收frame数据。当返回0表示成功
  15.                         ret = avcodec_receive_frame(d->avctx, frame);
  16.                         if (ret >= 0) {
  17.                             ffp->stat.vdps = SDL_SpeedSamplerAdd(&ffp->vdps_sampler, FFP_SHOW_VDPS_AVCODEC, "vdps[avcodec]");
  18.                             if (ffp->decoder_reorder_pts == -1) {
  19.                                 frame->pts = frame->best_effort_timestamp;
  20.                             } else if (!ffp->decoder_reorder_pts) {
  21.                                 frame->pts = frame->pkt_dts;
  22.                             }
  23.                         }
  24.                         break;
  25.                     case AVMEDIA_TYPE_AUDIO:
  26.                         //从解码器中接收frame数据。当返回0表示成功
  27.                         ret = avcodec_receive_frame(d->avctx, frame);
  28.                         if (ret >= 0) {
  29.                             AVRational tb = (AVRational){1, frame->sample_rate};
  30.                             if (frame->pts != AV_NOPTS_VALUE)
  31.                                 frame->pts = av_rescale_q(frame->pts, av_codec_get_pkt_timebase(d->avctx), tb);
  32.                             else if (d->next_pts != AV_NOPTS_VALUE)
  33.                                 frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb);
  34.                             if (frame->pts != AV_NOPTS_VALUE) {
  35.                                 d->next_pts = frame->pts + frame->nb_samples;
  36.                                 d->next_pts_tb = tb;
  37.                             }
  38.                         }
  39.                         break;
  40.                     default:
  41.                         break;
  42.                 }
  43.                 if (ret == AVERROR_EOF) {
  44.                     d->finished = d->pkt_serial;
  45.                     avcodec_flush_buffers(d->avctx);
  46.                     return 0;
  47.                 }
  48.                 //如果返回值>=0，表示avcodec_receive_frame函数解码成功，那么从外部函数decoder_decode_frame返回1。
  49.                 //视频，音频，字幕的解码都从这里返回，只要解码成功，都去读取ret然后返回给外面处理。
  50.                 if (ret >= 0)
  51.                     return 1;
  52.             } while (ret != AVERROR(EAGAIN));
  53.         }
  55.         do {
  56.             if (d->queue->nb_packets == 0)
  57.                 SDL_CondSignal(d->empty_queue_cond);
  58.             if (d->packet_pending) {
  59.                 av_packet_move_ref(&pkt, &d->pkt);
  60.                 d->packet_pending = 0;
  61.             } else {
  62.                 //从packet_queue中取出pkt，当packat_queue由于网络差等原因，没有足够的包可以取出时，则阻塞，直到有包能取出。
  63.                 if (packet_queue_get_or_buffering(ffp, d->queue, &pkt, &d->pkt_serial, &d->finished) < 0)
  64.                     return -1;
  65.             }
  66.         } while (d->queue->serial != d->pkt_serial);
  68.         if (pkt.data == flush_pkt.data) {
  69.             avcodec_flush_buffers(d->avctx);
  70.             d->finished = 0;
  71.             d->next_pts = d->start_pts;
  72.             d->next_pts_tb = d->start_pts_tb;
  73.         } else {
  74.             if (d->avctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
  75.                 int got_frame = 0;
  76.                 //解码字幕
  77.                 ret = avcodec_decode_subtitle2(d->avctx, sub, &got_frame, &pkt);
  78.                 if (ret < 0) {
  79.                     ret = AVERROR(EAGAIN);
  80.                 } else {
  81.                     if (got_frame && !pkt.data) {
  82.                        d->packet_pending = 1;
  83.                        av_packet_move_ref(&d->pkt, &pkt);
  84.                     }
  85.                     ret = got_frame ? 0 : (pkt.data ? AVERROR(EAGAIN) : AVERROR_EOF);
  86.                 }
  87.             } else {
  88.                 //往解码器里面发送包数据pkt
  89.                 if (avcodec_send_packet(d->avctx, &pkt) == AVERROR(EAGAIN)) {
  90.                     av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation.\n");
  91.                     d->packet_pending = 1;
  92.                     av_packet_move_ref(&d->pkt, &pkt);
  93.                 }
  94.             }
  95.             av_packet_unref(&pkt);
  96.         }
  97.     }
  98. }

2.2 decoder_decode_frame()，before version 0.8.7

  1. static int decoder_decode_frame(FFPlayer *ffp, Decoder *d, AVFrame *frame, AVSubtitle *sub) {
  2.     int got_frame = 0;
  4.     do {
  5.         int ret = -1;
  7.         if (d->queue->abort_request)
  8.             return -1;
  10.         if (!d->packet_pending || d->queue->serial != d->pkt_serial) {
  11.             AVPacket pkt;
  12.             do {
  13.                 if (d->queue->nb_packets == 0)
  14.                     SDL_CondSignal(d->empty_queue_cond);
  15.                   //从packet_queue中获取pkt
  16.                 if (packet_queue_get_or_buffering(ffp, d->queue, &pkt, &d->pkt_serial, &d->finished) < 0)
  17.                     return -1;
  18.                 if (pkt.data == flush_pkt.data) {
  19.                     avcodec_flush_buffers(d->avctx);
  20.                     d->finished = 0;
  21.                     d->next_pts = d->start_pts;
  22.                     d->next_pts_tb = d->start_pts_tb;
  23.                 }
  24.             } while (pkt.data == flush_pkt.data || d->queue->serial != d->pkt_serial);
  25.             av_packet_unref(&d->pkt);
  26.               //将包pkt传递给解码器d
  27.             d->pkt_temp = d->pkt = pkt;
  28.             d->packet_pending = 1;
  29.         }
  31.         switch (d->avctx->codec_type) {
  32.             case AVMEDIA_TYPE_VIDEO: {
  33.                   //调用ffmpeg方法：avcodec_deco_video2()来解码。
  34.                 ret = avcodec_decode_video2(d->avctx, frame, &got_frame, &d->pkt_temp);
  35.                 if (got_frame) {
  36.                     ffp->stat.vdps = SDL_SpeedSamplerAdd(&ffp->vdps_sampler, FFP_SHOW_VDPS_AVCODEC, "vdps[avcodec]");
  37.                     if (ffp->decoder_reorder_pts == -1) {
  38.                         frame->pts = av_frame_get_best_effort_timestamp(frame);
  39.                     } else if (!ffp->decoder_reorder_pts) {
  40.                         frame->pts = frame->pkt_dts;
  41.                     }
  42.                 }
  43.                 }
  44.                 break;
  45.             case AVMEDIA_TYPE_AUDIO:
  46.                     //调用ffmpeg方法：avcodec_decode_audio4()来解码
  47.                 ret = avcodec_decode_audio4(d->avctx, frame, &got_frame, &d->pkt_temp);
  48.                 if (got_frame) {
  49.                     AVRational tb = (AVRational){1, frame->sample_rate};
  50.                     if (frame->pts != AV_NOPTS_VALUE)
  51.                         frame->pts = av_rescale_q(frame->pts, av_codec_get_pkt_timebase(d->avctx), tb);
  52.                     else if (d->next_pts != AV_NOPTS_VALUE)
  53.                         frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb);
  54.                     if (frame->pts != AV_NOPTS_VALUE) {
  55.                         d->next_pts = frame->pts + frame->nb_samples;
  56.                         d->next_pts_tb = tb;
  57.                     }
  58.                 }
  59.                 break;
  60.             case AVMEDIA_TYPE_SUBTITLE:
  61.                 ret = avcodec_decode_subtitle2(d->avctx, sub, &got_frame, &d->pkt_temp);
  62.                 break;
  63.             default:
  64.                 break;
  65.         }
  67.         if (ret < 0) {
  68.             d->packet_pending = 0;
  69.         } else {
  70.             d->pkt_temp.dts =
  71.             d->pkt_temp.pts = AV_NOPTS_VALUE;
  72.             if (d->pkt_temp.data) {
  73.                 if (d->avctx->codec_type != AVMEDIA_TYPE_AUDIO)
  74.                     ret = d->pkt_temp.size;
  75.                 d->pkt_temp.data += ret;
  76.                 d->pkt_temp.size -= ret;
  77.                 if (d->pkt_temp.size <= 0)
  78.                     d->packet_pending = 0;
  79.             } else {
  80.                 if (!got_frame) {
  81.                     d->packet_pending = 0;
  82.                     d->finished = d->pkt_serial;
  83.                 }
  84.             }
  85.         }
  86.     } while (!got_frame && !d->finished);
  88.     return got_frame;
  89. }

那么再看回到最新的decoder_decode_frame()方法中，首先解码器要从包队列PakcetQueue中读取出包数据，再输送到ffmpeg解码器中。那么这个读取包队列中已经缓存好的包数据的方法是：

  1. static int packet_queue_get_or_buffering(FFPlayer *ffp, PacketQueue *q, AVPacket *pkt, int *serial, int *finished)
  2. {
  3.     assert(finished);
  4.     if (!ffp->packet_buffering)
  5.         return packet_queue_get(q, pkt, 1, serial);
  7.     while (1) {
  8.         int new_packet = packet_queue_get(q, pkt, 0, serial);
  9.         if (new_packet < 0)
  10.             return -1;
  11.         else if (new_packet == 0) {
  12.             //=0表示no packet，因此要再取
  13.             if (q->is_buffer_indicator && !*finished)
  14.                 ffp_toggle_buffering(ffp, 1);
  15.             //阻塞，直到从包队列中取出队列头的包，并填充到pkt
  16.             new_packet = packet_queue_get(q, pkt, 1, serial);
  17.             if (new_packet < 0)
  18.                 return -1;
  19.         }
  21.         if (*finished == *serial) {
  22.             av_packet_unref(pkt);
  23.             continue;
  24.         }
  25.         else
  26.             break;
  27.     }
  29.     return 1;
  30. }

即读取包pkt是会阻塞的，直到3.6.4章节介绍的视频读取线程读取并解封装包pkt，并放入PacketQueue，这里才能从阻塞返回并继续塞给解码器。

小结

  1. 0.8.7开始，decode_decode_frame()函数借助ffmpeg的两个方法来完成解码：
    1. int avcodec_send_packet(AVCodecContex* *avctx, const AVPacket *avpkt);往解码器里面发送pkt数据。
    2. int avcodec_receive_frame(AVCodecContext *avctx, AVFrame *frame);从解码器里面读取出frame帧数据。

  2. 而在0.8.7之前，音频和视频的解码都各自分别使用一个不同的解码函数：

    1. 视频： 

      1. //已被废弃
      2. int avcodec_decode_video2(AVCodecContext *avctx, AVFrame *picture,
      3.                  int *got_picture_ptr,
      4.                  const AVPacket *avpkt);

    2. 音频： 

      1. //已被废弃
      2. int avcodec_decode_audio4(AVCodecContext *avctx, AVFrame *frame,
      3.                   int *got_frame_ptr, const AVPacket *avpkt)

  3. 解码字幕的函数： 

    1. int avcodec_decode_subtitle2(AVCodecContext *avctx, AVSubtitle *sub,
    2.                         int *got_sub_ptr,
    3.                         AVPacket *avpkt);

  4. 从字幕的解码流程中可以看出解码的大致逻辑为：

    1. 循环地调用decoder_decode_frame（），在这个方法里面对视频，音频和字幕3种流用switch语句来分别处理解码。当然，在音频解码audio_thread和视频解码video_thread中同样会调用这个方法的。
    2. 解码前，先从PacketQueue读取包数据，这个数据从哪里来？从read_thread()函数中调用的ffmpeg的函数：av_read_frame(ic, pkt);来的。
    3. 解码时，先塞给解码器pkt数据，再从解码器中读出解码好的frame数据。
    4. 再把frame数据入队FrameQueue，留给稍后的渲染器来从FrameQueue中读取

3 音频解码线程audio_thread

  1. static int audio_thread(void *arg)
  2. {
  3.     FFPlayer *ffp = arg;
  4.     VideoState *is = ffp->is;
  5.     AVFrame *frame = av_frame_alloc();//分配一个AVFrame
  6.     Frame *af;//从FrameQueue sampq中取出来的，要写入数据的Frame
  7. #if CONFIG_AVFILTER
  8.     int last_serial = -1;
  9.     int64_t dec_channel_layout;
  10.     int reconfigure;
  11. #endif
  12.     int got_frame = 0;
  13.     AVRational tb;//分子分母对(ffmpeg为了准确性和避免转换，定义了一个分子分母对来取代float)
  14.     int ret = 0;
  15.     int audio_accurate_seek_fail = 0;
  16.     int64_t audio_seek_pos = 0;
  17.     double frame_pts = 0;
  18.     double audio_clock = 0;
  19.     int64_t now = 0;
  20.     double samples_duration = 0;
  21.     int64_t deviation = 0;
  22.     int64_t deviation2 = 0;
  23.     int64_t deviation3 = 0;
  25.     if (!frame)
  26.         return AVERROR(ENOMEM);
  28.     do {
  29.         ffp_audio_statistic_l(ffp);
  30.         //音频解码
  31.         if ((got_frame = decoder_decode_frame(ffp, &is->auddec, frame, NULL)) < 0)
  32.             goto the_end;
  33.         //当解码成功
  34.         if (got_frame) {
  35.                 tb = (AVRational){1, frame->sample_rate};
  36.                 //处理accurate_seek
  37.                 if (ffp->enable_accurate_seek && is->audio_accurate_seek_req && !is->seek_req) {
  38.                     frame_pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
  39.                     now = av_gettime_relative() / 1000;
  40.                     if (!isnan(frame_pts)) {
  41.                         samples_duration = (double) frame->nb_samples / frame->sample_rate;
  42.                         audio_clock = frame_pts + samples_duration;
  43.                         is->accurate_seek_aframe_pts = audio_clock * 1000 * 1000;
  44.                         audio_seek_pos = is->seek_pos;
  45.                         deviation = llabs((int64_t)(audio_clock * 1000 * 1000) - is->seek_pos);
  46.                         if ((audio_clock * 1000 * 1000 < is->seek_pos ) || deviation > MAX_DEVIATION) {
  47.                             if (is->drop_aframe_count == 0) {
  48.                                 SDL_LockMutex(is->accurate_seek_mutex);
  49.                                 if (is->accurate_seek_start_time <= 0 && (is->video_stream < 0 || is->video_accurate_seek_req)) {
  50.                                     is->accurate_seek_start_time = now;
  51.                                 }
  52.                                 SDL_UnlockMutex(is->accurate_seek_mutex);
  53.                                 av_log(NULL, AV_LOG_INFO, "audio accurate_seek start, is->seek_pos=%lld, audio_clock=%lf, is->accurate_seek_start_time = %lld\n", is->seek_pos, audio_clock, is->accurate_seek_start_time);
  54.                             }
  55.                             is->drop_aframe_count++;
  56.                             while (is->video_accurate_seek_req && !is->abort_request) {
  57.                                 int64_t vpts = is->accurate_seek_vframe_pts;
  58.                                 deviation2 = vpts  - audio_clock * 1000 * 1000;
  59.                                 deviation3 = vpts  - is->seek_pos;
  60.                                 if (deviation2 > -100 * 1000 && deviation3 < 0) {
  62.                                     break;
  63.                                 } else {
  64.                                     av_usleep(20 * 1000);
  65.                                 }
  66.                                 now = av_gettime_relative() / 1000;
  67.                                 if ((now - is->accurate_seek_start_time) > ffp->accurate_seek_timeout) {
  68.                                     break;
  69.                                 }
  70.                             }
  72.                             if(!is->video_accurate_seek_req && is->video_stream >= 0 && audio_clock * 1000 * 1000 > is->accurate_seek_vframe_pts) {
  73.                                 audio_accurate_seek_fail = 1;
  74.                             } else {
  75.                                 now = av_gettime_relative() / 1000;
  76.                                 if ((now - is->accurate_seek_start_time) <= ffp->accurate_seek_timeout) {
  77.                                     av_frame_unref(frame);
  78.                                     continue;  // drop some old frame when do accurate seek
  79.                                 } else {
  80.                                     audio_accurate_seek_fail = 1;
  81.                                 }
  82.                             }
  83.                         } else {
  84.                             if (audio_seek_pos == is->seek_pos) {
  85.                                 av_log(NULL, AV_LOG_INFO, "audio accurate_seek is ok, is->drop_aframe_count=%d, audio_clock = %lf\n", is->drop_aframe_count, audio_clock);
  86.                                 is->drop_aframe_count       = 0;
  87.                                 SDL_LockMutex(is->accurate_seek_mutex);
  88.                                 is->audio_accurate_seek_req = 0;
  89.                                 SDL_CondSignal(is->video_accurate_seek_cond);
  90.                                 if (audio_seek_pos == is->seek_pos && is->video_accurate_seek_req && !is->abort_request) {
  91.                                     SDL_CondWaitTimeout(is->audio_accurate_seek_cond, is->accurate_seek_mutex, ffp->accurate_seek_timeout);
  92.                                 } else {
  93.                                     ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, (int)(audio_clock * 1000));
  94.                                 }
  96.                                 if (audio_seek_pos != is->seek_pos && !is->abort_request) {
  97.                                     is->audio_accurate_seek_req = 1;
  98.                                     SDL_UnlockMutex(is->accurate_seek_mutex);
  99.                                     av_frame_unref(frame);
  100.                                     continue;
  101.                                 }
  103.                                 SDL_UnlockMutex(is->accurate_seek_mutex);
  104.                             }
  105.                         }
  106.                     } else {
  107.                         audio_accurate_seek_fail = 1;
  108.                     }
  109.                     if (audio_accurate_seek_fail) {
  110.                         av_log(NULL, AV_LOG_INFO, "audio accurate_seek is error, is->drop_aframe_count=%d, now = %lld, audio_clock = %lf\n", is->drop_aframe_count, now, audio_clock);
  111.                         is->drop_aframe_count       = 0;
  112.                         SDL_LockMutex(is->accurate_seek_mutex);
  113.                         is->audio_accurate_seek_req = 0;
  114.                         SDL_CondSignal(is->video_accurate_seek_cond);
  115.                         if (is->video_accurate_seek_req && !is->abort_request) {
  116.                             SDL_CondWaitTimeout(is->audio_accurate_seek_cond, is->accurate_seek_mutex, ffp->accurate_seek_timeout);
  117.                         } else {
  118.                             ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, (int)(audio_clock * 1000));
  119.                         }
  120.                         SDL_UnlockMutex(is->accurate_seek_mutex);
  121.                     }
  122.                     is->accurate_seek_start_time = 0;
  123.                     audio_accurate_seek_fail = 0;
  124.                 }
  126. #if CONFIG_AVFILTER
  127.                 dec_channel_layout = get_valid_channel_layout(frame->channel_layout, frame->channels);
  129.                 reconfigure =
  130.                     cmp_audio_fmts(is->audio_filter_src.fmt, is->audio_filter_src.channels,
  131.                                    frame->format, frame->channels)    ||
  132.                     is->audio_filter_src.channel_layout != dec_channel_layout ||
  133.                     is->audio_filter_src.freq           != frame->sample_rate ||
  134.                     is->auddec.pkt_serial               != last_serial        ||
  135.                     ffp->af_changed;
  137.                 if (reconfigure) {
  138.                     SDL_LockMutex(ffp->af_mutex);
  139.                     ffp->af_changed = 0;
  140.                     char buf1[1024], buf2[1024];
  141.                     av_get_channel_layout_string(buf1, sizeof(buf1), -1, is->audio_filter_src.channel_layout);
  142.                     av_get_channel_layout_string(buf2, sizeof(buf2), -1, dec_channel_layout);
  143.                     av_log(NULL, AV_LOG_DEBUG,
  144.                            "Audio frame changed from rate:%d ch:%d fmt:%s layout:%s serial:%d to rate:%d ch:%d fmt:%s layout:%s serial:%d\n",
  145.                            is->audio_filter_src.freq, is->audio_filter_src.channels, av_get_sample_fmt_name(is->audio_filter_src.fmt), buf1, last_serial,
  146.                            frame->sample_rate, frame->channels, av_get_sample_fmt_name(frame->format), buf2, is->auddec.pkt_serial);
  148.                     is->audio_filter_src.fmt            = frame->format;
  149.                     is->audio_filter_src.channels       = frame->channels;
  150.                     is->audio_filter_src.channel_layout = dec_channel_layout;
  151.                     is->audio_filter_src.freq           = frame->sample_rate;
  152.                     last_serial                         = is->auddec.pkt_serial;
  154.                     if ((ret = configure_audio_filters(ffp, ffp->afilters, 1)) < 0) {
  155.                         SDL_UnlockMutex(ffp->af_mutex);
  156.                         goto the_end;
  157.                     }
  158.                     SDL_UnlockMutex(ffp->af_mutex);
  159.                 }
  161.             if ((ret = av_buffersrc_add_frame(is->in_audio_filter, frame)) < 0)
  162.                 goto the_end;
  164.             while ((ret = av_buffersink_get_frame_flags(is->out_audio_filter, frame, 0)) >= 0) {
  165.                 tb = av_buffersink_get_time_base(is->out_audio_filter);
  166. #endif
  167.                 if (!(af = frame_queue_peek_writable(&is->sampq)))//如果sampq无法写入，则失败
  168.                     goto the_end;
  170.                 af->pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
  171.                 af->pos = frame->pkt_pos;
  172.                 af->serial = is->auddec.pkt_serial;
  173.                 af->duration = av_q2d((AVRational){frame->nb_samples, frame->sample_rate});
  174.                 //Move everything contained in src to dst and reset src.将解码出来的AVFrame传给af->frame
  175.                 av_frame_move_ref(af->frame, frame);
  176.                 //将af->frame入队
  177.                 frame_queue_push(&is->sampq);
  179. #if CONFIG_AVFILTER
  180.                 if (is->audioq.serial != is->auddec.pkt_serial)
  181.                     break;
  182.             }
  183.             if (ret == AVERROR_EOF)
  184.                 is->auddec.finished = is->auddec.pkt_serial;
  185. #endif
  186.         }
  187.     } while (ret >= 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF);
  188.  the_end:
  189. #if CONFIG_AVFILTER
  190.     avfilter_graph_free(&is->agraph);
  191. #endif
  192.     av_frame_free(&frame);
  193.     return ret;
  194. }

音频解码这里暂时不去分析解码之后的seek操作，所以和字幕解码没什么差别，没什么好分析的。

4 视频解码线程video_thread

终于来到视频解码了…

  1. static int video_thread(void *arg)
  2. {
  3.     FFPlayer *ffp = (FFPlayer *)arg;
  4.     int       ret = 0;
  5.         //如果node_vdec不为null。
  6.     if (ffp->node_vdec) {
  7.           //调用解码器的解码方法，进入循环
  8.         ret = ffpipenode_run_sync(ffp->node_vdec);
  9.     }
  10.     return ret;
  11. }

最后是走到了IJKFF_Pipenode的func_run_sync()函数中

  1. static int ffplay_video_thread(void *arg)
  2. {
  3.     FFPlayer *ffp = arg;
  4.     VideoState *is = ffp->is;
  5.     AVFrame *frame = av_frame_alloc();//创建一个新的AVFrame
  6.     double pts;
  7.     double duration;
  8.     int ret;
  9.     AVRational tb = is->video_st->time_base;
  10.     AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL);
  11.     int64_t dst_pts = -1;
  12.     int64_t last_dst_pts = -1;
  13.     int retry_convert_image = 0;
  14.     int convert_frame_count = 0;
  16. #if CONFIG_AVFILTER
  17.     AVFilterGraph *graph = avfilter_graph_alloc();
  18.     AVFilterContext *filt_out = NULL, *filt_in = NULL;
  19.     int last_w = 0;
  20.     int last_h = 0;
  21.     enum AVPixelFormat last_format = -2;
  22.     int last_serial = -1;
  23.     int last_vfilter_idx = 0;
  24.     if (!graph) {
  25.         av_frame_free(&frame);
  26.         return AVERROR(ENOMEM);
  27.     }
  29. #else
  30.     ffp_notify_msg2(ffp, FFP_MSG_VIDEO_ROTATION_CHANGED, ffp_get_video_rotate_degrees(ffp));
  31. #endif
  33.     if (!frame) {
  34. #if CONFIG_AVFILTER
  35.         avfilter_graph_free(&graph);
  36. #endif
  37.         return AVERROR(ENOMEM);
  38.     }
  39.     //开启无限循环，无限地去从packet_queue中拿取pkt来解码。
  40.     for (;;) {
  41.         ret = get_video_frame(ffp, frame);//解码，并将解码后的帧数据存放在frame中
  42.         if (ret < 0)
  43.             goto the_end;
  44.         if (!ret)
  45.             continue;
  47.         if (ffp->get_frame_mode) {
  48.             if (!ffp->get_img_info || ffp->get_img_info->count <= 0) {
  49.                 av_frame_unref(frame);
  50.                 continue;
  51.             }
  53.             last_dst_pts = dst_pts;
  55.             if (dst_pts < 0) {
  56.                 dst_pts = ffp->get_img_info->start_time;
  57.             } else {
  58.                 dst_pts += (ffp->get_img_info->end_time - ffp->get_img_info->start_time) / (ffp->get_img_info->num - 1);
  59.             }
  61.             pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
  62.             pts = pts * 1000;
  63.             if (pts >= dst_pts) {
  64.                 while (retry_convert_image <= MAX_RETRY_CONVERT_IMAGE) {
  65.                     ret = convert_image(ffp, frame, (int64_t)pts, frame->width, frame->height);
  66.                     if (!ret) {
  67.                         convert_frame_count++;
  68.                         break;
  69.                     }
  70.                     retry_convert_image++;
  71.                     av_log(NULL, AV_LOG_ERROR, "convert image error retry_convert_image = %d\n", retry_convert_image);
  72.                 }
  74.                 retry_convert_image = 0;
  75.                 if (ret || ffp->get_img_info->count <= 0) {
  76.                     if (ret) {
  77.                         av_log(NULL, AV_LOG_ERROR, "convert image abort ret = %d\n", ret);
  78.                         ffp_notify_msg3(ffp, FFP_MSG_GET_IMG_STATE, 0, ret);
  79.                     } else {
  80.                         av_log(NULL, AV_LOG_INFO, "convert image complete convert_frame_count = %d\n", convert_frame_count);
  81.                     }
  82.                     goto the_end;
  83.                 }
  84.             } else {
  85.                 dst_pts = last_dst_pts;
  86.             }
  87.             av_frame_unref(frame);
  88.             continue;
  89.         }
  91. //省略了AV_FILTER部分的代码
  92.             duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){frame_rate.den, frame_rate.num}) : 0);
  93.             pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
  94.             //将frame入队到pictq中，来让渲染线程读取。
  95.             ret = queue_picture(ffp, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);
  96.             av_frame_unref(frame);
  98.         if (ret < 0)
  99.             goto the_end;
  100.     }
  101.  the_end:
  102. #if CONFIG_AVFILTER
  103.     avfilter_graph_free(&graph);
  104. #endif
  105.     av_log(NULL, AV_LOG_INFO, "convert image convert_frame_count = %d\n", convert_frame_count);
  106.     av_frame_free(&frame);
  107.     return 0;
  108. }

简略为：

  1. static int ffplay_video_thread(void *arg){
  2.   for(;;){
  3.     ret = get_video_frame(ffp, frame);//解码，并将解码后的帧数据存放在frame中
  4.     //将frame入队到pictq中，来让渲染线程读取。
  5.     ret = queue_picture(ffp, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);
  6.   }
  7. }

那么看下解码函数：

  1. static int get_video_frame(FFPlayer *ffp, AVFrame *frame)
  2. {
  3.     VideoState *is = ffp->is;
  4.     int got_picture;
  6.     ffp_video_statistic_l(ffp);
  7.     //解码，并将视频帧数据填充到frame中，可能阻塞
  8.     if ((got_picture = decoder_decode_frame(ffp, &is->viddec, frame, NULL)) < 0)
  9.         return -1;
  11.     if (got_picture) {
  12.         double dpts = NAN;
  14.         if (frame->pts != AV_NOPTS_VALUE)
  15.             dpts = av_q2d(is->video_st->time_base) * frame->pts;
  17.         frame->sample_aspect_ratio = av_guess_sample_aspect_ratio(is->ic, is->video_st, frame);
  19.         if (ffp->framedrop>0 || (ffp->framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) {
  20.             ffp->stat.decode_frame_count++;
  21.             if (frame->pts != AV_NOPTS_VALUE) {
  22.                 double diff = dpts - get_master_clock(is);
  23.                 if (!isnan(diff) && fabs(diff) < AV_NOSYNC_THRESHOLD &&
  24.                     diff - is->frame_last_filter_delay < 0 &&
  25.                     is->viddec.pkt_serial == is->vidclk.serial &&
  26.                     is->videoq.nb_packets) {
  27.                     is->frame_drops_early++;
  28.                     is->continuous_frame_drops_early++;
  29.                     if (is->continuous_frame_drops_early > ffp->framedrop) {
  30.                         is->continuous_frame_drops_early = 0;
  31.                     } else {
  32.                         ffp->stat.drop_frame_count++;
  33.                         ffp->stat.drop_frame_rate = (float)(ffp->stat.drop_frame_count) / (float)(ffp->stat.decode_frame_count);
  34.                         av_frame_unref(frame);
  35.                         got_picture = 0;
  36.                     }
  37.                 }
  38.             }
  39.         }
  40.     }
  42.     return got_picture;
  43. }

那么这里又是用的和字幕解码、音频解码一样的解码函数：decoder_decode_frame，就不重复提了。

  1. //将src_frame入队到picq中，让渲染线程渲染。
  2. static int queue_picture(FFPlayer *ffp, AVFrame *src_frame, double pts, double duration, int64_t pos, int serial)
  3. {
  4.     VideoState *is = ffp->is;
  5.     Frame *vp;
  6.     int video_accurate_seek_fail = 0;
  7.     int64_t video_seek_pos = 0;
  8.     int64_t now = 0;
  9.     int64_t deviation = 0;
  11.     int64_t deviation2 = 0;
  12.     int64_t deviation3 = 0;
  13.     //处理精确seek
  14.     if (ffp->enable_accurate_seek && is->video_accurate_seek_req && !is->seek_req) {
  15.         if (!isnan(pts)) {
  16.             video_seek_pos = is->seek_pos;
  17.             is->accurate_seek_vframe_pts = pts * 1000 * 1000;
  18.             deviation = llabs((int64_t)(pts * 1000 * 1000) - is->seek_pos);
  19.             if ((pts * 1000 * 1000 < is->seek_pos) || deviation > MAX_DEVIATION) {
  20.                 now = av_gettime_relative() / 1000;
  21.                 if (is->drop_vframe_count == 0) {
  22.                     SDL_LockMutex(is->accurate_seek_mutex);
  23.                     if (is->accurate_seek_start_time <= 0 && (is->audio_stream < 0 || is->audio_accurate_seek_req)) {
  24.                         is->accurate_seek_start_time = now;
  25.                     }
  26.                     SDL_UnlockMutex(is->accurate_seek_mutex);
  27.                     av_log(NULL, AV_LOG_INFO, "video accurate_seek start, is->seek_pos=%lld, pts=%lf, is->accurate_seek_time = %lld\n", is->seek_pos, pts, is->accurate_seek_start_time);
  28.                 }
  29.                 is->drop_vframe_count++;
  31.                 while (is->audio_accurate_seek_req && !is->abort_request) {
  32.                     int64_t apts = is->accurate_seek_aframe_pts ;
  33.                     deviation2 = apts - pts * 1000 * 1000;
  34.                     deviation3 = apts - is->seek_pos;
  36.                     if (deviation2 > -100 * 1000 && deviation3 < 0) {
  37.                         break;
  38.                     } else {
  39.                         av_usleep(20 * 1000);
  40.                     }
  41.                     now = av_gettime_relative() / 1000;
  42.                     if ((now - is->accurate_seek_start_time) > ffp->accurate_seek_timeout) {
  43.                         break;
  44.                     }
  45.                 }
  47.                 if ((now - is->accurate_seek_start_time) <= ffp->accurate_seek_timeout) {
  48.                     return 1;  // drop some old frame when do accurate seek
  49.                 } else {
  50.                     av_log(NULL, AV_LOG_WARNING, "video accurate_seek is error, is->drop_vframe_count=%d, now = %lld, pts = %lf\n", is->drop_vframe_count, now, pts);
  51.                     video_accurate_seek_fail = 1;  // if KEY_FRAME interval too big, disable accurate seek
  52.                 }
  53.             } else {
  54.                 av_log(NULL, AV_LOG_INFO, "video accurate_seek is ok, is->drop_vframe_count =%d, is->seek_pos=%lld, pts=%lf\n", is->drop_vframe_count, is->seek_pos, pts);
  55.                 if (video_seek_pos == is->seek_pos) {
  56.                     is->drop_vframe_count       = 0;
  57.                     SDL_LockMutex(is->accurate_seek_mutex);
  58.                     is->video_accurate_seek_req = 0;
  59.                     SDL_CondSignal(is->audio_accurate_seek_cond);
  60.                     if (video_seek_pos == is->seek_pos && is->audio_accurate_seek_req && !is->abort_request) {
  61.                         SDL_CondWaitTimeout(is->video_accurate_seek_cond, is->accurate_seek_mutex, ffp->accurate_seek_timeout);
  62.                     } else {
  63.                         ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, (int)(pts * 1000));
  64.                     }
  65.                     if (video_seek_pos != is->seek_pos && !is->abort_request) {
  66.                         is->video_accurate_seek_req = 1;
  67.                         SDL_UnlockMutex(is->accurate_seek_mutex);
  68.                         return 1;
  69.                     }
  71.                     SDL_UnlockMutex(is->accurate_seek_mutex);
  72.                 }
  73.             }
  74.         } else {
  75.             video_accurate_seek_fail = 1;
  76.         }
  78.         if (video_accurate_seek_fail) {
  79.             is->drop_vframe_count = 0;
  80.             SDL_LockMutex(is->accurate_seek_mutex);
  81.             is->video_accurate_seek_req = 0;
  82.             SDL_CondSignal(is->audio_accurate_seek_cond);
  83.             if (is->audio_accurate_seek_req && !is->abort_request) {
  84.                 SDL_CondWaitTimeout(is->video_accurate_seek_cond, is->accurate_seek_mutex, ffp->accurate_seek_timeout);
  85.             } else {
  86.                 if (!isnan(pts)) {
  87.                     ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, (int)(pts * 1000));
  88.                 } else {
  89.                     ffp_notify_msg2(ffp, FFP_MSG_ACCURATE_SEEK_COMPLETE, 0);
  90.                 }
  91.             }
  92.             SDL_UnlockMutex(is->accurate_seek_mutex);
  93.         }
  94.         is->accurate_seek_start_time = 0;
  95.         video_accurate_seek_fail = 0;
  96.         is->accurate_seek_vframe_pts = 0;
  97.     }
  99. #if defined(DEBUG_SYNC)
  100.     printf("frame_type=%c pts=%0.3f\n",
  101.            av_get_picture_type_char(src_frame->pict_type), pts);
  102. #endif
  104.     if (!(vp = frame_queue_peek_writable(&is->pictq)))
  105.         return -1;
  107.     vp->sar = src_frame->sample_aspect_ratio;
  108. #ifdef FFP_MERGE
  109.     vp->uploaded = 0;
  110. #endif
  112.     /* alloc or resize hardware picture buffer */
  113.     if (!vp->bmp || !vp->allocated ||
  114.         vp->width  != src_frame->width ||
  115.         vp->height != src_frame->height ||
  116.         vp->format != src_frame->format) {
  118.         if (vp->width != src_frame->width || vp->height != src_frame->height)
  119.             ffp_notify_msg3(ffp, FFP_MSG_VIDEO_SIZE_CHANGED, src_frame->width, src_frame->height);
  121.         vp->allocated = 0;
  122.         vp->width = src_frame->width;
  123.         vp->height = src_frame->height;
  124.         vp->format = src_frame->format;
  126.         /* the allocation must be done in the main thread to avoid
  127.            locking problems. */
  128.         alloc_picture(ffp, src_frame->format);
  130.         if (is->videoq.abort_request)
  131.             return -1;
  132.     }
  134.     /* if the frame is not skipped, then display it */
  135.     if (vp->bmp) {
  136.         /* get a pointer on the bitmap */
  137.         SDL_VoutLockYUVOverlay(vp->bmp);//加锁
  139. #ifdef FFP_MERGE
  140. #if CONFIG_AVFILTER
  141.         // FIXME use direct rendering
  142.         av_image_copy(data, linesize, (const uint8_t **)src_frame->data, src_frame->linesize,
  143.                         src_frame->format, vp->width, vp->height);
  144. #else
  145.         // sws_getCachedContext(...);
  146. #endif
  147. #endif
  148.         // FIXME: set swscale options
  149.         //将src_frame中的帧数据填充到vp->bmp中，这个vp->bmp其实指的是bitmap？
  150.         if (SDL_VoutFillFrameYUVOverlay(vp->bmp, src_frame) < 0) {
  151.             av_log(NULL, AV_LOG_FATAL, "Cannot initialize the conversion context\n");
  152.             exit(1);
  153.         }
  154.         /* update the bitmap content */
  155.         SDL_VoutUnlockYUVOverlay(vp->bmp);//解锁
  157.         vp->pts = pts;
  158.         vp->duration = duration;
  159.         vp->pos = pos;
  160.         vp->serial = serial;
  161.         vp->sar = src_frame->sample_aspect_ratio;
  162.         vp->bmp->sar_num = vp->sar.num;
  163.         vp->bmp->sar_den = vp->sar.den;
  165. #ifdef FFP_MERGE
  166.         av_frame_move_ref(vp->frame, src_frame);
  167. #endif
  168.         frame_queue_push(&is->pictq);
  169.         if (!is->viddec.first_frame_decoded) {
  170.             ALOGD("Video: first frame decoded\n");
  171.             ffp_notify_msg1(ffp, FFP_MSG_VIDEO_DECODED_START);
  172.             is->viddec.first_frame_decoded_time = SDL_GetTickHR();
  173.             is->viddec.first_frame_decoded = 1;
  174.         }
  175.     }
  176.     return 0;
  177. }

这里重点看下将frame数据填充到vp->bmp数据中的这个操作。

bmp长得非常像bitmap，看来意思是将帧数据填充到图像数据中的意思了。

  1. int SDL_VoutFillFrameYUVOverlay(SDL_VoutOverlay *overlay, const AVFrame *frame)
  2. {
  3.     if (!overlay || !overlay->func_fill_frame)
  4.         return -1;
  6.     return overlay->func_fill_frame(overlay, frame);
  7. }
  1. static int func_fill_frame(SDL_VoutOverlay *overlay, const AVFrame *frame){
  2.     //...
  3.     overlay_fill(overlay, opaque->linked_frame, opaque->planes);
  4.     //...
  5. }
  7. static void overlay_fill(SDL_VoutOverlay *overlay, AVFrame *frame, int planes)
  8. {
  9.     overlay->planes = planes;
  11.     for (int i = 0; i < AV_NUM_DATA_POINTERS; ++i) {
  12.         //数组的复制 
  13.         overlay->pixels[i] = frame->data[i];
  14.         overlay->pitches[i] = frame->linesize[i];
  15.     }
  16. }

那么到这里，应该是将AVFrame中的数据全部复制到这个vp->bmp中了，而他是：*SDL_VoutOverlay*

5 视频渲染线程

  1. //创建视频刷新线程
  2. is->video_refresh_tid = SDL_CreateThreadEx(&is->_video_refresh_tid, video_refresh_thread, ffp, "ff_vout");

创建一个线程专门用于渲染视频。在看代码之前，先了解一下视频渲染要做什么：

  1. 从FrameQueue中拿取每一帧解码完的原始图像帧数据。
  2. 将帧数据发送到显示设备，让对应设备将图像数据画出来。
  3. 这是一个循环的过程，解码线程不断解码出图像帧，这边的渲染线程不断地读取图像帧并输送到渲染设备。
  1. //    ijkmedia/ijkplayer/ff_ffplay.c
  2. static int video_refresh_thread(void *arg)
  3. {
  4.     FFPlayer *ffp = arg;
  5.     VideoState *is = ffp->is;
  6.     double remaining_time = 0.0;
  7.       //循环，如果没有中断请求，那么就一直尝试去渲染。
  8.     while (!is->abort_request) {
  9.         if (remaining_time > 0.0)
  10.             av_usleep((int)(int64_t)(remaining_time * 1000000.0));
  11.         remaining_time = REFRESH_RATE;
  12.         if (is->show_mode != SHOW_MODE_NONE && (!is->paused || is->force_refresh))
  13.               //刷新视频
  14.             video_refresh(ffp, &remaining_time);
  15.     }
  17.     return 0;
  18. }
  1. //    ijkmedia/ijkplayer/ff_ffplay.c
  3. /* called to display each frame */
  4. static void video_refresh(FFPlayer *opaque, double *remaining_time)
  5. {
  6.     FFPlayer *ffp = opaque;
  7.     VideoState *is = ffp->is;
  8.     double time;
  10.     Frame *sp, *sp2;
  11.     //处理时钟。
  12.     if (!is->paused && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK && is->realtime)
  13.         check_external_clock_speed(is);
  15.     if (!ffp->display_disable && is->show_mode != SHOW_MODE_VIDEO && is->audio_st) {
  16.         time = av_gettime_relative() / 1000000.0;
  17.         if (is->force_refresh || is->last_vis_time + ffp->rdftspeed < time) {
  18.             //①
  19.             video_display2(ffp);
  20.             is->last_vis_time = time;
  21.         }
  22.         *remaining_time = FFMIN(*remaining_time, is->last_vis_time + ffp->rdftspeed - time);
  23.     }
  25.     if (is->video_st) {
  26. retry:
  27.         if (frame_queue_nb_remaining(&is->pictq) == 0) {
  28.             // nothing to do, no picture to display in the queue
  29.         } else {
  30.             double last_duration, duration, delay;
  31.             Frame *vp, *lastvp;
  33.             /* dequeue the picture */
  34.             lastvp = frame_queue_peek_last(&is->pictq);
  35.             vp = frame_queue_peek(&is->pictq);
  37.             if (vp->serial != is->videoq.serial) {
  38.                 frame_queue_next(&is->pictq);
  39.                 goto retry;
  40.             }
  42.             if (lastvp->serial != vp->serial)
  43.                 is->frame_timer = av_gettime_relative() / 1000000.0;
  45.             if (is->paused)
  46.                 goto display;
  48.             /* compute nominal last_duration */
  49.             last_duration = vp_duration(is, lastvp, vp);
  50.             delay = compute_target_delay(ffp, last_duration, is);
  52.             time= av_gettime_relative()/1000000.0;
  53.             if (isnan(is->frame_timer) || time < is->frame_timer)
  54.                 is->frame_timer = time;
  55.             if (time < is->frame_timer + delay) {
  56.                 *remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);
  57.                 goto display;
  58.             }
  60.             is->frame_timer += delay;
  61.             if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX)
  62.                 is->frame_timer = time;
  64.             SDL_LockMutex(is->pictq.mutex);
  65.             if (!isnan(vp->pts))
  66.                 update_video_pts(is, vp->pts, vp->pos, vp->serial);
  67.             SDL_UnlockMutex(is->pictq.mutex);
  69.             if (frame_queue_nb_remaining(&is->pictq) > 1) {
  70.                 Frame *nextvp = frame_queue_peek_next(&is->pictq);
  71.                 duration = vp_duration(is, vp, nextvp);
  72.                 if(!is->step && (ffp->framedrop > 0 || (ffp->framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration) {
  73.                     frame_queue_next(&is->pictq);
  74.                     goto retry;
  75.                 }
  76.             }
  78.             if (is->subtitle_st) {
  79.                 while (frame_queue_nb_remaining(&is->subpq) > 0) {
  80.                     sp = frame_queue_peek(&is->subpq);
  82.                     if (frame_queue_nb_remaining(&is->subpq) > 1)
  83.                         sp2 = frame_queue_peek_next(&is->subpq);
  84.                     else
  85.                         sp2 = NULL;
  87.                     if (sp->serial != is->subtitleq.serial
  88.                             || (is->vidclk.pts > (sp->pts + ((float) sp->sub.end_display_time / 1000)))
  89.                             || (sp2 && is->vidclk.pts > (sp2->pts + ((float) sp2->sub.start_display_time / 1000))))
  90.                     {
  91.                         if (sp->uploaded) {
  92.                             ffp_notify_msg4(ffp, FFP_MSG_TIMED_TEXT, 0, 0, "", 1);
  93.                         }
  94.                         frame_queue_next(&is->subpq);
  95.                     } else {
  96.                         break;
  97.                     }
  98.                 }
  99.             }
  101.             frame_queue_next(&is->pictq);
  102.             is->force_refresh = 1;
  104.             SDL_LockMutex(ffp->is->play_mutex);
  105.             if (is->step) {
  106.                 is->step = 0;
  107.                 if (!is->paused)
  108.                     stream_update_pause_l(ffp);
  109.             }
  110.             SDL_UnlockMutex(ffp->is->play_mutex);
  111.         }
  112. display:
  113.         /* display picture */
  114.         if (!ffp->display_disable && is->force_refresh && is->show_mode == SHOW_MODE_VIDEO && is->pictq.rindex_shown)
  115.             //①
  116.             video_display2(ffp);
  117.     }
  118.     is->force_refresh = 0;
  119.     if (ffp->show_status) {
  120.         static int64_t last_time;
  121.         int64_t cur_time;
  122.         int aqsize, vqsize, sqsize __unused;
  123.         double av_diff;
  125.         cur_time = av_gettime_relative();
  126.         if (!last_time || (cur_time - last_time) >= 30000) {
  127.             aqsize = 0;
  128.             vqsize = 0;
  129.             sqsize = 0;
  130.             if (is->audio_st)
  131.                 aqsize = is->audioq.size;
  132.             if (is->video_st)
  133.                 vqsize = is->videoq.size;
  134. #ifdef FFP_MERGE
  135.             if (is->subtitle_st)
  136.                 sqsize = is->subtitleq.size;
  137. #else
  138.             sqsize = 0;
  139. #endif
  140.             av_diff = 0;
  141.             if (is->audio_st && is->video_st)
  142.                 av_diff = get_clock(&is->audclk) - get_clock(&is->vidclk);
  143.             else if (is->video_st)
  144.                 av_diff = get_master_clock(is) - get_clock(&is->vidclk);
  145.             else if (is->audio_st)
  146.                 av_diff = get_master_clock(is) - get_clock(&is->audclk);
  147.             av_log(NULL, AV_LOG_INFO,
  148.                    "%7.2f %s:%7.3f fd=%4d aq=%5dKB vq=%5dKB sq=%5dB f=%"PRId64"/%"PRId64"   \r",
  149.                    get_master_clock(is),
  150.                    (is->audio_st && is->video_st) ? "A-V" : (is->video_st ? "M-V" : (is->audio_st ? "M-A" : "   ")),
  151.                    av_diff,
  152.                    is->frame_drops_early + is->frame_drops_late,
  153.                    aqsize / 1024,
  154.                    vqsize / 1024,
  155.                    sqsize,
  156.                    is->video_st ? is->viddec.avctx->pts_correction_num_faulty_dts : 0,
  157.                    is->video_st ? is->viddec.avctx->pts_correction_num_faulty_pts : 0);
  158.             fflush(stdout);
  159.             last_time = cur_time;
  160.         }
  161.     }
  162. }

一长串代码，貌似有一些根据时钟来同步音视频的代码？暂时不做分析，这里面要跳转到方法（用①做了标记，有两处）：

  1. //①
  2. video_display2(ffp);
  1. /* display the current picture, if any */
  2. static void video_display2(FFPlayer *ffp)
  3. {
  4.     VideoState *is = ffp->is;
  5.     if (is->video_st)
  6.         video_image_display2(ffp);
  7. }
  1. static void video_image_display2(FFPlayer *ffp)
  2. {
  3.     VideoState *is = ffp->is;
  4.     Frame *vp;
  5.     Frame *sp = NULL;
  6.     //is->pictq就是picture queue的意思。读取队列中最后一帧。
  7.     vp = frame_queue_peek_last(&is->pictq);
  8.     //如果帧中的SDL_VoutOverlay数据不为null，那么就开始渲染
  9.     if (vp->bmp) {
  10.         //如果字幕流不为空，去渲染字幕
  11.         if (is->subtitle_st) {
  12.             if (frame_queue_nb_remaining(&is->subpq) > 0) {
  13.                 sp = frame_queue_peek(&is->subpq);
  15.                 if (vp->pts >= sp->pts + ((float) sp->sub.start_display_time / 1000)) {
  16.                     if (!sp->uploaded) {
  17.                         if (sp->sub.num_rects > 0) {
  18.                             char buffered_text[4096];
  19.                             if (sp->sub.rects[0]->text) {
  20.                                 strncpy(buffered_text, sp->sub.rects[0]->text, 4096);
  21.                             }
  22.                             else if (sp->sub.rects[0]->ass) {
  23.                                 parse_ass_subtitle(sp->sub.rects[0]->ass, buffered_text);
  24.                             }
  25.                             ffp_notify_msg4(ffp, FFP_MSG_TIMED_TEXT, 0, 0, buffered_text, sizeof(buffered_text));
  26.                         }
  27.                         sp->uploaded = 1;
  28.                     }
  29.                 }
  30.             }
  31.         }
  32.         if (ffp->render_wait_start && !ffp->start_on_prepared && is->pause_req) {
  33.             if (!ffp->first_video_frame_rendered) {
  34.                 ffp->first_video_frame_rendered = 1;
  35.                 ffp_notify_msg1(ffp, FFP_MSG_VIDEO_RENDERING_START);
  36.             }
  37.             while (is->pause_req && !is->abort_request) {
  38.                 SDL_Delay(20);
  39.             }
  40.         }
  41.         //显示YUV数据。
  42.         SDL_VoutDisplayYUVOverlay(ffp->vout, vp->bmp);
  43.         ffp->stat.vfps = SDL_SpeedSamplerAdd(&ffp->vfps_sampler, FFP_SHOW_VFPS_FFPLAY, "vfps[ffplay]");
  44.         if (!ffp->first_video_frame_rendered) {
  45.             ffp->first_video_frame_rendered = 1;
  46.             ffp_notify_msg1(ffp, FFP_MSG_VIDEO_RENDERING_START);
  47.         }
  49.         if (is->latest_video_seek_load_serial == vp->serial) {
  50.             int latest_video_seek_load_serial = __atomic_exchange_n(&(is->latest_video_seek_load_serial), -1, memory_order_seq_cst);
  51.             if (latest_video_seek_load_serial == vp->serial) {
  52.                 ffp->stat.latest_seek_load_duration = (av_gettime() - is->latest_seek_load_start_at) / 1000;
  53.                 if (ffp->av_sync_type == AV_SYNC_VIDEO_MASTER) {
  54.                     ffp_notify_msg2(ffp, FFP_MSG_VIDEO_SEEK_RENDERING_START, 1);
  55.                 } else {
  56.                     ffp_notify_msg2(ffp, FFP_MSG_VIDEO_SEEK_RENDERING_START, 0);
  57.                 }
  58.             }
  59.         }
  60.     }
  61. }

首先看到取视频帧的这一段代码：

  1. Frame *vp;
  2. Frame *sp = NULL;
  3. //is->pictq就是picture queue的意思。读取队列中最后一帧。
  4. vp = frame_queue_peek_last(&is->pictq);

在这里，Frame就是每一帧解码后的图像数据，是直接拿去显示的。而is->pictq就是VideoState里面的解码后的图像队列FrameQueue。

看一下Frame和FrameQueue

  1. typedef struct Frame {
  2.     AVFrame *frame;//ffmpeg定义的数据结构，里面存着buffer，存着真实的yuv图像数据
  3.     AVSubtitle sub;//字幕数据
  4.     int serial;
  5.     double pts;           /* presentation timestamp for the frame */
  6.     double duration;      /* estimated duration of the frame */
  7.     int64_t pos;          /* byte position of the frame in the input file */
  8. #ifdef FFP_MERGE
  9.     SDL_Texture *bmp;
  10. #else
  11.     SDL_VoutOverlay *bmp;//vout设备
  12. #endif
  13.     int allocated;
  14.     int width;
  15.     int height;
  16.     int format;
  17.     AVRational sar;
  18.     int uploaded;
  19. } Frame;
  21. typedef struct FrameQueue {
  22.     Frame queue[FRAME_QUEUE_SIZE];//数组
  23.     int rindex;//read index。下一个读取的下标
  24.     int windex;//write index。下一个写入的下标
  25.     int size;
  26.     int max_size;
  27.     int keep_last;
  28.     int rindex_shown;
  29.     SDL_mutex *mutex;
  30.     SDL_cond *cond;
  31.     PacketQueue *pktq;//引用的未解码的包队列
  32. } FrameQueue;

然后是看到渲染的这一句：

  1. //显示YUV数据。这个vp是Frame，而这个bmp是bitmap的意思
  2. SDL_VoutDisplayYUVOverlay(ffp->vout, vp->bmp);

这里的意思是将vp->bmp中的数据输送到ffp->vout中。