AddTransceiver

image.png

SessionDescription

image.png
SimulcastDescription中
sendlayers：发送多少路simlucast流
receivelayers: 接收多少路simlucast流
如果一路流开启了Simulcast，则会有三路流，这些流信息保存在StreamParams的rids_中，每一项都是RidDescritiion 。

H264中的Encode函数

H264EncoderImpl::Encode
image.png
encoders长度，默认为1.开启Simulcast，则是3
image.png
OnEncodeImage 将编码后的数据上传到应用层。

  1. int32_t H264EncoderImpl::Encode(
  2.     const VideoFrame& input_frame,
  3.     const std::vector<VideoFrameType>* frame_types) {
  4.   if (encoders_.empty()) {
  5.     ReportError();
  6.     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  7.   }
  8.   if (!encoded_image_callback_) {
  9.     RTC_LOG(LS_WARNING)
  10.         << "InitEncode() has been called, but a callback function "
  11.            "has not been set with RegisterEncodeCompleteCallback()";
  12.     ReportError();
  13.     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
  14.   }
  16.   rtc::scoped_refptr<const I420BufferInterface> frame_buffer =
  17.       input_frame.video_frame_buffer()->ToI420();
  19.   bool send_key_frame = false;
  20.   for (size_t i = 0; i < configurations_.size(); ++i) {
  21.     if (configurations_[i].key_frame_request && configurations_[i].sending) {
  22.       send_key_frame = true;
  23.       break;
  24.     }
  25.   }
  27.   if (!send_key_frame && frame_types) {
  28.     for (size_t i = 0; i < configurations_.size(); ++i) {
  29.       const size_t simulcast_idx =
  30.           static_cast<size_t>(configurations_[i].simulcast_idx);
  31.       if (configurations_[i].sending && simulcast_idx < frame_types->size() &&
  32.           (*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey) {
  33.         send_key_frame = true;
  34.         break;
  35.       }
  36.     }
  37.   }
  39.   RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width());
  40.   RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height());
  42.   // Encode image for each layer.
  43.   for (size_t i = 0; i < encoders_.size(); ++i) {
  44.     // EncodeFrame input.
  45.     pictures_[i] = {0};
  46.     pictures_[i].iPicWidth = configurations_[i].width;
  47.     pictures_[i].iPicHeight = configurations_[i].height;
  48.     pictures_[i].iColorFormat = EVideoFormatType::videoFormatI420;
  49.     pictures_[i].uiTimeStamp = input_frame.ntp_time_ms();
  50.     // Downscale images on second and ongoing layers.
  51.     if (i == 0) {
  52.       pictures_[i].iStride[0] = frame_buffer->StrideY();
  53.       pictures_[i].iStride[1] = frame_buffer->StrideU();
  54.       pictures_[i].iStride[2] = frame_buffer->StrideV();
  55.       pictures_[i].pData[0] = const_cast<uint8_t*>(frame_buffer->DataY());
  56.       pictures_[i].pData[1] = const_cast<uint8_t*>(frame_buffer->DataU());
  57.       pictures_[i].pData[2] = const_cast<uint8_t*>(frame_buffer->DataV());
  58.     } else {
  59.       pictures_[i].iStride[0] = downscaled_buffers_[i - 1]->StrideY();
  60.       pictures_[i].iStride[1] = downscaled_buffers_[i - 1]->StrideU();
  61.       pictures_[i].iStride[2] = downscaled_buffers_[i - 1]->StrideV();
  62.       pictures_[i].pData[0] =
  63.           const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataY());
  64.       pictures_[i].pData[1] =
  65.           const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataU());
  66.       pictures_[i].pData[2] =
  67.           const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataV());
  68.       // Scale the image down a number of times by downsampling factor.
  69.       libyuv::I420Scale(pictures_[i - 1].pData[0], pictures_[i - 1].iStride[0],
  70.                         pictures_[i - 1].pData[1], pictures_[i - 1].iStride[1],
  71.                         pictures_[i - 1].pData[2], pictures_[i - 1].iStride[2],
  72.                         configurations_[i - 1].width,
  73.                         configurations_[i - 1].height, pictures_[i].pData[0],
  74.                         pictures_[i].iStride[0], pictures_[i].pData[1],
  75.                         pictures_[i].iStride[1], pictures_[i].pData[2],
  76.                         pictures_[i].iStride[2], configurations_[i].width,
  77.                         configurations_[i].height, libyuv::kFilterBilinear);
  78.     }
  80.     if (!configurations_[i].sending) {
  81.       continue;
  82.     }
  83.     if (frame_types != nullptr) {
  84.       // Skip frame?
  85.       if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) {
  86.         continue;
  87.       }
  88.     }
  89.     if (send_key_frame) {
  90.       // API doc says ForceIntraFrame(false) does nothing, but calling this
  91.       // function forces a key frame regardless of the |bIDR| argument's value.
  92.       // (If every frame is a key frame we get lag/delays.)
  93.       encoders_[i]->ForceIntraFrame(true);
  94.       configurations_[i].key_frame_request = false;
  95.     }
  96.     // EncodeFrame output.
  97.     SFrameBSInfo info;
  98.     memset(&info, 0, sizeof(SFrameBSInfo));
  100.     // Encode!
  101.     int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info);
  102.     if (enc_ret != 0) {
  103.       RTC_LOG(LS_ERROR)
  104.           << "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret
  105.           << ".";
  106.       ReportError();
  107.       return WEBRTC_VIDEO_CODEC_ERROR;
  108.     }
  110.     encoded_images_[i]._encodedWidth = configurations_[i].width;
  111.     encoded_images_[i]._encodedHeight = configurations_[i].height;
  112.     encoded_images_[i].SetTimestamp(input_frame.timestamp());
  113.     encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
  114.     encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx);
  116.     // Split encoded image up into fragments. This also updates
  117.     // |encoded_image_|.
  118.     RtpFragmentize(&encoded_images_[i], &info);
  120.     // Encoder can skip frames to save bandwidth in which case
  121.     // |encoded_images_[i]._length| == 0.
  122.     if (encoded_images_[i].size() > 0) {
  123.       // Parse QP.
  124.       h264_bitstream_parser_.ParseBitstream(encoded_images_[i]);
  125.       encoded_images_[i].qp_ =
  126.           h264_bitstream_parser_.GetLastSliceQp().value_or(-1);
  128.       // Deliver encoded image.
  129.       CodecSpecificInfo codec_specific;
  130.       codec_specific.codecType = kVideoCodecH264;
  131.       codec_specific.codecSpecific.H264.packetization_mode =
  132.           packetization_mode_;
  133.       codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
  134.       codec_specific.codecSpecific.H264.idr_frame =
  135.           info.eFrameType == videoFrameTypeIDR;
  136.       codec_specific.codecSpecific.H264.base_layer_sync = false;
  137.       if (configurations_[i].num_temporal_layers > 1) {
  138.         const uint8_t tid = info.sLayerInfo[0].uiTemporalId;
  139.         codec_specific.codecSpecific.H264.temporal_idx = tid;
  140.         codec_specific.codecSpecific.H264.base_layer_sync =
  141.             tid > 0 && tid < tl0sync_limit_[i];
  142.         if (codec_specific.codecSpecific.H264.base_layer_sync) {
  143.           tl0sync_limit_[i] = tid;
  144.         }
  145.         if (tid == 0) {
  146.           tl0sync_limit_[i] = configurations_[i].num_temporal_layers;
  147.         }
  148.       }
  149.       encoded_image_callback_->OnEncodedImage(encoded_images_[i],
  150.                                               &codec_specific);
  151.     }
  152.   }
  153.   return WEBRTC_VIDEO_CODEC_OK;
  154. }

VP8中的vpx_codec_encode函数

image.png
H:\webrtc-20210315\webrtc-20210315\webrtc\webrtc-checkout\src\third_party\libvpx\source\libvpx\vpx\src\vpx_encoder.c

  2. vpx_codec_err_t vpx_codec_encode(vpx_codec_ctx_t *ctx, const vpx_image_t *img,
  3.                                  vpx_codec_pts_t pts, unsigned long duration,
  4.                                  vpx_enc_frame_flags_t flags,
  5.                                  unsigned long deadline) {
  6.   vpx_codec_err_t res = VPX_CODEC_OK;
  8.   if (!ctx || (img && !duration))
  9.     res = VPX_CODEC_INVALID_PARAM;
  10.   else if (!ctx->iface || !ctx->priv)
  11.     res = VPX_CODEC_ERROR;
  12.   else if (!(ctx->iface->caps & VPX_CODEC_CAP_ENCODER))
  13.     res = VPX_CODEC_INCAPABLE;
  14.   else {
  15.     unsigned int num_enc = ctx->priv->enc.total_encoders;
  17.     /* Execute in a normalized floating point environment, if the platform
  18.      * requires it.
  19.      */
  20.     FLOATING_POINT_INIT();
  22.     if (num_enc == 1)
  23.       res = ctx->iface->enc.encode(get_alg_priv(ctx), img, pts, duration, flags,
  24.                                    deadline);
  25.     else {
  26.       /* Multi-resolution encoding:
  27.        * Encode multi-levels in reverse order. For example,
  28.        * if mr_total_resolutions = 3, first encode level 2,
  29.        * then encode level 1, and finally encode level 0.
  30.        */
  31.       int i;
  33.       ctx += num_enc - 1;
  34.       if (img) img += num_enc - 1;
  36.       for (i = num_enc - 1; i >= 0; i--) {
  37.         if ((res = ctx->iface->enc.encode(get_alg_priv(ctx), img, pts, duration,
  38.                                           flags, deadline)))
  39.           break;
  41.         ctx--;
  42.         if (img) img--;
  43.       }
  44.       ctx++;
  45.     }
  47.     FLOATING_POINT_RESTORE();
  48.   }
  50.   return SAVE_STATUS(ctx, res);
  51. }

Simulcast还需要SFU服务器的协助，将各路流发送到不同网络情况的PC端