Nack调用栈

image.png
数据送解码器的图
9-7 Channel-Stream与编解码器(重要)
10-18 视频解码渲染（重点）
RtpDemuxer::OnRtpPacket 收到的如果是Rtx包，则会调用RtxReceiveStream::OnRtpPacket，否则会直接调用 RtpVideoStreamReceiver::OnRtpPacket.
RtxReceiveStream::OnRtpPacket处理完后，也会调用RtpVideoStreamReceiver::OnRtpPacket。

OnReceivedPacket

说明

image.png
正常收到rtp会使用fec和nack，这里为了避免fec干扰到nack处理，将关闭fec red。
当前分析的是视频的rtp包。

源码

  1. void RtpVideoStreamReceiver::ReceivePacket(const RtpPacketReceived& packet) {
  2.   if (packet.payload_size() == 0) {
  3.     // Padding or keep-alive packet.
  4.     // TODO(nisse): Could drop empty packets earlier, but need to figure out how
  5.     // they should be counted in stats.
  6.     NotifyReceiverOfEmptyPacket(packet.SequenceNumber());
  7.     return;
  8.   }
  9.   if (packet.PayloadType() == config_.rtp.red_payload_type) {
  10.     ParseAndHandleEncapsulatingHeader(packet);
  11.     return;
  12.   }
  14.   const auto type_it = payload_type_map_.find(packet.PayloadType());
  15.   if (type_it == payload_type_map_.end()) {
  16.     return;
  17.   }
  18.   absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
  19.       type_it->second->Parse(packet.PayloadBuffer());
  20.   if (parsed_payload == absl::nullopt) {
  21.     RTC_LOG(LS_WARNING) << "Failed parsing payload.";
  22.     return;
  23.   }
  25.   OnReceivedPayloadData(std::move(parsed_payload->video_payload), packet,
  26.                         parsed_payload->video_header);
  27. }

OnReceivedPayloadData

说明

RtpVideoStreamReceiver::OnReceivedPayloadData
image.png
这里其实主要是三个if的分支，都是比较复杂和重要的。当前说的是第三个分支，检测丢包。 std::uniqueptr nack_module;
image.png

源码

  1. void RtpVideoStreamReceiver::OnReceivedPayloadData(
  2.     rtc::CopyOnWriteBuffer codec_payload,
  3.     const RtpPacketReceived& rtp_packet,
  4.     const RTPVideoHeader& video) {
  5.   RTC_DCHECK_RUN_ON(&worker_task_checker_);
  6.   auto packet = std::make_unique<video_coding::PacketBuffer::Packet>(
  7.       rtp_packet, video, clock_->TimeInMilliseconds());
  9.   // Try to extrapolate absolute capture time if it is missing.
  10.   packet->packet_info.set_absolute_capture_time(
  11.       absolute_capture_time_receiver_.OnReceivePacket(
  12.           AbsoluteCaptureTimeReceiver::GetSource(packet->packet_info.ssrc(),
  13.                                                  packet->packet_info.csrcs()),
  14.           packet->packet_info.rtp_timestamp(),
  15.           // Assume frequency is the same one for all video frames.
  16.           kVideoPayloadTypeFrequency,
  17.           packet->packet_info.absolute_capture_time()));
  19.   RTPVideoHeader& video_header = packet->video_header;
  20.   video_header.rotation = kVideoRotation_0;
  21.   video_header.content_type = VideoContentType::UNSPECIFIED;
  22.   video_header.video_timing.flags = VideoSendTiming::kInvalid;
  23.   video_header.is_last_packet_in_frame |= rtp_packet.Marker();
  25.   if (const auto* vp9_header =
  26.           absl::get_if<RTPVideoHeaderVP9>(&video_header.video_type_header)) {
  27.     video_header.is_last_packet_in_frame |= vp9_header->end_of_frame;
  28.     video_header.is_first_packet_in_frame |= vp9_header->beginning_of_frame;
  29.   }
  31.   rtp_packet.GetExtension<VideoOrientation>(&video_header.rotation);
  32.   rtp_packet.GetExtension<VideoContentTypeExtension>(
  33.       &video_header.content_type);
  34.   rtp_packet.GetExtension<VideoTimingExtension>(&video_header.video_timing);
  35.   if (forced_playout_delay_max_ms_ && forced_playout_delay_min_ms_) {
  36.     video_header.playout_delay.max_ms = *forced_playout_delay_max_ms_;
  37.     video_header.playout_delay.min_ms = *forced_playout_delay_min_ms_;
  38.   } else {
  39.     rtp_packet.GetExtension<PlayoutDelayLimits>(&video_header.playout_delay);
  40.   }
  42.   ParseGenericDependenciesResult generic_descriptor_state =
  43.       ParseGenericDependenciesExtension(rtp_packet, &video_header);
  44.   if (generic_descriptor_state == kDropPacket)
  45.     return;
  47.   // Color space should only be transmitted in the last packet of a frame,
  48.   // therefore, neglect it otherwise so that last_color_space_ is not reset by
  49.   // mistake.
  50.   if (video_header.is_last_packet_in_frame) {
  51.     video_header.color_space = rtp_packet.GetExtension<ColorSpaceExtension>();
  52.     if (video_header.color_space ||
  53.         video_header.frame_type == VideoFrameType::kVideoFrameKey) {
  54.       // Store color space since it's only transmitted when changed or for key
  55.       // frames. Color space will be cleared if a key frame is transmitted
  56.       // without color space information.
  57.       last_color_space_ = video_header.color_space;
  58.     } else if (last_color_space_) {
  59.       video_header.color_space = last_color_space_;
  60.     }
  61.   }
  63.   if (loss_notification_controller_) {
  64.     if (rtp_packet.recovered()) {
  65.       // TODO(bugs.webrtc.org/10336): Implement support for reordering.
  66.       RTC_LOG(LS_INFO)
  67.           << "LossNotificationController does not support reordering.";
  68.     } else if (generic_descriptor_state == kNoGenericDescriptor) {
  69.       RTC_LOG(LS_WARNING) << "LossNotificationController requires generic "
  70.                              "frame descriptor, but it is missing.";
  71.     } else {
  72.       if (video_header.is_first_packet_in_frame) {
  73.         RTC_DCHECK(video_header.generic);
  74.         LossNotificationController::FrameDetails frame;
  75.         frame.is_keyframe =
  76.             video_header.frame_type == VideoFrameType::kVideoFrameKey;
  77.         frame.frame_id = video_header.generic->frame_id;
  78.         frame.frame_dependencies = video_header.generic->dependencies;
  79.         loss_notification_controller_->OnReceivedPacket(
  80.             rtp_packet.SequenceNumber(), &frame);
  81.       } else {
  82.         loss_notification_controller_->OnReceivedPacket(
  83.             rtp_packet.SequenceNumber(), nullptr);
  84.       }
  85.     }
  86.   }
  88.   if (nack_module_) {
  89.     const bool is_keyframe =
  90.         video_header.is_first_packet_in_frame &&
  91.         video_header.frame_type == VideoFrameType::kVideoFrameKey;
  93.     packet->times_nacked = nack_module_->OnReceivedPacket(
  94.         rtp_packet.SequenceNumber(), is_keyframe, rtp_packet.recovered());
  95.   } else {
  96.     packet->times_nacked = -1;
  97.   }
  99.   if (codec_payload.size() == 0) {
  100.     NotifyReceiverOfEmptyPacket(packet->seq_num);
  101.     rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
  102.     return;
  103.   }
  105.   if (packet->codec() == kVideoCodecH264) {
  106.     // Only when we start to receive packets will we know what payload type
  107.     // that will be used. When we know the payload type insert the correct
  108.     // sps/pps into the tracker.
  109.     if (packet->payload_type != last_payload_type_) {
  110.       last_payload_type_ = packet->payload_type;
  111.       InsertSpsPpsIntoTracker(packet->payload_type);
  112.     }
  114.     video_coding::H264SpsPpsTracker::FixedBitstream fixed =
  115.         tracker_.CopyAndFixBitstream(
  116.             rtc::MakeArrayView(codec_payload.cdata(), codec_payload.size()),
  117.             &packet->video_header);
  119.     switch (fixed.action) {
  120.       case video_coding::H264SpsPpsTracker::kRequestKeyframe:
  121.         rtcp_feedback_buffer_.RequestKeyFrame();
  122.         rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
  123.         ABSL_FALLTHROUGH_INTENDED;
  124.       case video_coding::H264SpsPpsTracker::kDrop:
  125.         return;
  126.       case video_coding::H264SpsPpsTracker::kInsert:
  127.         packet->video_payload = std::move(fixed.bitstream);
  128.         break;
  129.     }
  131.   } else {
  132.     packet->video_payload = std::move(codec_payload);
  133.   }
  135.   rtcp_feedback_buffer_.SendBufferedRtcpFeedback();
  136.   frame_counter_.Add(packet->timestamp);
  137.   OnInsertedPacket(packet_buffer_.InsertPacket(std::move(packet)));
  138. }