image.png
通过found_frames可以知道那些帧是完整的帧。

找到潜在的新帧

image.png
1、获取当前包，和前面一个包
2、判断当前包是否为空，是否不存在，和是否是该帧的第一个包
3、判断前一个包是否存在（是否是连续包），前一个包跟当前包有没有关系seq_num，判断前一个包和当前包是否是在同一个帧。如果前面包是连续的，则可能存在新包。

找到帧的起始包

image.png

将属于同一帧的包都找出来

image.png

源码注释

  1. std::vector<std::unique_ptr<PacketBuffer::Packet>> PacketBuffer::FindFrames(
  2.     uint16_t seq_num) {
  3.   std::vector<std::unique_ptr<PacketBuffer::Packet>> found_frames;
  4.   for (size_t i = 0; i < buffer_.size() && PotentialNewFrame(seq_num); ++i) {
  5.     // 获取该包在buffer_中的位置
  6.     size_t index = seq_num % buffer_.size();
  7.     buffer_[index]->continuous = true;
  9.     // If all packets of the frame is continuous, find the first packet of the
  10.     // frame and add all packets of the frame to the returned packets.
  11.      // 判断这个包，是否是这个帧的最后一个包。
  12.     if (buffer_[index]->is_last_packet_in_frame()) {
  13.       uint16_t start_seq_num = seq_num;
  15.       // Find the start index by searching backward until the packet with
  16.       // the |frame_begin| flag is set.
  17.       int start_index = index;
  18.       size_t tested_packets = 0;
  19.       int64_t frame_timestamp = buffer_[start_index]->timestamp;
  21.       // Identify H.264 keyframes by means of SPS, PPS, and IDR.
  22.       bool is_h264 = buffer_[start_index]->codec() == kVideoCodecH264;
  23.       bool has_h264_sps = false;
  24.       bool has_h264_pps = false;
  25.       bool has_h264_idr = false;
  26.       bool is_h264_keyframe = false;
  27.       int idr_width = -1;
  28.       int idr_height = -1;
  29.       while (true) {
  30.         ++tested_packets;
  32.         if (!is_h264 && buffer_[start_index]->is_first_packet_in_frame())
  33.           break;
  35.         if (is_h264) {
  36.           const auto* h264_header = absl::get_if<RTPVideoHeaderH264>(
  37.               &buffer_[start_index]->video_header.video_type_header);
  38.           if (!h264_header || h264_header->nalus_length >= kMaxNalusPerPacket)
  39.             return found_frames;
  41.           for (size_t j = 0; j < h264_header->nalus_length; ++j) {
  42.             if (h264_header->nalus[j].type == H264::NaluType::kSps) {
  43.               has_h264_sps = true;
  44.             } else if (h264_header->nalus[j].type == H264::NaluType::kPps) {
  45.               has_h264_pps = true;
  46.             } else if (h264_header->nalus[j].type == H264::NaluType::kIdr) {
  47.               has_h264_idr = true;
  48.             }
  49.           }
  50.           if ((sps_pps_idr_is_h264_keyframe_ && has_h264_idr && has_h264_sps &&
  51.                has_h264_pps) ||
  52.               (!sps_pps_idr_is_h264_keyframe_ && has_h264_idr)) {
  53.             is_h264_keyframe = true;
  54.             // Store the resolution of key frame which is the packet with
  55.             // smallest index and valid resolution; typically its IDR or SPS
  56.             // packet; there may be packet preceeding this packet, IDR's
  57.             // resolution will be applied to them.
  58.             if (buffer_[start_index]->width() > 0 &&
  59.                 buffer_[start_index]->height() > 0) {
  60.               idr_width = buffer_[start_index]->width();
  61.               idr_height = buffer_[start_index]->height();
  62.             }
  63.           }
  64.         }
  66.         if (tested_packets == buffer_.size())
  67.           break;
  69.         start_index = start_index > 0 ? start_index - 1 : buffer_.size() - 1;
  71.         // In the case of H264 we don't have a frame_begin bit (yes,
  72.         // |frame_begin| might be set to true but that is a lie). So instead
  73.         // we traverese backwards as long as we have a previous packet and
  74.         // the timestamp of that packet is the same as this one. This may cause
  75.         // the PacketBuffer to hand out incomplete frames.
  76.         // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=7106
  77.         if (is_h264 && (buffer_[start_index] == nullptr ||
  78.                         buffer_[start_index]->timestamp != frame_timestamp)) {
  79.           break;
  80.         }
  82.         --start_seq_num;
  83.       }
  85.       if (is_h264) {
  86.         // Warn if this is an unsafe frame.
  87.         if (has_h264_idr && (!has_h264_sps || !has_h264_pps)) {
  88.           RTC_LOG(LS_WARNING)
  89.               << "Received H.264-IDR frame "
  90.                  "(SPS: "
  91.               << has_h264_sps << ", PPS: " << has_h264_pps << "). Treating as "
  92.               << (sps_pps_idr_is_h264_keyframe_ ? "delta" : "key")
  93.               << " frame since WebRTC-SpsPpsIdrIsH264Keyframe is "
  94.               << (sps_pps_idr_is_h264_keyframe_ ? "enabled." : "disabled");
  95.         }
  97.         // Now that we have decided whether to treat this frame as a key frame
  98.         // or delta frame in the frame buffer, we update the field that
  99.         // determines if the RtpFrameObject is a key frame or delta frame.
  100.         const size_t first_packet_index = start_seq_num % buffer_.size();
  101.         if (is_h264_keyframe) {
  102.           buffer_[first_packet_index]->video_header.frame_type =
  103.               VideoFrameType::kVideoFrameKey;
  104.           if (idr_width > 0 && idr_height > 0) {
  105.             // IDR frame was finalized and we have the correct resolution for
  106.             // IDR; update first packet to have same resolution as IDR.
  107.             buffer_[first_packet_index]->video_header.width = idr_width;
  108.             buffer_[first_packet_index]->video_header.height = idr_height;
  109.           }
  110.         } else {
  111.           buffer_[first_packet_index]->video_header.frame_type =
  112.               VideoFrameType::kVideoFrameDelta;
  113.         }
  115.         // If this is not a keyframe, make sure there are no gaps in the packet
  116.         // sequence numbers up until this point.
  117.         if (!is_h264_keyframe && missing_packets_.upper_bound(start_seq_num) !=
  118.                                      missing_packets_.begin()) {
  119.           return found_frames;
  120.         }
  121.       }
  123.       const uint16_t end_seq_num = seq_num + 1;
  124.       // Use uint16_t type to handle sequence number wrap around case.
  125.       uint16_t num_packets = end_seq_num - start_seq_num;
  126.       found_frames.reserve(found_frames.size() + num_packets);
  127.       for (uint16_t i = start_seq_num; i != end_seq_num; ++i) {
  128.         std::unique_ptr<Packet>& packet = buffer_[i % buffer_.size()];
  129.         RTC_DCHECK(packet);
  130.         RTC_DCHECK_EQ(i, packet->seq_num);
  131.         // Ensure frame boundary flags are properly set.
  132.         packet->video_header.is_first_packet_in_frame = (i == start_seq_num);
  133.         packet->video_header.is_last_packet_in_frame = (i == seq_num);
  134.         found_frames.push_back(std::move(packet));
  135.       }
  137.       missing_packets_.erase(missing_packets_.begin(),
  138.                              missing_packets_.upper_bound(seq_num));
  139.     }
  140.     ++seq_num;
  141.   }
  142.   return found_frames;
  143. }