总体流程image.png

代码分析

AudioTransportImpl::RecordedDataIsAvailable

  1. // Not used in Chromium. Process captured audio and distribute to all sending
  2. // streams, and try to do this at the lowest possible sample rate.
  3. int32_t AudioTransportImpl::RecordedDataIsAvailable(
  4.     const void* audio_data,
  5.     const size_t number_of_frames,
  6.     const size_t bytes_per_sample,
  7.     const size_t number_of_channels,
  8.     const uint32_t sample_rate,
  9.     const uint32_t audio_delay_milliseconds,
  10.     const int32_t /*clock_drift*/,
  11.     const uint32_t /*volume*/,
  12.     const bool key_pressed,
  13.     uint32_t& /*new_mic_volume*/) {  // NOLINT: to avoid changing APIs
  14.   RTC_DCHECK(audio_data);
  15.   RTC_DCHECK_GE(number_of_channels, 1);
  16.   RTC_DCHECK_LE(number_of_channels, 2);
  17.   RTC_DCHECK_EQ(2 * number_of_channels, bytes_per_sample);
  18.   RTC_DCHECK_GE(sample_rate, AudioProcessing::NativeRate::kSampleRate8kHz);
  19.   // 100 = 1 second / data duration (10 ms).
  20.   RTC_DCHECK_EQ(number_of_frames * 100, sample_rate);
  21.   RTC_DCHECK_LE(bytes_per_sample * number_of_frames * number_of_channels,
  22.                 AudioFrame::kMaxDataSizeBytes);
  24.   int send_sample_rate_hz = 0;
  25.   size_t send_num_channels = 0;
  26.   bool swap_stereo_channels = false;
  27.   {
  28.     MutexLock lock(&capture_lock_);
  29.     send_sample_rate_hz = send_sample_rate_hz_;
  30.     send_num_channels = send_num_channels_;
  31.     swap_stereo_channels = swap_stereo_channels_;
  32.   }
  34.   std::unique_ptr<AudioFrame> audio_frame(new AudioFrame());
  35.   InitializeCaptureFrame(sample_rate, send_sample_rate_hz, number_of_channels,
  36.                          send_num_channels, audio_frame.get());
  37.   voe::RemixAndResample(static_cast<const int16_t*>(audio_data),
  38.                         number_of_frames, number_of_channels, sample_rate,
  39.                         &capture_resampler_, audio_frame.get());
  40.   ProcessCaptureFrame(audio_delay_milliseconds, key_pressed,
  41.                       swap_stereo_channels, audio_processing_,
  42.                       audio_frame.get());
  44.   // Typing detection (utilizes the APM/VAD decision). We let the VAD determine
  45.   // if we're using this feature or not.
  46.   // TODO(solenberg): GetConfig() takes a lock. Work around that.
  47.   bool typing_detected = false;
  48.   if (audio_processing_ &&
  49.       audio_processing_->GetConfig().voice_detection.enabled) {
  50.     if (audio_frame->vad_activity_ != AudioFrame::kVadUnknown) {
  51.       bool vad_active = audio_frame->vad_activity_ == AudioFrame::kVadActive;
  52.       typing_detected = typing_detection_.Process(key_pressed, vad_active);
  53.     }
  54.   }
  56.   // Copy frame and push to each sending stream. The copy is required since an
  57.   // encoding task will be posted internally to each stream.
  58.   {
  59.     MutexLock lock(&capture_lock_);
  60.     typing_noise_detected_ = typing_detected;
  61.   }
  63.   RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0);
  64.   if (async_audio_processing_)
  65.     async_audio_processing_->Process(std::move(audio_frame));
  66.   else
  67.     SendProcessedData(std::move(audio_frame));
  69.   return 0;
  70. }

image.png
—》

AudioTransportImpl::SendProcessedData

  1. void AudioTransportImpl::SendProcessedData(
  2.     std::unique_ptr<AudioFrame> audio_frame) {
  3.   RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0);
  4.   MutexLock lock(&capture_lock_);
  5.   if (audio_senders_.empty())
  6.     return;
  8.   auto it = audio_senders_.begin();
  9.   while (++it != audio_senders_.end()) {
  10.     auto audio_frame_copy = std::make_unique<AudioFrame>();
  11.     audio_frame_copy->CopyFrom(*audio_frame);
  12.     (*it)->SendAudioData(std::move(audio_frame_copy));
  13.   }
  14.   // Send the original frame to the first stream w/o copying.
  15.   (*audio_senders_.begin())->SendAudioData(std::move(audio_frame));
  16. }

—》

AudioSendStream::SendAudioData

  1. void AudioSendStream::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) {
  2.   RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_);
  3.   RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0);
  4.   double duration = static_cast<double>(audio_frame->samples_per_channel_) /
  5.                     audio_frame->sample_rate_hz_;
  6.   {
  7.     // Note: SendAudioData() passes the frame further down the pipeline and it
  8.     // may eventually get sent. But this method is invoked even if we are not
  9.     // connected, as long as we have an AudioSendStream (created as a result of
  10.     // an O/A exchange). This means that we are calculating audio levels whether
  11.     // or not we are sending samples.
  12.     // TODO(https://crbug.com/webrtc/10771): All "media-source" related stats
  13.     // should move from send-streams to the local audio sources or tracks; a
  14.     // send-stream should not be required to read the microphone audio levels.
  15.     MutexLock lock(&audio_level_lock_);
  16.     audio_level_.ComputeLevel(*audio_frame, duration);
  17.   }
  18.   channel_send_->ProcessAndEncodeAudio(std::move(audio_frame));
  19. }

—》开始跟编码层打交道了， channelsend->ProcessAndEncodeAudio(std::move(audio_frame));

ChannelSend::ProcessAndEncodeAudio

  1. void ChannelSend::ProcessAndEncodeAudio(
  2.     std::unique_ptr<AudioFrame> audio_frame) {
  3.   RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_);
  4.   RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0);
  5.   RTC_DCHECK_LE(audio_frame->num_channels_, 8);
  7.   // Profile time between when the audio frame is added to the task queue and
  8.   // when the task is actually executed.
  9.   audio_frame->UpdateProfileTimeStamp();
  10.     // 添加一个任务到encoder_queue_，然后触发编码线程来处理
  11.   encoder_queue_.PostTask(
  12.       [this, audio_frame = std::move(audio_frame)]() mutable {
  13.         RTC_DCHECK_RUN_ON(&encoder_queue_);
  14.         if (!encoder_queue_is_active_) {
  15.           if (fixing_timestamp_stall_) {
  16.             _timeStamp +=
  17.                 static_cast<uint32_t>(audio_frame->samples_per_channel_);
  18.           }
  19.           return;
  20.         }
  21.         // Measure time between when the audio frame is added to the task queue
  22.         // and when the task is actually executed. Goal is to keep track of
  23.         // unwanted extra latency added by the task queue.
  24.         RTC_HISTOGRAM_COUNTS_10000("WebRTC.Audio.EncodingTaskQueueLatencyMs",
  25.                                    audio_frame->ElapsedProfileTimeMs());
  27.         // 根据输入mute属性，修改数据
  28.          bool is_muted = InputMute();
  29.         AudioFrameOperations::Mute(audio_frame.get(), previous_frame_muted_,
  30.                                    is_muted);
  32.         if (_includeAudioLevelIndication) {
  33.           size_t length =
  34.               audio_frame->samples_per_channel_ * audio_frame->num_channels_;
  35.           RTC_CHECK_LE(length, AudioFrame::kMaxDataSizeBytes);
  36.           if (is_muted && previous_frame_muted_) {
  37.             rms_level_.AnalyzeMuted(length);
  38.           } else {
  39.             rms_level_.Analyze(
  40.                 rtc::ArrayView<const int16_t>(audio_frame->data(), length));
  41.           }
  42.         }
  43.         previous_frame_muted_ = is_muted;
  45.         // Add 10ms of raw (PCM) audio data to the encoder @ 32kHz.
  47.         // The ACM resamples internally.
  48.         audio_frame->timestamp_ = _timeStamp;
  49.         // This call will trigger AudioPacketizationCallback::SendData if
  50.         // encoding is done and payload is ready for packetization and
  51.         // transmission. Otherwise, it will return without invoking the
  52.         // callback.
  53.         if (audio_coding_->Add10MsData(*audio_frame) < 0) {
  54.           RTC_DLOG(LS_ERROR) << "ACM::Add10MsData() failed.";
  55.           return;
  56.         }
  58.         _timeStamp += static_cast<uint32_t>(audio_frame->samples_per_channel_);
  59.       });
  60. }

—》

AudioCodingModuleImpl::Add10MsData

  1. // Add 10MS of raw (PCM) audio data to the encoder.
  2. int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
  3.   MutexLock lock(&acm_mutex_);
  4.   int r = Add10MsDataInternal(audio_frame, &input_data_);
  5.   // TODO(bugs.webrtc.org/10739): add dcheck that
  6.   // |audio_frame.absolute_capture_timestamp_ms()| always has a value.
  7.   return r < 0
  8.              ? r
  9.              : Encode(input_data_, audio_frame.absolute_capture_timestamp_ms());
  10. }

—》调用Encode对数据开始编码
AudioCodingModuleImpl::Encode

  1. int32_t AudioCodingModuleImpl::Encode(
  2.     const InputData& input_data,
  3.     absl::optional<int64_t> absolute_capture_timestamp_ms) {
  4.   // TODO(bugs.webrtc.org/10739): add dcheck that
  5.   // |audio_frame.absolute_capture_timestamp_ms()| always has a value.
  6.   AudioEncoder::EncodedInfo encoded_info;
  7.   uint8_t previous_pltype;
  9.   // Check if there is an encoder before.
  10.   if (!HaveValidEncoder("Process"))
  11.     return -1;
  13.   if (!first_frame_) {
  14.     RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
  15.         << "Time should not move backwards";
  16.   }
  18.   // Scale the timestamp to the codec's RTP timestamp rate.
  19.   uint32_t rtp_timestamp =
  20.       first_frame_
  21.           ? input_data.input_timestamp
  22.           : last_rtp_timestamp_ +
  23.                 rtc::dchecked_cast<uint32_t>(rtc::CheckedDivExact(
  24.                     int64_t{input_data.input_timestamp - last_timestamp_} *
  25.                         encoder_stack_->RtpTimestampRateHz(),
  26.                     int64_t{encoder_stack_->SampleRateHz()}));
  28.   last_timestamp_ = input_data.input_timestamp;
  29.   last_rtp_timestamp_ = rtp_timestamp;
  30.   first_frame_ = false;
  32.   // Clear the buffer before reuse - encoded data will get appended.
  33.   encode_buffer_.Clear();
  34.   encoded_info = encoder_stack_->Encode(
  35.       rtp_timestamp,
  36.       rtc::ArrayView<const int16_t>(
  37.           input_data.audio,
  38.           input_data.audio_channel * input_data.length_per_channel),
  39.       &encode_buffer_);
  41.   bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
  42.   // 采集是每10ms，但是编码是20ms，所以需要第二次才会开始编码
  43.   if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
  44.     // Not enough data.
  45.     return 0; // 第一次进来会为0，第一次的数据会缓存到编码器中
  46.   }
  47.   previous_pltype = previous_pltype_;  // Read it while we have the critsect.
  49.   // Log codec type to histogram once every 500 packets.
  50.   if (encoded_info.encoded_bytes == 0) {
  51.     ++number_of_consecutive_empty_packets_;
  52.   } else {
  53.     size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
  54.     codec_histogram_bins_log_[codec_type] +=
  55.         number_of_consecutive_empty_packets_ + 1;
  56.     number_of_consecutive_empty_packets_ = 0;
  57.     if (codec_histogram_bins_log_[codec_type] >= 500) {
  58.       codec_histogram_bins_log_[codec_type] -= 500;
  59.       UpdateCodecTypeHistogram(codec_type);
  60.     }
  61.   }
  63.   AudioFrameType frame_type;
  64.   if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
  65.     frame_type = AudioFrameType::kEmptyFrame;
  66.     encoded_info.payload_type = previous_pltype;
  67.   } else {
  68.     RTC_DCHECK_GT(encode_buffer_.size(), 0);
  69.     frame_type = encoded_info.speech ? AudioFrameType::kAudioFrameSpeech
  70.                                      : AudioFrameType::kAudioFrameCN;
  71.   }
  73.   {
  74.     MutexLock lock(&callback_mutex_);
  75.     if (packetization_callback_) { //  触发ChannelSend::SendData
  76.       packetization_callback_->SendData(
  77.           frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
  78.           encode_buffer_.data(), encode_buffer_.size(),
  79.           absolute_capture_timestamp_ms.value_or(-1));
  80.     }
  81.   }
  82.   previous_pltype_ = encoded_info.payload_type;
  83.   return static_cast<int32_t>(encode_buffer_.size());
  84. }

—》

ChannelSend::SendData

  1. int32_t
  2. (AudioFrameType frameType,
  3.                               uint8_t payloadType,
  4.                               uint32_t rtp_timestamp,
  5.                               const uint8_t* payloadData,
  6.                               size_t payloadSize,
  7.                               int64_t absolute_capture_timestamp_ms) {
  8.   RTC_DCHECK_RUN_ON(&encoder_queue_);
  9.   rtc::ArrayView<const uint8_t> payload(payloadData, payloadSize);
  10.   if (frame_transformer_delegate_) {
  11.     // Asynchronously transform the payload before sending it. After the payload
  12.     // is transformed, the delegate will call SendRtpAudio to send it.
  13.     frame_transformer_delegate_->Transform(
  14.         frameType, payloadType, rtp_timestamp, rtp_rtcp_->StartTimestamp(),
  15.         payloadData, payloadSize, absolute_capture_timestamp_ms,
  16.         rtp_rtcp_->SSRC());
  17.     return 0;
  18.   }
  19.   return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload,
  20.                       absolute_capture_timestamp_ms);
  21. }

—》

ChannelSend::SendRtpAudio

  1. int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType,
  2.                                   uint8_t payloadType,
  3.                                   uint32_t rtp_timestamp,
  4.                                   rtc::ArrayView<const uint8_t> payload,
  5.                                   int64_t absolute_capture_timestamp_ms) {
  6.   if (_includeAudioLevelIndication) {
  7.     // Store current audio level in the RTP sender.
  8.     // The level will be used in combination with voice-activity state
  9.     // (frameType) to add an RTP header extension
  10.     rtp_sender_audio_->SetAudioLevel(rms_level_.Average());
  11.   }
  13.   // E2EE Custom Audio Frame Encryption (This is optional).
  14.   // Keep this buffer around for the lifetime of the send call.
  15.   rtc::Buffer encrypted_audio_payload;
  16.   // We don't invoke encryptor if payload is empty, which means we are to send
  17.   // DTMF, or the encoder entered DTX.
  18.   // TODO(minyue): see whether DTMF packets should be encrypted or not. In
  19.   // current implementation, they are not.
  20.   if (!payload.empty()) {
  21.     if (frame_encryptor_ != nullptr) {
  22.       // TODO(benwright@webrtc.org) - Allocate enough to always encrypt inline.
  23.       // Allocate a buffer to hold the maximum possible encrypted payload.
  24.       size_t max_ciphertext_size = frame_encryptor_->GetMaxCiphertextByteSize(
  25.           cricket::MEDIA_TYPE_AUDIO, payload.size());
  26.       encrypted_audio_payload.SetSize(max_ciphertext_size);
  28.       // Encrypt the audio payload into the buffer.
  29.       size_t bytes_written = 0;
  30.       int encrypt_status = frame_encryptor_->Encrypt(
  31.           cricket::MEDIA_TYPE_AUDIO, rtp_rtcp_->SSRC(),
  32.           /*additional_data=*/nullptr, payload, encrypted_audio_payload,
  33.           &bytes_written);
  34.       if (encrypt_status != 0) {
  35.         RTC_DLOG(LS_ERROR)
  36.             << "Channel::SendData() failed encrypt audio payload: "
  37.             << encrypt_status;
  38.         return -1;
  39.       }
  40.       // Resize the buffer to the exact number of bytes actually used.
  41.       encrypted_audio_payload.SetSize(bytes_written);
  42.       // Rewrite the payloadData and size to the new encrypted payload.
  43.       payload = encrypted_audio_payload;
  44.     } else if (crypto_options_.sframe.require_frame_encryption) {
  45.       RTC_DLOG(LS_ERROR) << "Channel::SendData() failed sending audio payload: "
  46.                             "A frame encryptor is required but one is not set.";
  47.       return -1;
  48.     }
  49.   }
  51.   // Push data from ACM to RTP/RTCP-module to deliver audio frame for
  52.   // packetization.
  53.   if (!rtp_rtcp_->OnSendingRtpFrame(rtp_timestamp,
  54.                                     // Leaving the time when this frame was
  55.                                     // received from the capture device as
  56.                                     // undefined for voice for now.
  57.                                     -1, payloadType,
  58.                                     /*force_sender_report=*/false)) {
  59.     return -1;
  60.   }
  62.   // RTCPSender has it's own copy of the timestamp offset, added in
  63.   // RTCPSender::BuildSR, hence we must not add the in the offset for the above
  64.   // call.
  65.   // TODO(nisse): Delete RTCPSender:timestamp_offset_, and see if we can confine
  66.   // knowledge of the offset to a single place.
  68.   // This call will trigger Transport::SendPacket() from the RTP/RTCP module.
  69.   if (!rtp_sender_audio_->SendAudio(
  70.           frameType, payloadType, rtp_timestamp + rtp_rtcp_->StartTimestamp(),
  71.           payload.data(), payload.size(), absolute_capture_timestamp_ms)) {
  72.     RTC_DLOG(LS_ERROR)
  73.         << "ChannelSend::SendData() failed to send data to RTP/RTCP module";
  74.     return -1;
  75.   }
  77.   return 0;
  78. }

—》

RTPSenderAudio::SendAudio

  1. bool RTPSenderAudio::SendAudio(AudioFrameType frame_type,
  2.                                int8_t payload_type,
  3.                                uint32_t rtp_timestamp,
  4.                                const uint8_t* payload_data,
  5.                                size_t payload_size,
  6.                                int64_t absolute_capture_timestamp_ms) {
  7. #if RTC_TRACE_EVENTS_ENABLED
  8.   TRACE_EVENT_ASYNC_STEP1("webrtc", "Audio", rtp_timestamp, "Send", "type",
  9.                           FrameTypeToString(frame_type));
  10.   #endif
  13. *****
  14. // 创建一个rtp包，并且填入数据
  15.   std::unique_ptr<RtpPacketToSend> packet = rtp_sender_->AllocatePacket();
  16.   packet->SetMarker(MarkerBit(frame_type, payload_type));
  17.   packet->SetPayloadType(payload_type);
  18.   packet->SetTimestamp(rtp_timestamp);
  19.   packet->set_capture_time_ms(clock_->TimeInMilliseconds());
  20.   // Update audio level extension, if included.
  21.   packet->SetExtension<AudioLevel>(
  22.       frame_type == AudioFrameType::kAudioFrameSpeech, audio_level_dbov);
  24.   // Send absolute capture time periodically in order to optimize and save
  25.   // network traffic. Missing absolute capture times can be interpolated on the
  26.   // receiving end if sending intervals are small enough.
  27.   auto absolute_capture_time = absolute_capture_time_sender_.OnSendPacket(
  28.       AbsoluteCaptureTimeSender::GetSource(packet->Ssrc(), packet->Csrcs()),
  29.       packet->Timestamp(),
  30.       // Replace missing value with 0 (invalid frequency), this will trigger
  31.       // absolute capture time sending.
  32.       encoder_rtp_timestamp_frequency.value_or(0),
  33.       Int64MsToUQ32x32(absolute_capture_timestamp_ms + NtpOffsetMs()),
  34.       /*estimated_capture_clock_offset=*/
  35.       include_capture_clock_offset_ ? absl::make_optional(0) : absl::nullopt);
  36.   if (absolute_capture_time) {
  37.     // It also checks that extension was registered during SDP negotiation. If
  38.     // not then setter won't do anything.
  39.     packet->SetExtension<AbsoluteCaptureTimeExtension>(*absolute_capture_time);
  40.   }
  42.   uint8_t* payload = packet->AllocatePayload(payload_size);
  43.   if (!payload)  // Too large payload buffer.
  44.     return false;
  45.   memcpy(payload, payload_data, payload_size);
  47.   if (!rtp_sender_->AssignSequenceNumber(packet.get()))
  48.     return false;
  50.   {
  51.     MutexLock lock(&send_audio_mutex_);
  52.     last_payload_type_ = payload_type;
  53.   }
  54.   TRACE_EVENT_ASYNC_END2("webrtc", "Audio", rtp_timestamp, "timestamp",
  55.                          packet->Timestamp(), "seqnum",
  56.                          packet->SequenceNumber());
  57.   packet->set_packet_type(RtpPacketMediaType::kAudio);
  58.   packet->set_allow_retransmission(true);
  59.   bool send_result = rtp_sender_->SendToNetwork(std::move(packet));
  60.   if (first_packet_sent_()) {
  61.     RTC_LOG(LS_INFO) << "First audio RTP packet sent to pacer";
  62.   }
  63.   return send_result;
  64. }

—》

RTPSender::SendToNetwork

  1. bool RTPSender::SendToNetwork(std::unique_ptr<RtpPacketToSend> packet) {
  2.   RTC_DCHECK(packet);
  3.   int64_t now_ms = clock_->TimeInMilliseconds();
  5.   auto packet_type = packet->packet_type();
  6.   RTC_CHECK(packet_type) << "Packet type must be set before sending.";
  8.   if (packet->capture_time_ms() <= 0) {
  9.     packet->set_capture_time_ms(now_ms);
  10.   }
  12.   std::vector<std::unique_ptr<RtpPacketToSend>> packets;
  13.   packets.emplace_back(std::move(packet));
  15.   // 将packets插入到pace的队列中
  16.   paced_sender_->EnqueuePackets(std::move(packets));
  18.   return true;
  19. }

—>

RtpPacketSenderProxy::EnqueuePackets

  1. class RtpPacketSenderProxy : public RtpPacketSender {
  2.  public:
  3. void EnqueuePackets(
  4.       std::vector<std::unique_ptr<RtpPacketToSend>> packets) override {
  5.     MutexLock lock(&mutex_);
  6.     rtp_packet_pacer_->EnqueuePackets(std::move(packets));
  7.   }
  8.     ***

—>

PacedSender::EnqueuePackets

  1. void PacedSender::EnqueuePackets(
  2.     std::vector<std::unique_ptr<RtpPacketToSend>> packets) {
  3.   {
  4.     TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("webrtc"),
  5.                  "PacedSender::EnqueuePackets");
  6.     MutexLock lock(&mutex_);
  7.     for (auto& packet : packets) {
  8.       TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("webrtc"),
  9.                    "PacedSender::EnqueuePackets::Loop", "sequence_number",
  10.                    packet->SequenceNumber(), "rtp_timestamp",
  11.                    packet->Timestamp());
  13.       RTC_DCHECK_GE(packet->capture_time_ms(), 0);
  14.       pacing_controller_.EnqueuePacket(std::move(packet));
  15.     }
  16.   }
  17.   MaybeWakupProcessThread();
  18. }

—》

PacingController::EnqueuePacket

H:\webrtc-20210315\webrtc-20210315\webrtc\webrtc-checkout\src\modules\pacing\pacing_controller.cc

  1. void PacingController::EnqueuePacket(std::unique_ptr<RtpPacketToSend> packet) {
  2.   RTC_DCHECK(pacing_bitrate_ > DataRate::Zero())
  3.       << "SetPacingRate must be called before InsertPacket.";
  4.   RTC_CHECK(packet->packet_type());
  5.   // Get priority first and store in temporary, to avoid chance of object being
  6.   // moved before GetPriorityForType() being called.
  7.   const int priority = GetPriorityForType(*packet->packet_type());
  8.   EnqueuePacketInternal(std::move(packet), priority);
  9. }

image.png
—》

PacingController::EnqueuePacketInternal

  1. void PacingController::EnqueuePacketInternal(
  2.     std::unique_ptr<RtpPacketToSend> packet,
  3.     int priority) {
  4.   prober_.OnIncomingPacket(DataSize::Bytes(packet->payload_size()));
  6.   Timestamp now = CurrentTime();
  8.   if (mode_ == ProcessMode::kDynamic && packet_queue_.Empty() &&
  9.       NextSendTime() <= now) {
  10.     TimeDelta elapsed_time = UpdateTimeAndGetElapsed(now);
  11.     UpdateBudgetWithElapsedTime(elapsed_time);
  12.   }
  13.   packet_queue_.Push(priority, now, packet_counter_++, std::move(packet));
  14. }

最后是通过PacingController::GetPendingPacket来获取添加的packet数据然后发出去。

PacingController::GetPendingPacket

  1. std::unique_ptr<RtpPacketToSend> PacingController::GetPendingPacket(
  2.     const PacedPacketInfo& pacing_info,
  3.     Timestamp target_send_time,
  4.     Timestamp now) {
  5.   if (packet_queue_.Empty()) {
  6.     return nullptr;
  7.   }
  9.   // First, check if there is any reason _not_ to send the next queued packet.
  11.   // Unpaced audio packets and probes are exempted from send checks.
  12.   bool unpaced_audio_packet =
  13.       !pace_audio_ && packet_queue_.LeadingAudioPacketEnqueueTime().has_value();
  14.   bool is_probe = pacing_info.probe_cluster_id != PacedPacketInfo::kNotAProbe;
  15.   if (!unpaced_audio_packet && !is_probe) {
  16.     if (Congested()) {
  17.       // Don't send anything if congested.
  18.       return nullptr;
  19.     }
  21.     if (mode_ == ProcessMode::kPeriodic) {
  22.       if (media_budget_.bytes_remaining() <= 0) {
  23.         // Not enough budget.
  24.         return nullptr;
  25.       }
  26.     } else {
  27.       // Dynamic processing mode.
  28.       if (now <= target_send_time) {
  29.         // We allow sending slightly early if we think that we would actually
  30.         // had been able to, had we been right on time - i.e. the current debt
  31.         // is not more than would be reduced to zero at the target sent time.
  32.         TimeDelta flush_time = media_debt_ / media_rate_;
  33.         if (now + flush_time > target_send_time) {
  34.           return nullptr;
  35.         }
  36.       }
  37.     }
  38.   }
  40.   return packet_queue_.Pop();
  41. }

image.png
可以看到是PacingController::ProcessPackets中调用了获取PacingController::GetPendingPacket数据。

PacingController::ProcessPackets

  1. void PacingController::ProcessPackets() {
  2.   Timestamp now = CurrentTime();
  3.   Timestamp target_send_time = now;
  4.   if (mode_ == ProcessMode::kDynamic) {
  5.     target_send_time = NextSendTime();
  6.     TimeDelta early_execute_margin =
  7.         prober_.is_probing() ? kMaxEarlyProbeProcessing : TimeDelta::Zero();
  8.     if (target_send_time.IsMinusInfinity()) {
  9.       target_send_time = now;
  10.     } else if (now < target_send_time - early_execute_margin) {
  11.       // We are too early, but if queue is empty still allow draining some debt.
  12.       // Probing is allowed to be sent up to kMinSleepTime early.
  13.       TimeDelta elapsed_time = UpdateTimeAndGetElapsed(now);
  14.       UpdateBudgetWithElapsedTime(elapsed_time);
  15.       return;
  16.     }
  18.     if (target_send_time < last_process_time_) {
  19.       // After the last process call, at time X, the target send time
  20.       // shifted to be earlier than X. This should normally not happen
  21.       // but we want to make sure rounding errors or erratic behavior
  22.       // of NextSendTime() does not cause issue. In particular, if the
  23.       // buffer reduction of
  24.       // rate * (target_send_time - previous_process_time)
  25.       // in the main loop doesn't clean up the existing debt we may not
  26.       // be able to send again. We don't want to check this reordering
  27.       // there as it is the normal exit condtion when the buffer is
  28.       // exhausted and there are packets in the queue.
  29.       UpdateBudgetWithElapsedTime(last_process_time_ - target_send_time);
  30.       target_send_time = last_process_time_;
  31.     }
  32.   }
  34.   Timestamp previous_process_time = last_process_time_;
  35.   TimeDelta elapsed_time = UpdateTimeAndGetElapsed(now);
  37.   if (ShouldSendKeepalive(now)) {
  38.     // We can not send padding unless a normal packet has first been sent. If
  39.     // we do, timestamps get messed up.
  40.     if (packet_counter_ == 0) {
  41.       last_send_time_ = now;
  42.     } else {
  43.       DataSize keepalive_data_sent = DataSize::Zero();
  44.       std::vector<std::unique_ptr<RtpPacketToSend>> keepalive_packets =
  45.           packet_sender_->GeneratePadding(DataSize::Bytes(1));
  46.       for (auto& packet : keepalive_packets) {
  47.         keepalive_data_sent +=
  48.             DataSize::Bytes(packet->payload_size() + packet->padding_size());
  49.         packet_sender_->SendPacket(std::move(packet), PacedPacketInfo());
  50.         for (auto& packet : packet_sender_->FetchFec()) {
  51.           EnqueuePacket(std::move(packet));
  52.         }
  53.       }
  54.       OnPaddingSent(keepalive_data_sent);
  55.     }
  56.   }
  58.   if (paused_) {
  59.     return;
  60.   }
  62.   if (elapsed_time > TimeDelta::Zero()) {
  63.     DataRate target_rate = pacing_bitrate_;
  64.     DataSize queue_size_data = packet_queue_.Size();
  65.     if (queue_size_data > DataSize::Zero()) {
  66.       // Assuming equal size packets and input/output rate, the average packet
  67.       // has avg_time_left_ms left to get queue_size_bytes out of the queue, if
  68.       // time constraint shall be met. Determine bitrate needed for that.
  69.       packet_queue_.UpdateQueueTime(now);
  70.       if (drain_large_queues_) {
  71.         TimeDelta avg_time_left =
  72.             std::max(TimeDelta::Millis(1),
  73.                      queue_time_limit - packet_queue_.AverageQueueTime());
  74.         DataRate min_rate_needed = queue_size_data / avg_time_left;
  75.         if (min_rate_needed > target_rate) {
  76.           target_rate = min_rate_needed;
  77.           RTC_LOG(LS_VERBOSE) << "bwe:large_pacing_queue pacing_rate_kbps="
  78.                               << target_rate.kbps();
  79.         }
  80.       }
  81.     }
  83.     if (mode_ == ProcessMode::kPeriodic) {
  84.       // In periodic processing mode, the IntevalBudget allows positive budget
  85.       // up to (process interval duration) * (target rate), so we only need to
  86.       // update it once before the packet sending loop.
  87.       media_budget_.set_target_rate_kbps(target_rate.kbps());
  88.       UpdateBudgetWithElapsedTime(elapsed_time);
  89.     } else {
  90.       media_rate_ = target_rate;
  91.     }
  92.   }
  94.   bool first_packet_in_probe = false;
  95.   PacedPacketInfo pacing_info;
  96.   DataSize recommended_probe_size = DataSize::Zero();
  97.   bool is_probing = prober_.is_probing();
  98.   if (is_probing) {
  99.     // Probe timing is sensitive, and handled explicitly by BitrateProber, so
  100.     // use actual send time rather than target.
  101.     pacing_info = prober_.CurrentCluster(now).value_or(PacedPacketInfo());
  102.     if (pacing_info.probe_cluster_id != PacedPacketInfo::kNotAProbe) {
  103.       first_packet_in_probe = pacing_info.probe_cluster_bytes_sent == 0;
  104.       recommended_probe_size = prober_.RecommendedMinProbeSize();
  105.       RTC_DCHECK_GT(recommended_probe_size, DataSize::Zero());
  106.     } else {
  107.       // No valid probe cluster returned, probe might have timed out.
  108.       is_probing = false;
  109.     }
  110.   }
  112.   DataSize data_sent = DataSize::Zero();
  114.   // The paused state is checked in the loop since it leaves the critical
  115.   // section allowing the paused state to be changed from other code.
  116.   while (!paused_) {
  117.     if (first_packet_in_probe) {
  118.       // If first packet in probe, insert a small padding packet so we have a
  119.       // more reliable start window for the rate estimation.
  120.       auto padding = packet_sender_->GeneratePadding(DataSize::Bytes(1));
  121.       // If no RTP modules sending media are registered, we may not get a
  122.       // padding packet back.
  123.       if (!padding.empty()) {
  124.         // Insert with high priority so larger media packets don't preempt it.
  125.         EnqueuePacketInternal(std::move(padding[0]), kFirstPriority);
  126.         // We should never get more than one padding packets with a requested
  127.         // size of 1 byte.
  128.         RTC_DCHECK_EQ(padding.size(), 1u);
  129.       }
  130.       first_packet_in_probe = false;
  131.     }
  133.     if (mode_ == ProcessMode::kDynamic &&
  134.         previous_process_time < target_send_time) {
  135.       // Reduce buffer levels with amount corresponding to time between last
  136.       // process and target send time for the next packet.
  137.       // If the process call is late, that may be the time between the optimal
  138.       // send times for two packets we should already have sent.
  139.       UpdateBudgetWithElapsedTime(target_send_time - previous_process_time);
  140.       previous_process_time = target_send_time;
  141.     }
  143.     // Fetch the next packet, so long as queue is not empty or budget is not
  144.     // exhausted.
  145.     std::unique_ptr<RtpPacketToSend> rtp_packet =
  146.         GetPendingPacket(pacing_info, target_send_time, now);
  148.     if (rtp_packet == nullptr) {
  149.       // No packet available to send, check if we should send padding.
  150.       DataSize padding_to_add = PaddingToAdd(recommended_probe_size, data_sent);
  151.       if (padding_to_add > DataSize::Zero()) {
  152.         std::vector<std::unique_ptr<RtpPacketToSend>> padding_packets =
  153.             packet_sender_->GeneratePadding(padding_to_add);
  154.         if (padding_packets.empty()) {
  155.           // No padding packets were generated, quite send loop.
  156.           break;
  157.         }
  158.         for (auto& packet : padding_packets) {
  159.           EnqueuePacket(std::move(packet));
  160.         }
  161.         // Continue loop to send the padding that was just added.
  162.         continue;
  163.       }
  165.       // Can't fetch new packet and no padding to send, exit send loop.
  166.       break;
  167.     }
  169.     RTC_DCHECK(rtp_packet);
  170.     RTC_DCHECK(rtp_packet->packet_type().has_value());
  171.     const RtpPacketMediaType packet_type = *rtp_packet->packet_type();
  172.     DataSize packet_size = DataSize::Bytes(rtp_packet->payload_size() +
  173.                                            rtp_packet->padding_size());
  175.     if (include_overhead_) {
  176.       packet_size += DataSize::Bytes(rtp_packet->headers_size()) +
  177.                      transport_overhead_per_packet_;
  178.     }
  180.     packet_sender_->SendPacket(std::move(rtp_packet), pacing_info);
  181.     for (auto& packet : packet_sender_->FetchFec()) {
  182.       EnqueuePacket(std::move(packet));
  183.     }
  184.     data_sent += packet_size;
  186.     // Send done, update send/process time to the target send time.
  187.     OnPacketSent(packet_type, packet_size, target_send_time);
  189.     // If we are currently probing, we need to stop the send loop when we have
  190.     // reached the send target.
  191.     if (is_probing && data_sent >= recommended_probe_size) {
  192.       break;
  193.     }
  195.     if (mode_ == ProcessMode::kDynamic) {
  196.       // Update target send time in case that are more packets that we are late
  197.       // in processing.
  198.       Timestamp next_send_time = NextSendTime();
  199.       if (next_send_time.IsMinusInfinity()) {
  200.         target_send_time = now;
  201.       } else {
  202.         target_send_time = std::min(now, next_send_time);
  203.       }
  204.     }
  205.   }
  207.   last_process_time_ = std::max(last_process_time_, previous_process_time);
  209.   if (is_probing) {
  210.     probing_send_failure_ = data_sent == DataSize::Zero();
  211.     if (!probing_send_failure_) {
  212.       prober_.ProbeSent(CurrentTime(), data_sent);
  213.     }
  214.   }
  215. }

里面是调用了packetsender->SendPacket(std::move(packet), PacedPacketInfo());来发送数据。
image.png