几种缓冲区

image.png
packet_buffer 解码后保存到decoded_buffer，decoded_buffer经过DSP处理后的数据保存到algorithm_buffer，然后从algorithm_buffer中拷贝数据过来到sync_buffer 。

PacketBuffer入队

image.png

PacketBuffer出队

image.png

NetEqImpl::GetAudioInternal

image.png
h:\webrtc-20210315\webrtc-20210315\webrtc\webrtc-checkout\src\modules\audio_coding\neteq\neteq_impl.cc

  1. int NetEqImpl::GetAudioInternal(AudioFrame* audio_frame,
  2.                                 bool* muted,
  3.                                 absl::optional<Operation> action_override) {
  4.   PacketList packet_list;
  5.   DtmfEvent dtmf_event;
  6.   Operation operation;
  7.   bool play_dtmf;
  8.   *muted = false;
  9.   last_decoded_timestamps_.clear();
  10.   last_decoded_packet_infos_.clear();
  11.   tick_timer_->Increment();
  12.   stats_->IncreaseCounter(output_size_samples_, fs_hz_);
  13.   const auto lifetime_stats = stats_->GetLifetimeStatistics();
  14.   expand_uma_logger_.UpdateSampleCounter(lifetime_stats.concealed_samples,
  15.                                          fs_hz_);
  16.   speech_expand_uma_logger_.UpdateSampleCounter(
  17.       lifetime_stats.concealed_samples -
  18.           lifetime_stats.silent_concealed_samples,
  19.       fs_hz_);
  21.   // Check for muted state.
  22.   if (enable_muted_state_ && expand_->Muted() && packet_buffer_->Empty()) {
  23.     RTC_DCHECK_EQ(last_mode_, Mode::kExpand);
  24.     audio_frame->Reset();
  25.     RTC_DCHECK(audio_frame->muted());  // Reset() should mute the frame.
  26.     playout_timestamp_ += static_cast<uint32_t>(output_size_samples_);
  27.     audio_frame->sample_rate_hz_ = fs_hz_;
  28.     audio_frame->samples_per_channel_ = output_size_samples_;
  29.     audio_frame->timestamp_ =
  30.         first_packet_
  31.             ? 0
  32.             : timestamp_scaler_->ToExternal(playout_timestamp_) -
  33.                   static_cast<uint32_t>(audio_frame->samples_per_channel_);
  34.     audio_frame->num_channels_ = sync_buffer_->Channels();
  35.     stats_->ExpandedNoiseSamples(output_size_samples_, false);
  36.     controller_->NotifyMutedState();
  37.     *muted = true;
  38.     return 0;
  39.   }
  40.   int return_value = GetDecision(&operation, &packet_list, &dtmf_event,
  41.                                  &play_dtmf, action_override);
  42.   if (return_value != 0) {
  43.     last_mode_ = Mode::kError;
  44.     return return_value;
  45.   }
  47.   AudioDecoder::SpeechType speech_type;
  48.   int length = 0;
  49.   const size_t start_num_packets = packet_list.size();
  50.   int decode_return_value =
  51.       Decode(&packet_list, &operation, &length, &speech_type);
  53.   assert(vad_.get());
  54.   bool sid_frame_available =
  55.       (operation == Operation::kRfc3389Cng && !packet_list.empty());
  56.   vad_->Update(decoded_buffer_.get(), static_cast<size_t>(length), speech_type,
  57.                sid_frame_available, fs_hz_);
  59.   // This is the criterion that we did decode some data through the speech
  60.   // decoder, and the operation resulted in comfort noise.
  61.   const bool codec_internal_sid_frame =
  62.       (speech_type == AudioDecoder::kComfortNoise &&
  63.        start_num_packets > packet_list.size());
  65.   if (sid_frame_available || codec_internal_sid_frame) {
  66.     // Start a new stopwatch since we are decoding a new CNG packet.
  67.     generated_noise_stopwatch_ = tick_timer_->GetNewStopwatch();
  68.   }
  70.   algorithm_buffer_->Clear();
  71.   switch (operation) {
  72.     case Operation::kNormal: {
  73.       DoNormal(decoded_buffer_.get(), length, speech_type, play_dtmf);
  74.       if (length > 0) {
  75.         stats_->DecodedOutputPlayed();
  76.       }
  77.       break;
  78.     }
  79.     case Operation::kMerge: {
  80.       DoMerge(decoded_buffer_.get(), length, speech_type, play_dtmf);
  81.       break;
  82.     }
  83.     case Operation::kExpand: {
  84.       RTC_DCHECK_EQ(return_value, 0);
  85.       if (!current_rtp_payload_type_ || !DoCodecPlc()) {
  86.         return_value = DoExpand(play_dtmf);
  87.       }
  88.       RTC_DCHECK_GE(sync_buffer_->FutureLength() - expand_->overlap_length(),
  89.                     output_size_samples_);
  90.       break;
  91.     }
  92.     case Operation::kAccelerate:
  93.     case Operation::kFastAccelerate: {
  94.       const bool fast_accelerate =
  95.           enable_fast_accelerate_ && (operation == Operation::kFastAccelerate);
  96.       return_value = DoAccelerate(decoded_buffer_.get(), length, speech_type,
  97.                                   play_dtmf, fast_accelerate);
  98.       break;
  99.     }
  100.     case Operation::kPreemptiveExpand: {
  101.       return_value = DoPreemptiveExpand(decoded_buffer_.get(), length,
  102.                                         speech_type, play_dtmf);
  103.       break;
  104.     }
  105.     case Operation::kRfc3389Cng:
  106.     case Operation::kRfc3389CngNoPacket: {
  107.       return_value = DoRfc3389Cng(&packet_list, play_dtmf);
  108.       break;
  109.     }
  110.     case Operation::kCodecInternalCng: {
  111.       // This handles the case when there is no transmission and the decoder
  112.       // should produce internal comfort noise.
  113.       // TODO(hlundin): Write test for codec-internal CNG.
  114.       DoCodecInternalCng(decoded_buffer_.get(), length);
  115.       break;
  116.     }
  117.     case Operation::kDtmf: {
  118.       // TODO(hlundin): Write test for this.
  119.       return_value = DoDtmf(dtmf_event, &play_dtmf);
  120.       break;
  121.     }
  122.     case Operation::kUndefined: {
  123.       RTC_LOG(LS_ERROR) << "Invalid operation kUndefined.";
  124.       assert(false);  // This should not happen.
  125.       last_mode_ = Mode::kError;
  126.       return kInvalidOperation;
  127.     }
  128.   }  // End of switch.
  129.   last_operation_ = operation;
  130.   if (return_value < 0) {
  131.     return return_value;
  132.   }
  134.   if (last_mode_ != Mode::kRfc3389Cng) {
  135.     comfort_noise_->Reset();
  136.   }
  138.   // We treat it as if all packets referenced to by |last_decoded_packet_infos_|
  139.   // were mashed together when creating the samples in |algorithm_buffer_|.
  140.   RtpPacketInfos packet_infos(last_decoded_packet_infos_);
  142.   // Copy samples from |algorithm_buffer_| to |sync_buffer_|.
  143.   //
  144.   // TODO(bugs.webrtc.org/10757):
  145.   //   We would in the future also like to pass |packet_infos| so that we can do
  146.   //   sample-perfect tracking of that information across |sync_buffer_|.
  147.   sync_buffer_->PushBack(*algorithm_buffer_);
  149.   // Extract data from |sync_buffer_| to |output|.
  150.   size_t num_output_samples_per_channel = output_size_samples_;
  151.   size_t num_output_samples = output_size_samples_ * sync_buffer_->Channels();
  152.   if (num_output_samples > AudioFrame::kMaxDataSizeSamples) {
  153.     RTC_LOG(LS_WARNING) << "Output array is too short. "
  154.                         << AudioFrame::kMaxDataSizeSamples << " < "
  155.                         << output_size_samples_ << " * "
  156.                         << sync_buffer_->Channels();
  157.     num_output_samples = AudioFrame::kMaxDataSizeSamples;
  158.     num_output_samples_per_channel =
  159.         AudioFrame::kMaxDataSizeSamples / sync_buffer_->Channels();
  160.   }
  161.   sync_buffer_->GetNextAudioInterleaved(num_output_samples_per_channel,
  162.                                         audio_frame);
  163.   audio_frame->sample_rate_hz_ = fs_hz_;
  164.   // TODO(bugs.webrtc.org/10757):
  165.   //   We don't have the ability to properly track individual packets once their
  166.   //   audio samples have entered |sync_buffer_|. So for now, treat it as if
  167.   //   |packet_infos| from packets decoded by the current |GetAudioInternal()|
  168.   //   call were all consumed assembling the current audio frame and the current
  169.   //   audio frame only.
  170.   audio_frame->packet_infos_ = std::move(packet_infos);
  171.   if (sync_buffer_->FutureLength() < expand_->overlap_length()) {
  172.     // The sync buffer should always contain |overlap_length| samples, but now
  173.     // too many samples have been extracted. Reinstall the |overlap_length|
  174.     // lookahead by moving the index.
  175.     const size_t missing_lookahead_samples =
  176.         expand_->overlap_length() - sync_buffer_->FutureLength();
  177.     RTC_DCHECK_GE(sync_buffer_->next_index(), missing_lookahead_samples);
  178.     sync_buffer_->set_next_index(sync_buffer_->next_index() -
  179.                                  missing_lookahead_samples);
  180.   }
  181.   if (audio_frame->samples_per_channel_ != output_size_samples_) {
  182.     RTC_LOG(LS_ERROR) << "audio_frame->samples_per_channel_ ("
  183.                       << audio_frame->samples_per_channel_
  184.                       << ") != output_size_samples_ (" << output_size_samples_
  185.                       << ")";
  186.     // TODO(minyue): treatment of under-run, filling zeros
  187.     audio_frame->Mute();
  188.     return kSampleUnderrun;
  189.   }
  191.   // Should always have overlap samples left in the |sync_buffer_|.
  192.   RTC_DCHECK_GE(sync_buffer_->FutureLength(), expand_->overlap_length());
  194.   // TODO(yujo): For muted frames, this can be a copy rather than an addition.
  195.   if (play_dtmf) {
  196.     return_value = DtmfOverdub(dtmf_event, sync_buffer_->Channels(),
  197.                                audio_frame->mutable_data());
  198.   }
  200.   // Update the background noise parameters if last operation wrote data
  201.   // straight from the decoder to the |sync_buffer_|. That is, none of the
  202.   // operations that modify the signal can be followed by a parameter update.
  203.   if ((last_mode_ == Mode::kNormal) || (last_mode_ == Mode::kAccelerateFail) ||
  204.       (last_mode_ == Mode::kPreemptiveExpandFail) ||
  205.       (last_mode_ == Mode::kRfc3389Cng) ||
  206.       (last_mode_ == Mode::kCodecInternalCng)) {
  207.     background_noise_->Update(*sync_buffer_, *vad_.get());
  208.   }
  210.   if (operation == Operation::kDtmf) {
  211.     // DTMF data was written the end of |sync_buffer_|.
  212.     // Update index to end of DTMF data in |sync_buffer_|.
  213.     sync_buffer_->set_dtmf_index(sync_buffer_->Size());
  214.   }
  216.   if (last_mode_ != Mode::kExpand && last_mode_ != Mode::kCodecPlc) {
  217.     // If last operation was not expand, calculate the |playout_timestamp_| from
  218.     // the |sync_buffer_|. However, do not update the |playout_timestamp_| if it
  219.     // would be moved "backwards".
  220.     uint32_t temp_timestamp =
  221.         sync_buffer_->end_timestamp() -
  222.         static_cast<uint32_t>(sync_buffer_->FutureLength());
  223.     if (static_cast<int32_t>(temp_timestamp - playout_timestamp_) > 0) {
  224.       playout_timestamp_ = temp_timestamp;
  225.     }
  226.   } else {
  227.     // Use dead reckoning to estimate the |playout_timestamp_|.
  228.     playout_timestamp_ += static_cast<uint32_t>(output_size_samples_);
  229.   }
  230.   // Set the timestamp in the audio frame to zero before the first packet has
  231.   // been inserted. Otherwise, subtract the frame size in samples to get the
  232.   // timestamp of the first sample in the frame (playout_timestamp_ is the
  233.   // last + 1).
  234.   audio_frame->timestamp_ =
  235.       first_packet_
  236.           ? 0
  237.           : timestamp_scaler_->ToExternal(playout_timestamp_) -
  238.                 static_cast<uint32_t>(audio_frame->samples_per_channel_);
  240.   if (!(last_mode_ == Mode::kRfc3389Cng ||
  241.         last_mode_ == Mode::kCodecInternalCng || last_mode_ == Mode::kExpand ||
  242.         last_mode_ == Mode::kCodecPlc)) {
  243.     generated_noise_stopwatch_.reset();
  244.   }
  246.   if (decode_return_value)
  247.     return decode_return_value;
  248.   return return_value;
  249. }