2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
11 #include "webrtc/modules/audio_coding/main/source/audio_coding_module_impl.h"
16 #include <algorithm> // For std::max.
18 #include "webrtc/engine_configurations.h"
19 #include "webrtc/modules/audio_coding/main/source/acm_codec_database.h"
20 #include "webrtc/modules/audio_coding/main/acm2/acm_common_defs.h"
21 #include "webrtc/modules/audio_coding/main/acm2/call_statistics.h"
22 #include "webrtc/modules/audio_coding/main/source/acm_dtmf_detection.h"
23 #include "webrtc/modules/audio_coding/main/source/acm_generic_codec.h"
24 #include "webrtc/modules/audio_coding/main/source/acm_resampler.h"
25 #include "webrtc/modules/audio_coding/main/acm2/nack.h"
26 #include "webrtc/system_wrappers/interface/clock.h"
27 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
28 #include "webrtc/system_wrappers/interface/logging.h"
29 #include "webrtc/system_wrappers/interface/rw_lock_wrapper.h"
30 #include "webrtc/system_wrappers/interface/tick_util.h"
31 #include "webrtc/system_wrappers/interface/trace.h"
32 #include "webrtc/system_wrappers/interface/trace_event.h"
42 // Maximum number of bytes in one packet (PCM16B, 20 ms packets, stereo).
47 // Maximum number of payloads that can be packed in one RED payload. For
48 // regular FEC, we only pack two payloads. In case of dual-streaming, in worst
49 // case we might pack 3 payloads in one RED payload.
51 kNumFecFragmentationVectors = 2,
52 kMaxNumFragmentationVectors = 3
55 static const uint32_t kMaskTimestamp = 0x03ffffff;
56 static const int kDefaultTimestampDiff = 960; // 20 ms @ 48 kHz.
58 // If packet N is arrived all packets prior to N - |kNackThresholdPackets| which
59 // are not received are considered as lost, and appear in NACK list.
60 static const int kNackThresholdPackets = 2;
64 bool IsCodecRED(const CodecInst* codec) {
65 return (STR_CASE_CMP(codec->plname, "RED") == 0);
68 bool IsCodecRED(int index) {
69 return (IsCodecRED(&ACMCodecDB::database_[index]));
72 bool IsCodecCN(const CodecInst* codec) {
73 return (STR_CASE_CMP(codec->plname, "CN") == 0);
76 bool IsCodecCN(int index) {
77 return (IsCodecCN(&ACMCodecDB::database_[index]));
80 // Stereo-to-mono can be used as in-place.
81 int DownMix(const AudioFrame& frame, int length_out_buff, int16_t* out_buff) {
82 if (length_out_buff < frame.samples_per_channel_) {
85 for (int n = 0; n < frame.samples_per_channel_; ++n)
86 out_buff[n] = (frame.data_[2 * n] + frame.data_[2 * n + 1]) >> 1;
90 // Mono-to-stereo can be used as in-place.
91 int UpMix(const AudioFrame& frame, int length_out_buff, int16_t* out_buff) {
92 if (length_out_buff < frame.samples_per_channel_) {
95 for (int n = frame.samples_per_channel_ - 1; n >= 0; --n) {
96 out_buff[2 * n + 1] = frame.data_[n];
97 out_buff[2 * n] = frame.data_[n];
102 // Return 1 if timestamp t1 is less than timestamp t2, while compensating for
104 int TimestampLessThan(uint32_t t1, uint32_t t2) {
105 uint32_t kHalfFullRange = static_cast<uint32_t>(0xFFFFFFFF) / 2;
108 } else if (t1 < t2) {
109 if (t2 - t1 < kHalfFullRange)
113 if (t1 - t2 < kHalfFullRange)
121 AudioCodingModuleImpl::AudioCodingModuleImpl(const int32_t id, Clock* clock)
122 : packetization_callback_(NULL),
124 last_timestamp_(0xD87F3F9F),
125 last_in_timestamp_(0xD87F3F9F),
129 cng_swb_pltype_(255),
134 vad_mode_(VADNormal),
135 stereo_receive_registered_(false),
137 prev_received_channel_(0),
138 expected_channels_(1),
139 current_send_codec_idx_(-1),
140 current_receive_codec_idx_(-1),
141 send_codec_registered_(false),
142 acm_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
144 last_recv_audio_codec_pltype_(255),
147 last_fec_timestamp_(0),
148 receive_red_pltype_(255),
149 previous_pltype_(255),
150 dummy_rtp_header_(NULL),
151 recv_pl_frame_size_smpls_(0),
152 receiver_initialized_(false),
153 dtmf_detector_(NULL),
154 dtmf_callback_(NULL),
155 last_detected_tone_(kACMToneEnd),
156 callback_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
157 secondary_send_codec_inst_(),
158 initial_delay_ms_(0),
159 num_packets_accumulated_(0),
160 num_bytes_accumulated_(0),
161 accumulated_audio_ms_(0),
162 first_payload_received_(false),
163 last_incoming_send_timestamp_(0),
164 track_neteq_buffer_(false),
167 last_timestamp_diff_(kDefaultTimestampDiff),
168 last_sequence_number_(0),
170 last_packet_was_sync_(false),
173 nack_enabled_(false) {
175 // Nullify send codec memory, set payload type and set codec name to
177 const char no_name[] = "noCodecRegistered";
178 strncpy(send_codec_inst_.plname, no_name, RTP_PAYLOAD_NAME_SIZE - 1);
179 send_codec_inst_.pltype = -1;
181 strncpy(secondary_send_codec_inst_.plname, no_name,
182 RTP_PAYLOAD_NAME_SIZE - 1);
183 secondary_send_codec_inst_.pltype = -1;
185 for (int i = 0; i < ACMCodecDB::kMaxNumCodecs; i++) {
187 registered_pltypes_[i] = -1;
188 stereo_receive_[i] = false;
189 slave_codecs_[i] = NULL;
190 mirror_codec_idx_[i] = -1;
195 // Allocate memory for RED.
196 red_buffer_ = new uint8_t[MAX_PAYLOAD_SIZE_BYTE];
198 // TODO(turajs): This might not be exactly how this class is supposed to work.
199 // The external usage might be that |fragmentationVectorSize| has to match
200 // the allocated space for the member-arrays, while here, we allocate
201 // according to the maximum number of fragmentations and change
202 // |fragmentationVectorSize| on-the-fly based on actual number of
203 // fragmentations. However, due to copying to local variable before calling
204 // SendData, the RTP module receives a "valid" fragmentation, where allocated
205 // space matches |fragmentationVectorSize|, therefore, this should not cause
206 // any problem. A better approach is not using RTPFragmentationHeader as
207 // member variable, instead, use an ACM-specific structure to hold RED-related
208 // data. See module_common_type.h for the definition of
209 // RTPFragmentationHeader.
210 fragmentation_.VerifyAndAllocateFragmentationHeader(
211 kMaxNumFragmentationVectors);
213 // Register the default payload type for RED and for CNG at sampling rates of
214 // 8, 16, 32 and 48 kHz.
215 for (int i = (ACMCodecDB::kNumCodecs - 1); i >= 0; i--) {
217 red_pltype_ = static_cast<uint8_t>(ACMCodecDB::database_[i].pltype);
218 } else if (IsCodecCN(i)) {
219 if (ACMCodecDB::database_[i].plfreq == 8000) {
220 cng_nb_pltype_ = static_cast<uint8_t>(ACMCodecDB::database_[i].pltype);
221 } else if (ACMCodecDB::database_[i].plfreq == 16000) {
222 cng_wb_pltype_ = static_cast<uint8_t>(ACMCodecDB::database_[i].pltype);
223 } else if (ACMCodecDB::database_[i].plfreq == 32000) {
224 cng_swb_pltype_ = static_cast<uint8_t>(ACMCodecDB::database_[i].pltype);
225 } else if (ACMCodecDB::database_[i].plfreq == 48000) {
226 cng_fb_pltype_ = static_cast<uint8_t>(ACMCodecDB::database_[i].pltype);
231 if (InitializeReceiverSafe() < 0) {
232 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
233 "Cannot initialize receiver");
235 WEBRTC_TRACE(webrtc::kTraceMemory, webrtc::kTraceAudioCoding, id, "Created");
238 AudioCodingModuleImpl::~AudioCodingModuleImpl() {
240 CriticalSectionScoped lock(acm_crit_sect_);
241 current_send_codec_idx_ = -1;
243 for (int i = 0; i < ACMCodecDB::kMaxNumCodecs; i++) {
244 if (codecs_[i] != NULL) {
245 // True stereo codecs share the same memory for master and
246 // slave, so slave codec need to be nullified here, since the
247 // memory will be deleted.
248 if (slave_codecs_[i] == codecs_[i]) {
249 slave_codecs_[i] = NULL;
252 // Mirror index holds the address of the codec memory.
253 assert(mirror_codec_idx_[i] > -1);
254 if (codecs_[mirror_codec_idx_[i]] != NULL) {
255 delete codecs_[mirror_codec_idx_[i]];
256 codecs_[mirror_codec_idx_[i]] = NULL;
262 if (slave_codecs_[i] != NULL) {
263 // Delete memory for stereo usage of mono codecs.
264 assert(mirror_codec_idx_[i] > -1);
265 if (slave_codecs_[mirror_codec_idx_[i]] != NULL) {
266 delete slave_codecs_[mirror_codec_idx_[i]];
267 slave_codecs_[mirror_codec_idx_[i]] = NULL;
269 slave_codecs_[i] = NULL;
273 if (dtmf_detector_ != NULL) {
274 delete dtmf_detector_;
275 dtmf_detector_ = NULL;
277 if (dummy_rtp_header_ != NULL) {
278 delete dummy_rtp_header_;
279 dummy_rtp_header_ = NULL;
281 if (red_buffer_ != NULL) {
282 delete[] red_buffer_;
287 delete callback_crit_sect_;
288 callback_crit_sect_ = NULL;
290 delete acm_crit_sect_;
291 acm_crit_sect_ = NULL;
292 WEBRTC_TRACE(webrtc::kTraceMemory, webrtc::kTraceAudioCoding, id_,
296 int32_t AudioCodingModuleImpl::ChangeUniqueId(const int32_t id) {
298 CriticalSectionScoped lock(acm_crit_sect_);
301 for (int i = 0; i < ACMCodecDB::kMaxNumCodecs; i++) {
302 if (codecs_[i] != NULL) {
303 codecs_[i]->SetUniqueID(id);
312 // Returns the number of milliseconds until the module want a
313 // worker thread to call Process.
314 int32_t AudioCodingModuleImpl::TimeUntilNextProcess() {
315 CriticalSectionScoped lock(acm_crit_sect_);
317 if (!HaveValidEncoder("TimeUntilNextProcess")) {
320 return codecs_[current_send_codec_idx_]->SamplesLeftToEncode() /
321 (send_codec_inst_.plfreq / 1000);
324 int32_t AudioCodingModuleImpl::Process() {
327 CriticalSectionScoped lock(acm_crit_sect_);
328 dual_stream = (secondary_encoder_.get() != NULL);
331 return ProcessDualStream();
333 return ProcessSingleStream();
336 int AudioCodingModuleImpl::EncodeFragmentation(int fragmentation_index,
338 uint32_t current_timestamp,
339 ACMGenericCodec* encoder,
341 int16_t len_bytes = MAX_PAYLOAD_SIZE_BYTE;
342 uint32_t rtp_timestamp;
343 WebRtcACMEncodingType encoding_type;
344 if (encoder->Encode(stream, &len_bytes, &rtp_timestamp, &encoding_type) < 0) {
347 assert(encoding_type == kActiveNormalEncoded);
348 assert(len_bytes > 0);
350 fragmentation_.fragmentationLength[fragmentation_index] = len_bytes;
351 fragmentation_.fragmentationPlType[fragmentation_index] = payload_type;
352 fragmentation_.fragmentationTimeDiff[fragmentation_index] =
353 static_cast<uint16_t>(current_timestamp - rtp_timestamp);
354 fragmentation_.fragmentationVectorSize++;
358 // Primary payloads are sent immediately, whereas a single secondary payload is
359 // buffered to be combined with "the next payload."
360 // Normally "the next payload" would be a primary payload. In case two
361 // consecutive secondary payloads are generated with no primary payload in
362 // between, then two secondary payloads are packed in one RED.
363 int AudioCodingModuleImpl::ProcessDualStream() {
364 uint8_t stream[kMaxNumFragmentationVectors * MAX_PAYLOAD_SIZE_BYTE];
365 uint32_t current_timestamp;
366 int16_t length_bytes = 0;
367 RTPFragmentationHeader my_fragmentation;
369 uint8_t my_red_payload_type;
372 CriticalSectionScoped lock(acm_crit_sect_);
373 // Check if there is an encoder before.
374 if (!HaveValidEncoder("ProcessDualStream") ||
375 secondary_encoder_.get() == NULL) {
378 ACMGenericCodec* primary_encoder = codecs_[current_send_codec_idx_];
379 // If primary encoder has a full frame of audio to generate payload.
380 bool primary_ready_to_encode = primary_encoder->HasFrameToEncode();
381 // If the secondary encoder has a frame of audio to generate a payload.
382 bool secondary_ready_to_encode = secondary_encoder_->HasFrameToEncode();
384 if (!primary_ready_to_encode && !secondary_ready_to_encode) {
388 int len_bytes_previous_secondary = static_cast<int>(
389 fragmentation_.fragmentationLength[2]);
390 assert(len_bytes_previous_secondary <= MAX_PAYLOAD_SIZE_BYTE);
391 bool has_previous_payload = len_bytes_previous_secondary > 0;
393 uint32_t primary_timestamp = primary_encoder->EarliestTimestamp();
394 uint32_t secondary_timestamp = secondary_encoder_->EarliestTimestamp();
396 if (!has_previous_payload && !primary_ready_to_encode &&
397 secondary_ready_to_encode) {
398 // Secondary payload will be the ONLY bit-stream. Encode by secondary
399 // encoder, store the payload, and return. No packet is sent.
400 int16_t len_bytes = MAX_PAYLOAD_SIZE_BYTE;
401 WebRtcACMEncodingType encoding_type;
402 if (secondary_encoder_->Encode(red_buffer_, &len_bytes,
403 &last_fec_timestamp_,
404 &encoding_type) < 0) {
405 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
406 "ProcessDual(): Encoding of secondary encoder Failed");
409 assert(len_bytes > 0);
410 assert(encoding_type == kActiveNormalEncoded);
411 assert(len_bytes <= MAX_PAYLOAD_SIZE_BYTE);
412 fragmentation_.fragmentationLength[2] = len_bytes;
416 // Initialize with invalid but different values, so later can have sanity
417 // check if they are different.
418 int index_primary = -1;
419 int index_secondary = -2;
420 int index_previous_secondary = -3;
422 if (primary_ready_to_encode) {
423 index_primary = secondary_ready_to_encode ?
424 TimestampLessThan(primary_timestamp, secondary_timestamp) : 0;
425 index_primary += has_previous_payload ?
426 TimestampLessThan(primary_timestamp, last_fec_timestamp_) : 0;
429 if (secondary_ready_to_encode) {
430 // Timestamp of secondary payload can only be less than primary payload,
431 // but is always larger than the timestamp of previous secondary payload.
432 index_secondary = primary_ready_to_encode ?
433 (1 - TimestampLessThan(primary_timestamp, secondary_timestamp)) : 0;
436 if (has_previous_payload) {
437 index_previous_secondary = primary_ready_to_encode ?
438 (1 - TimestampLessThan(primary_timestamp, last_fec_timestamp_)) : 0;
439 // If secondary is ready it always have a timestamp larger than previous
440 // secondary. So the index is either 0 or 1.
441 index_previous_secondary += secondary_ready_to_encode ? 1 : 0;
444 // Indices must not be equal.
445 assert(index_primary != index_secondary);
446 assert(index_primary != index_previous_secondary);
447 assert(index_secondary != index_previous_secondary);
449 // One of the payloads has to be at position zero.
450 assert(index_primary == 0 || index_secondary == 0 ||
451 index_previous_secondary == 0);
453 // Timestamp of the RED payload.
454 if (index_primary == 0) {
455 current_timestamp = primary_timestamp;
456 } else if (index_secondary == 0) {
457 current_timestamp = secondary_timestamp;
459 current_timestamp = last_fec_timestamp_;
462 fragmentation_.fragmentationVectorSize = 0;
463 if (has_previous_payload) {
464 assert(index_previous_secondary >= 0 &&
465 index_previous_secondary < kMaxNumFragmentationVectors);
466 assert(len_bytes_previous_secondary <= MAX_PAYLOAD_SIZE_BYTE);
467 memcpy(&stream[index_previous_secondary * MAX_PAYLOAD_SIZE_BYTE],
468 red_buffer_, sizeof(stream[0]) * len_bytes_previous_secondary);
469 fragmentation_.fragmentationLength[index_previous_secondary] =
470 len_bytes_previous_secondary;
471 fragmentation_.fragmentationPlType[index_previous_secondary] =
472 secondary_send_codec_inst_.pltype;
473 fragmentation_.fragmentationTimeDiff[index_previous_secondary] =
474 static_cast<uint16_t>(current_timestamp - last_fec_timestamp_);
475 fragmentation_.fragmentationVectorSize++;
478 if (primary_ready_to_encode) {
479 assert(index_primary >= 0 && index_primary < kMaxNumFragmentationVectors);
480 int i = index_primary * MAX_PAYLOAD_SIZE_BYTE;
481 if (EncodeFragmentation(index_primary, send_codec_inst_.pltype,
482 current_timestamp, primary_encoder,
484 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
485 "ProcessDualStream(): Encoding of primary encoder Failed");
490 if (secondary_ready_to_encode) {
491 assert(index_secondary >= 0 &&
492 index_secondary < kMaxNumFragmentationVectors - 1);
493 int i = index_secondary * MAX_PAYLOAD_SIZE_BYTE;
494 if (EncodeFragmentation(index_secondary,
495 secondary_send_codec_inst_.pltype,
496 current_timestamp, secondary_encoder_.get(),
498 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
499 "ProcessDualStream(): Encoding of secondary encoder "
504 // Copy to local variable, as it will be used outside the ACM lock.
505 my_fragmentation.CopyFrom(fragmentation_);
506 my_red_payload_type = red_pltype_;
508 for (int n = 0; n < fragmentation_.fragmentationVectorSize; n++) {
509 length_bytes += fragmentation_.fragmentationLength[n];
514 CriticalSectionScoped lock(callback_crit_sect_);
515 if (packetization_callback_ != NULL) {
516 // Callback with payload data, including redundant data (FEC/RED).
517 if (packetization_callback_->SendData(kAudioFrameSpeech,
519 current_timestamp, stream,
521 &my_fragmentation) < 0) {
528 CriticalSectionScoped lock(acm_crit_sect_);
529 // Now that data is sent, clean up fragmentation.
530 ResetFragmentation(0);
535 // Process any pending tasks such as timeouts.
536 int AudioCodingModuleImpl::ProcessSingleStream() {
537 // Make room for 1 RED payload.
538 uint8_t stream[2 * MAX_PAYLOAD_SIZE_BYTE];
539 int16_t length_bytes = 2 * MAX_PAYLOAD_SIZE_BYTE;
540 int16_t red_length_bytes = length_bytes;
541 uint32_t rtp_timestamp;
543 WebRtcACMEncodingType encoding_type;
544 FrameType frame_type = kAudioFrameSpeech;
545 uint8_t current_payload_type = 0;
546 bool has_data_to_send = false;
547 bool fec_active = false;
548 RTPFragmentationHeader my_fragmentation;
550 // Keep the scope of the ACM critical section limited.
552 CriticalSectionScoped lock(acm_crit_sect_);
553 // Check if there is an encoder before.
554 if (!HaveValidEncoder("ProcessSingleStream")) {
557 status = codecs_[current_send_codec_idx_]->Encode(stream, &length_bytes,
562 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
563 "ProcessSingleStream(): Encoding Failed");
566 } else if (status == 0) {
570 switch (encoding_type) {
572 current_payload_type = previous_pltype_;
573 frame_type = kFrameEmpty;
577 case kActiveNormalEncoded:
578 case kPassiveNormalEncoded: {
579 current_payload_type = static_cast<uint8_t>(send_codec_inst_.pltype);
580 frame_type = kAudioFrameSpeech;
583 case kPassiveDTXNB: {
584 current_payload_type = cng_nb_pltype_;
585 frame_type = kAudioFrameCN;
586 is_first_red_ = true;
589 case kPassiveDTXWB: {
590 current_payload_type = cng_wb_pltype_;
591 frame_type = kAudioFrameCN;
592 is_first_red_ = true;
595 case kPassiveDTXSWB: {
596 current_payload_type = cng_swb_pltype_;
597 frame_type = kAudioFrameCN;
598 is_first_red_ = true;
601 case kPassiveDTXFB: {
602 current_payload_type = cng_fb_pltype_;
603 frame_type = kAudioFrameCN;
604 is_first_red_ = true;
608 has_data_to_send = true;
609 previous_pltype_ = current_payload_type;
611 // Redundancy encode is done here. The two bitstreams packetized into
612 // one RTP packet and the fragmentation points are set.
613 // Only apply RED on speech data.
614 if ((fec_enabled_) &&
615 ((encoding_type == kActiveNormalEncoded) ||
616 (encoding_type == kPassiveNormalEncoded))) {
617 // FEC is enabled within this scope.
619 // Note that, a special solution exists for iSAC since it is the only
620 // codec for which GetRedPayload has a non-empty implementation.
622 // Summary of the FEC scheme below (use iSAC as example):
624 // 1st (is_first_red_ is true) encoded iSAC frame (primary #1) =>
625 // - call GetRedPayload() and store redundancy for packet #1 in
626 // second fragment of RED buffer (old data)
627 // - drop the primary iSAC frame
628 // - don't call SendData
629 // 2nd (is_first_red_ is false) encoded iSAC frame (primary #2) =>
630 // - store primary #2 in 1st fragment of RED buffer and send the
632 // - the transmitted packet contains primary #2 (new) and
633 // reduncancy for packet #1 (old)
634 // - call GetRed_Payload() and store redundancy for packet #2 in
635 // second fragment of RED buffer
639 // Nth encoded iSAC frame (primary #N) =>
640 // - store primary #N in 1st fragment of RED buffer and send the
642 // - the transmitted packet contains primary #N (new) and
643 // reduncancy for packet #(N-1) (old)
644 // - call GetRedPayload() and store redundancy for packet #N in
645 // second fragment of RED buffer
647 // For all other codecs, GetRedPayload does nothing and returns -1 =>
648 // redundant data is only a copy.
650 // First combined packet contains : #2 (new) and #1 (old)
651 // Second combined packet contains: #3 (new) and #2 (old)
652 // Third combined packet contains : #4 (new) and #3 (old)
654 // Hence, even if every second packet is dropped, perfect
655 // reconstruction is possible.
658 has_data_to_send = false;
659 // Skip the following part for the first packet in a RED session.
660 if (!is_first_red_) {
661 // Rearrange stream such that FEC packets are included.
662 // Replace stream now that we have stored current stream.
663 memcpy(stream + fragmentation_.fragmentationOffset[1], red_buffer_,
664 fragmentation_.fragmentationLength[1]);
665 // Update the fragmentation time difference vector, in number of
667 uint16_t time_since_last = static_cast<uint16_t>(rtp_timestamp -
668 last_fec_timestamp_);
670 // Update fragmentation vectors.
671 fragmentation_.fragmentationPlType[1] =
672 fragmentation_.fragmentationPlType[0];
673 fragmentation_.fragmentationTimeDiff[1] = time_since_last;
674 has_data_to_send = true;
677 // Insert new packet length.
678 fragmentation_.fragmentationLength[0] = length_bytes;
680 // Insert new packet payload type.
681 fragmentation_.fragmentationPlType[0] = current_payload_type;
682 last_fec_timestamp_ = rtp_timestamp;
684 // Can be modified by the GetRedPayload() call if iSAC is utilized.
685 red_length_bytes = length_bytes;
687 // A fragmentation header is provided => packetization according to
688 // RFC 2198 (RTP Payload for Redundant Audio Data) will be used.
689 // First fragment is the current data (new).
690 // Second fragment is the previous data (old).
691 length_bytes = static_cast<int16_t>(
692 fragmentation_.fragmentationLength[0] +
693 fragmentation_.fragmentationLength[1]);
695 // Get, and store, redundant data from the encoder based on the recently
697 // NOTE - only iSAC contains an implementation; all other codecs does
698 // nothing and returns -1.
699 if (codecs_[current_send_codec_idx_]->GetRedPayload(
701 &red_length_bytes) == -1) {
702 // The codec was not iSAC => use current encoder output as redundant
703 // data instead (trivial FEC scheme).
704 memcpy(red_buffer_, stream, red_length_bytes);
707 is_first_red_ = false;
708 // Update payload type with RED payload type.
709 current_payload_type = red_pltype_;
710 // We have packed 2 payloads.
711 fragmentation_.fragmentationVectorSize = kNumFecFragmentationVectors;
713 // Copy to local variable, as it will be used outside ACM lock.
714 my_fragmentation.CopyFrom(fragmentation_);
716 fragmentation_.fragmentationLength[1] = red_length_bytes;
721 if (has_data_to_send) {
722 CriticalSectionScoped lock(callback_crit_sect_);
724 if (packetization_callback_ != NULL) {
726 // Callback with payload data, including redundant data (FEC/RED).
727 packetization_callback_->SendData(frame_type, current_payload_type,
728 rtp_timestamp, stream,
732 // Callback with payload data.
733 packetization_callback_->SendData(frame_type, current_payload_type,
734 rtp_timestamp, stream,
739 if (vad_callback_ != NULL) {
740 // Callback with VAD decision.
741 vad_callback_->InFrameType(static_cast<int16_t>(encoding_type));
747 /////////////////////////////////////////
751 // Initialize send codec.
752 int32_t AudioCodingModuleImpl::InitializeSender() {
753 CriticalSectionScoped lock(acm_crit_sect_);
755 // Start with invalid values.
756 send_codec_registered_ = false;
757 current_send_codec_idx_ = -1;
758 send_codec_inst_.plname[0] = '\0';
760 // Delete all encoders to start fresh.
761 for (int id = 0; id < ACMCodecDB::kMaxNumCodecs; id++) {
762 if (codecs_[id] != NULL) {
763 codecs_[id]->DestructEncoder();
767 // Initialize FEC/RED.
768 is_first_red_ = true;
769 if (fec_enabled_ || secondary_encoder_.get() != NULL) {
770 if (red_buffer_ != NULL) {
771 memset(red_buffer_, 0, MAX_PAYLOAD_SIZE_BYTE);
774 ResetFragmentation(kNumFecFragmentationVectors);
776 ResetFragmentation(0);
783 int32_t AudioCodingModuleImpl::ResetEncoder() {
784 CriticalSectionScoped lock(acm_crit_sect_);
785 if (!HaveValidEncoder("ResetEncoder")) {
788 return codecs_[current_send_codec_idx_]->ResetEncoder();
791 void AudioCodingModuleImpl::UnregisterSendCodec() {
792 CriticalSectionScoped lock(acm_crit_sect_);
793 send_codec_registered_ = false;
794 current_send_codec_idx_ = -1;
795 // If send Codec is unregistered then remove the secondary codec as well.
796 if (secondary_encoder_.get() != NULL)
797 secondary_encoder_.reset();
801 ACMGenericCodec* AudioCodingModuleImpl::CreateCodec(const CodecInst& codec) {
802 ACMGenericCodec* my_codec = NULL;
804 my_codec = ACMCodecDB::CreateCodecInstance(&codec);
805 if (my_codec == NULL) {
806 // Error, could not create the codec.
807 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
808 "ACMCodecDB::CreateCodecInstance() failed in CreateCodec()");
811 my_codec->SetUniqueID(id_);
812 my_codec->SetNetEqDecodeLock(neteq_.DecodeLock());
817 // Check if the given codec is a valid to be registered as send codec.
818 static int IsValidSendCodec(const CodecInst& send_codec,
819 bool is_primary_encoder,
822 if ((send_codec.channels != 1) && (send_codec.channels != 2)) {
823 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, acm_id,
824 "Wrong number of channels (%d, only mono and stereo are "
825 "supported) for %s encoder", send_codec.channels,
826 is_primary_encoder ? "primary" : "secondary");
830 int codec_id = ACMCodecDB::CodecNumber(&send_codec, mirror_id);
832 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, acm_id,
833 "Invalid settings for the send codec.");
837 // TODO(tlegrand): Remove this check. Already taken care of in
838 // ACMCodecDB::CodecNumber().
839 // Check if the payload-type is valid
840 if (!ACMCodecDB::ValidPayloadType(send_codec.pltype)) {
841 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, acm_id,
842 "Invalid payload-type %d for %s.", send_codec.pltype,
847 // Telephone-event cannot be a send codec.
848 if (!STR_CASE_CMP(send_codec.plname, "telephone-event")) {
849 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, acm_id,
850 "telephone-event cannot be a send codec");
855 if (ACMCodecDB::codec_settings_[codec_id].channel_support
856 < send_codec.channels) {
857 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, acm_id,
858 "%d number of channels not supportedn for %s.",
859 send_codec.channels, send_codec.plname);
864 if (!is_primary_encoder) {
865 // If registering the secondary encoder, then RED and CN are not valid
866 // choices as encoder.
867 if (IsCodecRED(&send_codec)) {
868 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, acm_id,
869 "RED cannot be secondary codec");
874 if (IsCodecCN(&send_codec)) {
875 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, acm_id,
876 "DTX cannot be secondary codec");
884 int AudioCodingModuleImpl::RegisterSecondarySendCodec(
885 const CodecInst& send_codec) {
886 CriticalSectionScoped lock(acm_crit_sect_);
887 if (!send_codec_registered_) {
890 // Primary and Secondary codecs should have the same sampling rates.
891 if (send_codec.plfreq != send_codec_inst_.plfreq) {
895 int codec_id = IsValidSendCodec(send_codec, false, id_, &mirror_id);
899 ACMGenericCodec* encoder = CreateCodec(send_codec);
900 WebRtcACMCodecParams codec_params;
901 // Initialize the codec before registering. For secondary codec VAD & DTX are
903 memcpy(&(codec_params.codec_inst), &send_codec, sizeof(CodecInst));
904 codec_params.enable_vad = false;
905 codec_params.enable_dtx = false;
906 codec_params.vad_mode = VADNormal;
907 // Force initialization.
908 if (encoder->InitEncoder(&codec_params, true) < 0) {
909 // Could not initialize, therefore cannot be registered.
913 secondary_encoder_.reset(encoder);
914 memcpy(&secondary_send_codec_inst_, &send_codec, sizeof(send_codec));
916 // Disable VAD & DTX.
917 SetVADSafe(false, false, VADNormal);
921 memset(red_buffer_, 0, MAX_PAYLOAD_SIZE_BYTE);
923 ResetFragmentation(0);
927 void AudioCodingModuleImpl::UnregisterSecondarySendCodec() {
928 CriticalSectionScoped lock(acm_crit_sect_);
929 if (secondary_encoder_.get() == NULL) {
932 secondary_encoder_.reset();
933 ResetFragmentation(0);
936 int AudioCodingModuleImpl::SecondarySendCodec(
937 CodecInst* secondary_codec) const {
938 CriticalSectionScoped lock(acm_crit_sect_);
939 if (secondary_encoder_.get() == NULL) {
942 memcpy(secondary_codec, &secondary_send_codec_inst_,
943 sizeof(secondary_send_codec_inst_));
947 // Can be called multiple times for Codec, CNG, RED.
948 int32_t AudioCodingModuleImpl::RegisterSendCodec(
949 const CodecInst& send_codec) {
951 int codec_id = IsValidSendCodec(send_codec, true, id_, &mirror_id);
953 CriticalSectionScoped lock(acm_crit_sect_);
955 // Check for reported errors from function IsValidSendCodec().
957 if (!send_codec_registered_) {
958 // This values has to be NULL if there is no codec registered.
959 current_send_codec_idx_ = -1;
964 // RED can be registered with other payload type. If not registered a default
965 // payload type is used.
966 if (IsCodecRED(&send_codec)) {
967 // TODO(tlegrand): Remove this check. Already taken care of in
968 // ACMCodecDB::CodecNumber().
969 // Check if the payload-type is valid
970 if (!ACMCodecDB::ValidPayloadType(send_codec.pltype)) {
971 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
972 "Invalid payload-type %d for %s.", send_codec.pltype,
976 // Set RED payload type.
977 red_pltype_ = static_cast<uint8_t>(send_codec.pltype);
981 // CNG can be registered with other payload type. If not registered the
982 // default payload types from codec database will be used.
983 if (IsCodecCN(&send_codec)) {
984 // CNG is registered.
985 switch (send_codec.plfreq) {
987 cng_nb_pltype_ = static_cast<uint8_t>(send_codec.pltype);
991 cng_wb_pltype_ = static_cast<uint8_t>(send_codec.pltype);
995 cng_swb_pltype_ = static_cast<uint8_t>(send_codec.pltype);
999 cng_fb_pltype_ = static_cast<uint8_t>(send_codec.pltype);
1003 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1004 "RegisterSendCodec() failed, invalid frequency for CNG "
1012 // Set Stereo, and make sure VAD and DTX is turned off.
1013 if (send_codec.channels == 2) {
1014 stereo_send_ = true;
1015 if (vad_enabled_ || dtx_enabled_) {
1016 WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceAudioCoding, id_,
1017 "VAD/DTX is turned off, not supported when sending stereo.");
1019 vad_enabled_ = false;
1020 dtx_enabled_ = false;
1022 stereo_send_ = false;
1025 // Check if the codec is already registered as send codec.
1027 if (send_codec_registered_) {
1028 int send_codec_mirror_id;
1029 int send_codec_id = ACMCodecDB::CodecNumber(&send_codec_inst_,
1030 &send_codec_mirror_id);
1031 assert(send_codec_id >= 0);
1032 is_send_codec = (send_codec_id == codec_id) ||
1033 (mirror_id == send_codec_mirror_id);
1035 is_send_codec = false;
1038 // If there is secondary codec registered and the new send codec has a
1039 // sampling rate different than that of secondary codec, then unregister the
1041 if (secondary_encoder_.get() != NULL &&
1042 secondary_send_codec_inst_.plfreq != send_codec.plfreq) {
1043 secondary_encoder_.reset();
1044 ResetFragmentation(0);
1047 // If new codec, or new settings, register.
1048 if (!is_send_codec) {
1049 if (codecs_[mirror_id] == NULL) {
1050 codecs_[mirror_id] = CreateCodec(send_codec);
1051 if (codecs_[mirror_id] == NULL) {
1052 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1053 "Cannot Create the codec");
1056 mirror_codec_idx_[mirror_id] = mirror_id;
1059 if (mirror_id != codec_id) {
1060 codecs_[codec_id] = codecs_[mirror_id];
1061 mirror_codec_idx_[codec_id] = mirror_id;
1064 ACMGenericCodec* codec_ptr = codecs_[codec_id];
1065 WebRtcACMCodecParams codec_params;
1067 memcpy(&(codec_params.codec_inst), &send_codec, sizeof(CodecInst));
1068 codec_params.enable_vad = vad_enabled_;
1069 codec_params.enable_dtx = dtx_enabled_;
1070 codec_params.vad_mode = vad_mode_;
1071 // Force initialization.
1072 if (codec_ptr->InitEncoder(&codec_params, true) < 0) {
1073 // Could not initialize the encoder.
1075 // Check if already have a registered codec.
1076 // Depending on that different messages are logged.
1077 if (!send_codec_registered_) {
1078 current_send_codec_idx_ = -1;
1079 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1080 "Cannot Initialize the encoder No Encoder is registered");
1082 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1083 "Cannot Initialize the encoder, continue encoding with "
1084 "the previously registered codec");
1090 dtx_enabled_ = codec_params.enable_dtx;
1091 vad_enabled_ = codec_params.enable_vad;
1092 vad_mode_ = codec_params.vad_mode;
1094 // Everything is fine so we can replace the previous codec with this one.
1095 if (send_codec_registered_) {
1096 // If we change codec we start fresh with FEC.
1097 // This is not strictly required by the standard.
1098 is_first_red_ = true;
1100 codec_ptr->SetVAD(&dtx_enabled_, &vad_enabled_, &vad_mode_);
1103 current_send_codec_idx_ = codec_id;
1104 send_codec_registered_ = true;
1105 memcpy(&send_codec_inst_, &send_codec, sizeof(CodecInst));
1106 previous_pltype_ = send_codec_inst_.pltype;
1109 // If codec is the same as already registered check if any parameters
1110 // has changed compared to the current values.
1111 // If any parameter is valid then apply it and record.
1112 bool force_init = false;
1114 if (mirror_id != codec_id) {
1115 codecs_[codec_id] = codecs_[mirror_id];
1116 mirror_codec_idx_[codec_id] = mirror_id;
1119 // Check the payload type.
1120 if (send_codec.pltype != send_codec_inst_.pltype) {
1121 // At this point check if the given payload type is valid.
1122 // Record it later when the sampling frequency is changed
1124 if (!ACMCodecDB::ValidPayloadType(send_codec.pltype)) {
1125 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1126 "Out of range payload type");
1131 // If there is a codec that ONE instance of codec supports multiple
1132 // sampling frequencies, then we need to take care of it here.
1133 // one such a codec is iSAC. Both WB and SWB are encoded and decoded
1134 // with one iSAC instance. Therefore, we need to update the encoder
1135 // frequency if required.
1136 if (send_codec_inst_.plfreq != send_codec.plfreq) {
1139 // If sampling frequency is changed we have to start fresh with RED.
1140 is_first_red_ = true;
1143 // If packet size or number of channels has changed, we need to
1144 // re-initialize the encoder.
1145 if (send_codec_inst_.pacsize != send_codec.pacsize) {
1148 if (send_codec_inst_.channels != send_codec.channels) {
1153 WebRtcACMCodecParams codec_params;
1155 memcpy(&(codec_params.codec_inst), &send_codec, sizeof(CodecInst));
1156 codec_params.enable_vad = vad_enabled_;
1157 codec_params.enable_dtx = dtx_enabled_;
1158 codec_params.vad_mode = vad_mode_;
1160 // Force initialization.
1161 if (codecs_[current_send_codec_idx_]->InitEncoder(&codec_params,
1163 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1164 "Could not change the codec packet-size.");
1168 send_codec_inst_.plfreq = send_codec.plfreq;
1169 send_codec_inst_.pacsize = send_codec.pacsize;
1170 send_codec_inst_.channels = send_codec.channels;
1173 // If the change of sampling frequency has been successful then
1174 // we store the payload-type.
1175 send_codec_inst_.pltype = send_codec.pltype;
1177 // Check if a change in Rate is required.
1178 if (send_codec.rate != send_codec_inst_.rate) {
1179 if (codecs_[codec_id]->SetBitRate(send_codec.rate) < 0) {
1180 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1181 "Could not change the codec rate.");
1184 send_codec_inst_.rate = send_codec.rate;
1186 previous_pltype_ = send_codec_inst_.pltype;
1192 // Get current send codec.
1193 int32_t AudioCodingModuleImpl::SendCodec(
1194 CodecInst* current_codec) const {
1195 WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceAudioCoding, id_,
1197 CriticalSectionScoped lock(acm_crit_sect_);
1199 assert(current_codec);
1200 if (!send_codec_registered_) {
1201 WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceAudioCoding, id_,
1202 "SendCodec Failed, no codec is registered");
1206 WebRtcACMCodecParams encoder_param;
1207 codecs_[current_send_codec_idx_]->EncoderParams(&encoder_param);
1208 encoder_param.codec_inst.pltype = send_codec_inst_.pltype;
1209 memcpy(current_codec, &(encoder_param.codec_inst), sizeof(CodecInst));
1214 // Get current send frequency.
1215 int32_t AudioCodingModuleImpl::SendFrequency() const {
1216 WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceAudioCoding, id_,
1218 CriticalSectionScoped lock(acm_crit_sect_);
1220 if (!send_codec_registered_) {
1221 WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceAudioCoding, id_,
1222 "SendFrequency Failed, no codec is registered");
1227 return send_codec_inst_.plfreq;
1230 // Get encode bitrate.
1231 // Adaptive rate codecs return their current encode target rate, while other
1232 // codecs return there longterm avarage or their fixed rate.
1233 int32_t AudioCodingModuleImpl::SendBitrate() const {
1234 CriticalSectionScoped lock(acm_crit_sect_);
1236 if (!send_codec_registered_) {
1237 WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceAudioCoding, id_,
1238 "SendBitrate Failed, no codec is registered");
1243 WebRtcACMCodecParams encoder_param;
1244 codecs_[current_send_codec_idx_]->EncoderParams(&encoder_param);
1246 return encoder_param.codec_inst.rate;
1249 // Set available bandwidth, inform the encoder about the estimated bandwidth
1250 // received from the remote party.
1251 int32_t AudioCodingModuleImpl::SetReceivedEstimatedBandwidth(
1253 return codecs_[current_send_codec_idx_]->SetEstimatedBandwidth(bw);
1256 // Register a transport callback which will be called to deliver
1257 // the encoded buffers.
1258 int32_t AudioCodingModuleImpl::RegisterTransportCallback(
1259 AudioPacketizationCallback* transport) {
1260 CriticalSectionScoped lock(callback_crit_sect_);
1261 packetization_callback_ = transport;
1265 // Add 10MS of raw (PCM) audio data to the encoder.
1266 int32_t AudioCodingModuleImpl::Add10MsData(
1267 const AudioFrame& audio_frame) {
1268 if (audio_frame.samples_per_channel_ <= 0) {
1270 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1271 "Cannot Add 10 ms audio, payload length is negative or "
1276 if (audio_frame.sample_rate_hz_ > 48000) {
1278 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1279 "Cannot Add 10 ms audio, input frequency not valid");
1283 // If the length and frequency matches. We currently just support raw PCM.
1284 if ((audio_frame.sample_rate_hz_ / 100)
1285 != audio_frame.samples_per_channel_) {
1286 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1287 "Cannot Add 10 ms audio, input frequency and length doesn't"
1292 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2) {
1293 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1294 "Cannot Add 10 ms audio, invalid number of channels.");
1298 CriticalSectionScoped lock(acm_crit_sect_);
1299 // Do we have a codec registered?
1300 if (!HaveValidEncoder("Add10MsData")) {
1304 const AudioFrame* ptr_frame;
1305 // Perform a resampling, also down-mix if it is required and can be
1306 // performed before resampling (a down mix prior to resampling will take
1307 // place if both primary and secondary encoders are mono and input is in
1309 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
1312 TRACE_EVENT_ASYNC_BEGIN1("webrtc", "Audio", ptr_frame->timestamp_,
1313 "now", clock_->TimeInMilliseconds());
1315 // Check whether we need an up-mix or down-mix?
1316 bool remix = ptr_frame->num_channels_ != send_codec_inst_.channels;
1317 if (secondary_encoder_.get() != NULL) {
1319 (ptr_frame->num_channels_ != secondary_send_codec_inst_.channels);
1322 // If a re-mix is required (up or down), this buffer will store re-mixed
1323 // version of the input.
1324 int16_t buffer[WEBRTC_10MS_PCM_AUDIO];
1326 if (ptr_frame->num_channels_ == 1) {
1327 if (UpMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, buffer) < 0)
1330 if (DownMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, buffer) < 0)
1335 // When adding data to encoders this pointer is pointing to an audio buffer
1336 // with correct number of channels.
1337 const int16_t* ptr_audio = ptr_frame->data_;
1339 // For pushing data to primary, point the |ptr_audio| to correct buffer.
1340 if (send_codec_inst_.channels != ptr_frame->num_channels_)
1343 if (codecs_[current_send_codec_idx_]->Add10MsData(
1344 ptr_frame->timestamp_, ptr_audio, ptr_frame->samples_per_channel_,
1345 send_codec_inst_.channels) < 0)
1348 if (secondary_encoder_.get() != NULL) {
1349 // For pushing data to secondary, point the |ptr_audio| to correct buffer.
1350 ptr_audio = ptr_frame->data_;
1351 if (secondary_send_codec_inst_.channels != ptr_frame->num_channels_)
1354 if (secondary_encoder_->Add10MsData(
1355 ptr_frame->timestamp_, ptr_audio, ptr_frame->samples_per_channel_,
1356 secondary_send_codec_inst_.channels) < 0)
1363 // Perform a resampling and down-mix if required. We down-mix only if
1364 // encoder is mono and input is stereo. In case of dual-streaming, both
1365 // encoders has to be mono for down-mix to take place.
1366 // |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
1367 // is required, |*ptr_out| points to |in_frame|.
1368 int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
1369 const AudioFrame** ptr_out) {
1370 // Primary and secondary (if exists) should have the same sampling rate.
1371 assert((secondary_encoder_.get() != NULL) ?
1372 secondary_send_codec_inst_.plfreq == send_codec_inst_.plfreq : true);
1374 bool resample = static_cast<int32_t>(in_frame.sample_rate_hz_) !=
1375 send_codec_inst_.plfreq;
1377 // This variable is true if primary codec and secondary codec (if exists)
1378 // are both mono and input is stereo.
1380 if (secondary_encoder_.get() != NULL) {
1381 down_mix = (in_frame.num_channels_ == 2) &&
1382 (send_codec_inst_.channels == 1) &&
1383 (secondary_send_codec_inst_.channels == 1);
1385 down_mix = (in_frame.num_channels_ == 2) &&
1386 (send_codec_inst_.channels == 1);
1389 if (!down_mix && !resample) {
1390 // No pre-processing is required.
1391 last_in_timestamp_ = in_frame.timestamp_;
1392 last_timestamp_ = in_frame.timestamp_;
1393 *ptr_out = &in_frame;
1397 *ptr_out = &preprocess_frame_;
1398 preprocess_frame_.num_channels_ = in_frame.num_channels_;
1399 int16_t audio[WEBRTC_10MS_PCM_AUDIO];
1400 const int16_t* src_ptr_audio = in_frame.data_;
1401 int16_t* dest_ptr_audio = preprocess_frame_.data_;
1403 // If a resampling is required the output of a down-mix is written into a
1404 // local buffer, otherwise, it will be written to the output frame.
1406 dest_ptr_audio = audio;
1407 if (DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio) < 0)
1409 preprocess_frame_.num_channels_ = 1;
1410 // Set the input of the resampler is the down-mixed signal.
1411 src_ptr_audio = audio;
1414 preprocess_frame_.timestamp_ = in_frame.timestamp_;
1415 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
1416 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
1417 // If it is required, we have to do a resampling.
1419 // The result of the resampler is written to output frame.
1420 dest_ptr_audio = preprocess_frame_.data_;
1422 uint32_t timestamp_diff;
1424 // Calculate the timestamp of this frame.
1425 if (last_in_timestamp_ > in_frame.timestamp_) {
1426 // A wrap around has happened.
1427 timestamp_diff = (static_cast<uint32_t>(0xFFFFFFFF) - last_in_timestamp_)
1428 + in_frame.timestamp_;
1430 timestamp_diff = in_frame.timestamp_ - last_in_timestamp_;
1432 preprocess_frame_.timestamp_ = last_timestamp_ +
1433 static_cast<uint32_t>(timestamp_diff *
1434 (static_cast<double>(send_codec_inst_.plfreq) /
1435 static_cast<double>(in_frame.sample_rate_hz_)));
1437 preprocess_frame_.samples_per_channel_ = input_resampler_.Resample10Msec(
1438 src_ptr_audio, in_frame.sample_rate_hz_, dest_ptr_audio,
1439 send_codec_inst_.plfreq, preprocess_frame_.num_channels_);
1441 if (preprocess_frame_.samples_per_channel_ < 0) {
1442 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1443 "Cannot add 10 ms audio, resampling failed");
1446 preprocess_frame_.sample_rate_hz_ = send_codec_inst_.plfreq;
1448 last_in_timestamp_ = in_frame.timestamp_;
1449 last_timestamp_ = preprocess_frame_.timestamp_;
1454 /////////////////////////////////////////
1455 // (FEC) Forward Error Correction
1458 bool AudioCodingModuleImpl::FECStatus() const {
1459 CriticalSectionScoped lock(acm_crit_sect_);
1460 return fec_enabled_;
1463 // Configure FEC status i.e on/off.
1465 AudioCodingModuleImpl::SetFECStatus(
1466 #ifdef WEBRTC_CODEC_RED
1467 const bool enable_fec) {
1468 CriticalSectionScoped lock(acm_crit_sect_);
1470 if (fec_enabled_ != enable_fec) {
1471 // Reset the RED buffer.
1472 memset(red_buffer_, 0, MAX_PAYLOAD_SIZE_BYTE);
1474 // Reset fragmentation buffers.
1475 ResetFragmentation(kNumFecFragmentationVectors);
1476 // Set fec_enabled_.
1477 fec_enabled_ = enable_fec;
1479 is_first_red_ = true; // Make sure we restart FEC.
1482 const bool /* enable_fec */) {
1483 fec_enabled_ = false;
1484 WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceAudioCoding, id_,
1485 " WEBRTC_CODEC_RED is undefined => fec_enabled_ = %d",
1491 /////////////////////////////////////////
1492 // (VAD) Voice Activity Detection
1494 int32_t AudioCodingModuleImpl::SetVAD(bool enable_dtx, bool enable_vad,
1496 CriticalSectionScoped lock(acm_crit_sect_);
1497 return SetVADSafe(enable_dtx, enable_vad, mode);
1500 int AudioCodingModuleImpl::SetVADSafe(bool enable_dtx, bool enable_vad,
1502 // Sanity check of the mode.
1503 if ((mode != VADNormal) && (mode != VADLowBitrate)
1504 && (mode != VADAggr) && (mode != VADVeryAggr)) {
1505 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1506 "Invalid VAD Mode %d, no change is made to VAD/DTX status",
1507 static_cast<int>(mode));
1511 // Check that the send codec is mono. We don't support VAD/DTX for stereo
1513 if ((enable_dtx || enable_vad) && stereo_send_) {
1514 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1515 "VAD/DTX not supported for stereo sending.");
1516 dtx_enabled_ = false;
1517 vad_enabled_ = false;
1522 // We don't support VAD/DTX when dual-streaming is enabled, i.e.
1523 // secondary-encoder is registered.
1524 if ((enable_dtx || enable_vad) && secondary_encoder_.get() != NULL) {
1525 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1526 "VAD/DTX not supported when dual-streaming is enabled.");
1527 dtx_enabled_ = false;
1528 vad_enabled_ = false;
1533 // Store VAD/DTX settings. Values can be changed in the call to "SetVAD"
1535 dtx_enabled_ = enable_dtx;
1536 vad_enabled_ = enable_vad;
1539 // If a send codec is registered, set VAD/DTX for the codec.
1540 if (HaveValidEncoder("SetVAD")) {
1541 if (codecs_[current_send_codec_idx_]->SetVAD(&dtx_enabled_, &vad_enabled_,
1544 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1546 dtx_enabled_ = false;
1547 vad_enabled_ = false;
1555 // Get VAD/DTX settings.
1556 // TODO(tlegrand): Change this method to void.
1557 int32_t AudioCodingModuleImpl::VAD(bool* dtx_enabled, bool* vad_enabled,
1558 ACMVADMode* mode) const {
1559 CriticalSectionScoped lock(acm_crit_sect_);
1561 *dtx_enabled = dtx_enabled_;
1562 *vad_enabled = vad_enabled_;
1568 /////////////////////////////////////////
1572 int32_t AudioCodingModuleImpl::InitializeReceiver() {
1573 CriticalSectionScoped lock(acm_crit_sect_);
1574 return InitializeReceiverSafe();
1577 // Initialize receiver, resets codec database etc.
1578 int32_t AudioCodingModuleImpl::InitializeReceiverSafe() {
1579 initial_delay_ms_ = 0;
1580 num_packets_accumulated_ = 0;
1581 num_bytes_accumulated_ = 0;
1582 accumulated_audio_ms_ = 0;
1583 first_payload_received_ = 0;
1584 last_incoming_send_timestamp_ = 0;
1585 track_neteq_buffer_ = false;
1587 // If the receiver is already initialized then we want to destroy any
1588 // existing decoders. After a call to this function, we should have a clean
1590 if (receiver_initialized_) {
1591 for (int i = 0; i < ACMCodecDB::kNumCodecs; i++) {
1592 if (UnregisterReceiveCodecSafe(i) < 0) {
1593 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1594 "InitializeReceiver() failed, Could not unregister codec");
1599 if (neteq_.Init() != 0) {
1600 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1601 "InitializeReceiver() failed, Could not initialize NetEQ");
1605 if (neteq_.AllocatePacketBuffer(ACMCodecDB::NetEQDecoders(),
1606 ACMCodecDB::kNumCodecs) != 0) {
1607 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1608 "NetEQ cannot allocate_packet Buffer");
1612 // Register RED and CN.
1613 for (int i = 0; i < ACMCodecDB::kNumCodecs; i++) {
1614 if (IsCodecRED(i) || IsCodecCN(i)) {
1615 if (RegisterRecCodecMSSafe(ACMCodecDB::database_[i], i, i,
1616 ACMNetEQ::kMasterJb) < 0) {
1617 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1618 "Cannot register master codec.");
1621 registered_pltypes_[i] = ACMCodecDB::database_[i].pltype;
1625 receiver_initialized_ = true;
1629 // Reset the decoder state.
1630 int32_t AudioCodingModuleImpl::ResetDecoder() {
1631 CriticalSectionScoped lock(acm_crit_sect_);
1633 for (int id = 0; id < ACMCodecDB::kMaxNumCodecs; id++) {
1634 if ((codecs_[id] != NULL) && (registered_pltypes_[id] != -1)) {
1635 if (codecs_[id]->ResetDecoder(registered_pltypes_[id]) < 0) {
1636 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1637 "ResetDecoder failed:");
1642 return neteq_.FlushBuffers();
1645 // Get current receive frequency.
1646 int32_t AudioCodingModuleImpl::ReceiveFrequency() const {
1647 WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceAudioCoding, id_,
1648 "ReceiveFrequency()");
1649 WebRtcACMCodecParams codec_params;
1651 CriticalSectionScoped lock(acm_crit_sect_);
1652 if (DecoderParamByPlType(last_recv_audio_codec_pltype_, codec_params) < 0) {
1653 return neteq_.CurrentSampFreqHz();
1654 } else if (codec_params.codec_inst.plfreq == 48000) {
1655 // TODO(tlegrand): Remove this option when we have full 48 kHz support.
1658 return codec_params.codec_inst.plfreq;
1662 // Get current playout frequency.
1663 int32_t AudioCodingModuleImpl::PlayoutFrequency() const {
1664 WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceAudioCoding, id_,
1665 "PlayoutFrequency()");
1667 CriticalSectionScoped lock(acm_crit_sect_);
1669 return neteq_.CurrentSampFreqHz();
1672 // Register possible receive codecs, can be called multiple times,
1673 // for codecs, CNG (NB, WB and SWB), DTMF, RED.
1674 int32_t AudioCodingModuleImpl::RegisterReceiveCodec(
1675 const CodecInst& receive_codec) {
1676 CriticalSectionScoped lock(acm_crit_sect_);
1678 if (receive_codec.channels > 2) {
1679 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1680 "More than 2 audio channel is not supported.");
1685 int codec_id = ACMCodecDB::ReceiverCodecNumber(&receive_codec, &mirror_id);
1687 if (codec_id < 0 || codec_id >= ACMCodecDB::kNumCodecs) {
1688 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1689 "Wrong codec params to be registered as receive codec");
1692 // Check if the payload-type is valid.
1693 if (!ACMCodecDB::ValidPayloadType(receive_codec.pltype)) {
1694 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1695 "Invalid payload-type %d for %s.", receive_codec.pltype,
1696 receive_codec.plname);
1700 if (!receiver_initialized_) {
1701 if (InitializeReceiverSafe() < 0) {
1702 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1703 "Cannot initialize reciver, so failed registering a codec.");
1708 // If codec already registered, unregister. Except for CN where we only
1709 // unregister if payload type is changing.
1710 if ((registered_pltypes_[codec_id] == receive_codec.pltype)
1711 && IsCodecCN(&receive_codec)) {
1712 // Codec already registered as receiver with this payload type. Nothing
1715 } else if (registered_pltypes_[codec_id] != -1) {
1716 if (UnregisterReceiveCodecSafe(codec_id) < 0) {
1717 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1718 "Cannot register master codec.");
1723 if (RegisterRecCodecMSSafe(receive_codec, codec_id, mirror_id,
1724 ACMNetEQ::kMasterJb) < 0) {
1725 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1726 "Cannot register master codec.");
1730 // TODO(andrew): Refactor how the slave is initialized. Can we instead
1731 // always start up a slave and pre-register CN and RED? We should be able
1732 // to get rid of stereo_receive_registered_.
1733 // http://code.google.com/p/webrtc/issues/detail?id=453
1735 // Register stereo codecs with the slave, or, if we've had already seen a
1736 // stereo codec, register CN or RED as a special case.
1737 if (receive_codec.channels == 2 ||
1738 (stereo_receive_registered_ && (IsCodecCN(&receive_codec) ||
1739 IsCodecRED(&receive_codec)))) {
1740 // TODO(andrew): refactor this block to combine with InitStereoSlave().
1742 if (!stereo_receive_registered_) {
1743 // This is the first time a stereo codec has been registered. Make
1744 // some stereo preparations.
1746 // Add a stereo slave.
1747 assert(neteq_.num_slaves() == 0);
1748 if (neteq_.AddSlave(ACMCodecDB::NetEQDecoders(),
1749 ACMCodecDB::kNumCodecs) < 0) {
1750 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1751 "Cannot add slave jitter buffer to NetEQ.");
1755 // Register any existing CN or RED codecs with the slave and as stereo.
1756 for (int i = 0; i < ACMCodecDB::kNumCodecs; i++) {
1757 if (registered_pltypes_[i] != -1 && (IsCodecRED(i) || IsCodecCN(i))) {
1758 stereo_receive_[i] = true;
1761 memcpy(&codec, &ACMCodecDB::database_[i], sizeof(CodecInst));
1762 codec.pltype = registered_pltypes_[i];
1763 if (RegisterRecCodecMSSafe(codec, i, i, ACMNetEQ::kSlaveJb) < 0) {
1764 WEBRTC_TRACE(kTraceError, kTraceAudioCoding, id_,
1765 "Cannot register slave codec.");
1772 if (RegisterRecCodecMSSafe(receive_codec, codec_id, mirror_id,
1773 ACMNetEQ::kSlaveJb) < 0) {
1774 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1775 "Cannot register slave codec.");
1779 if (!stereo_receive_[codec_id] &&
1780 (last_recv_audio_codec_pltype_ == receive_codec.pltype)) {
1781 // The last received payload type is the same as the one we are
1782 // registering. Expected number of channels to receive is one (mono),
1783 // but we are now registering the receiving codec as stereo (number of
1785 // Set |last_recv_audio_coded_pltype_| to invalid value to trigger a
1786 // flush in NetEq, and a reset of expected number of channels next time a
1787 // packet is received in AudioCodingModuleImpl::IncomingPacket().
1788 last_recv_audio_codec_pltype_ = -1;
1791 stereo_receive_[codec_id] = true;
1792 stereo_receive_registered_ = true;
1794 if (last_recv_audio_codec_pltype_ == receive_codec.pltype &&
1795 expected_channels_ == 2) {
1796 // The last received payload type is the same as the one we are
1797 // registering. Expected number of channels to receive is two (stereo),
1798 // but we are now registering the receiving codec as mono (number of
1800 // Set |last_recv_audio_coded_pl_type_| to invalid value to trigger a
1801 // flush in NetEq, and a reset of expected number of channels next time a
1802 // packet is received in AudioCodingModuleImpl::IncomingPacket().
1803 last_recv_audio_codec_pltype_ = -1;
1805 stereo_receive_[codec_id] = false;
1808 registered_pltypes_[codec_id] = receive_codec.pltype;
1810 if (IsCodecRED(&receive_codec)) {
1811 receive_red_pltype_ = receive_codec.pltype;
1816 int32_t AudioCodingModuleImpl::RegisterRecCodecMSSafe(
1817 const CodecInst& receive_codec, int16_t codec_id,
1818 int16_t mirror_id, ACMNetEQ::JitterBuffer jitter_buffer) {
1819 ACMGenericCodec** codecs;
1820 if (jitter_buffer == ACMNetEQ::kMasterJb) {
1821 codecs = &codecs_[0];
1822 } else if (jitter_buffer == ACMNetEQ::kSlaveJb) {
1823 codecs = &slave_codecs_[0];
1824 if (codecs_[codec_id]->IsTrueStereoCodec()) {
1825 // True stereo codecs need to use the same codec memory
1826 // for both master and slave.
1827 slave_codecs_[mirror_id] = codecs_[mirror_id];
1828 mirror_codec_idx_[mirror_id] = mirror_id;
1831 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1832 "RegisterReceiveCodecMSSafe failed, jitter_buffer is neither "
1833 "master or slave ");
1837 if (codecs[mirror_id] == NULL) {
1838 codecs[mirror_id] = CreateCodec(receive_codec);
1839 if (codecs[mirror_id] == NULL) {
1840 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1841 "Cannot create codec to register as receive codec");
1844 mirror_codec_idx_[mirror_id] = mirror_id;
1846 if (mirror_id != codec_id) {
1847 codecs[codec_id] = codecs[mirror_id];
1848 mirror_codec_idx_[codec_id] = mirror_id;
1851 codecs[codec_id]->SetIsMaster(jitter_buffer == ACMNetEQ::kMasterJb);
1854 WebRtcACMCodecParams codec_params;
1855 memcpy(&(codec_params.codec_inst), &receive_codec, sizeof(CodecInst));
1856 codec_params.enable_vad = false;
1857 codec_params.enable_dtx = false;
1858 codec_params.vad_mode = VADNormal;
1859 if (!codecs[codec_id]->DecoderInitialized()) {
1860 // Force initialization.
1861 status = codecs[codec_id]->InitDecoder(&codec_params, true);
1863 // Could not initialize the decoder, we don't want to
1864 // continue if we could not initialize properly.
1866 webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1867 "could not initialize the receive codec, codec not registered");
1871 } else if (mirror_id != codec_id) {
1872 // Currently this only happens for iSAC.
1873 // We have to store the decoder parameters.
1874 codecs[codec_id]->SaveDecoderParam(&codec_params);
1877 if (codecs[codec_id]->RegisterInNetEq(&neteq_, receive_codec) != 0) {
1878 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1879 "Receive codec could not be registered in NetEQ");
1882 // Guarantee that the same payload-type that is
1883 // registered in NetEQ is stored in the codec.
1884 codecs[codec_id]->SaveDecoderParam(&codec_params);
1889 // Get current received codec.
1890 int32_t AudioCodingModuleImpl::ReceiveCodec(
1891 CodecInst* current_codec) const {
1892 WebRtcACMCodecParams decoder_param;
1893 CriticalSectionScoped lock(acm_crit_sect_);
1895 for (int id = 0; id < ACMCodecDB::kMaxNumCodecs; id++) {
1896 if (codecs_[id] != NULL) {
1897 if (codecs_[id]->DecoderInitialized()) {
1898 if (codecs_[id]->DecoderParams(&decoder_param,
1899 last_recv_audio_codec_pltype_)) {
1900 memcpy(current_codec, &decoder_param.codec_inst,
1908 // If we are here then we haven't found any codec. Set codec pltype to -1 to
1909 // indicate that the structure is invalid and return -1.
1910 current_codec->pltype = -1;
1914 // Incoming packet from network parsed and ready for decode.
1915 int32_t AudioCodingModuleImpl::IncomingPacket(
1916 const uint8_t* incoming_payload,
1917 const int32_t payload_length,
1918 const WebRtcRTPHeader& rtp_info) {
1919 WebRtcRTPHeader rtp_header;
1921 memcpy(&rtp_header, &rtp_info, sizeof(WebRtcRTPHeader));
1923 if (payload_length < 0) {
1924 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
1925 "IncomingPacket() Error, payload-length cannot be negative");
1930 // Store the payload Type. This will be used to retrieve "received codec"
1931 // and "received frequency."
1932 CriticalSectionScoped lock(acm_crit_sect_);
1934 // Check there are packets missed between the last injected packet, and the
1935 // latest received packet. If so and we are in AV-sync mode then we would
1936 // like to fill the gap. Shouldn't be the first payload.
1937 if (av_sync_ && first_payload_received_ &&
1938 rtp_info.header.sequenceNumber > last_sequence_number_ + 1) {
1939 // If the last packet pushed was sync-packet account for all missing
1940 // packets. Otherwise leave some room for PLC.
1941 if (last_packet_was_sync_) {
1942 while (rtp_info.header.sequenceNumber > last_sequence_number_ + 2) {
1943 PushSyncPacketSafe();
1946 // Leave two packet room for NetEq perform PLC.
1947 if (rtp_info.header.sequenceNumber > last_sequence_number_ + 3) {
1948 last_sequence_number_ += 2;
1949 last_incoming_send_timestamp_ += last_timestamp_diff_ * 2;
1950 last_receive_timestamp_ += 2 * last_timestamp_diff_;
1951 while (rtp_info.header.sequenceNumber > last_sequence_number_ + 1)
1952 PushSyncPacketSafe();
1957 uint8_t my_payload_type;
1959 // Check if this is an RED payload.
1960 if (rtp_info.header.payloadType == receive_red_pltype_) {
1961 // Get the primary payload-type.
1962 my_payload_type = incoming_payload[0] & 0x7F;
1964 my_payload_type = rtp_info.header.payloadType;
1967 // If payload is audio, check if received payload is different from
1969 if (!rtp_info.type.Audio.isCNG) {
1970 // This is Audio not CNG.
1972 if (my_payload_type != last_recv_audio_codec_pltype_) {
1973 // We detect a change in payload type. It is necessary for iSAC
1974 // we are going to use ONE iSAC instance for decoding both WB and
1975 // SWB payloads. If payload is changed there might be a need to reset
1976 // sampling rate of decoder. depending what we have received "now".
1977 for (int i = 0; i < ACMCodecDB::kMaxNumCodecs; i++) {
1978 if (registered_pltypes_[i] == my_payload_type) {
1979 if (UpdateUponReceivingCodec(i) != 0)
1984 // Codec is changed, there might be a jump in timestamp, therefore,
1985 // we have to reset some variables that track NetEq buffer.
1986 if (track_neteq_buffer_ || av_sync_) {
1987 last_incoming_send_timestamp_ = rtp_info.header.timestamp;
1990 if (nack_enabled_) {
1991 assert(nack_.get());
1992 // Codec is changed, reset NACK and update sampling rate.
1994 nack_->UpdateSampleRate(
1995 ACMCodecDB::database_[current_receive_codec_idx_].plfreq);
1998 last_recv_audio_codec_pltype_ = my_payload_type;
2001 // Current timestamp based on the receiver sampling frequency.
2002 last_receive_timestamp_ = NowTimestamp(current_receive_codec_idx_);
2004 if (nack_enabled_) {
2005 assert(nack_.get());
2006 nack_->UpdateLastReceivedPacket(rtp_header.header.sequenceNumber,
2007 rtp_header.header.timestamp);
2011 int per_neteq_payload_length = payload_length;
2012 // Split the payload for stereo packets, so that first half of payload
2013 // vector holds left channel, and second half holds right channel.
2014 if (expected_channels_ == 2) {
2015 if (!rtp_info.type.Audio.isCNG) {
2016 // Create a new vector for the payload, maximum payload size.
2017 int32_t length = payload_length;
2018 uint8_t payload[kMaxPacketSize];
2019 assert(payload_length <= kMaxPacketSize);
2020 memcpy(payload, incoming_payload, payload_length);
2021 codecs_[current_receive_codec_idx_]->SplitStereoPacket(payload, &length);
2022 rtp_header.type.Audio.channel = 2;
2023 per_neteq_payload_length = length / 2;
2024 // Insert packet into NetEQ.
2025 if (neteq_.RecIn(payload, length, rtp_header,
2026 last_receive_timestamp_) < 0)
2029 // If we receive a CNG packet while expecting stereo, we ignore the
2030 // packet and continue. CNG is not supported for stereo.
2034 if (neteq_.RecIn(incoming_payload, payload_length, rtp_header,
2035 last_receive_timestamp_) < 0)
2040 CriticalSectionScoped lock(acm_crit_sect_);
2042 // Update buffering uses |last_incoming_send_timestamp_| so it should be
2043 // before the next block.
2044 if (track_neteq_buffer_)
2045 UpdateBufferingSafe(rtp_header, per_neteq_payload_length);
2048 if (rtp_info.header.sequenceNumber == last_sequence_number_ + 1) {
2049 last_timestamp_diff_ = rtp_info.header.timestamp -
2050 last_incoming_send_timestamp_;
2052 last_sequence_number_ = rtp_info.header.sequenceNumber;
2053 last_ssrc_ = rtp_info.header.ssrc;
2054 last_packet_was_sync_ = false;
2057 if (av_sync_ || track_neteq_buffer_) {
2058 last_incoming_send_timestamp_ = rtp_info.header.timestamp;
2061 // Set the following regardless of tracking NetEq buffer or being in
2062 // AV-sync mode. Only if the received packet is not CNG.
2063 if (!rtp_info.type.Audio.isCNG)
2064 first_payload_received_ = true;
2069 int AudioCodingModuleImpl::UpdateUponReceivingCodec(int index) {
2070 if (codecs_[index] == NULL) {
2071 WEBRTC_TRACE(kTraceError, kTraceAudioCoding, id_,
2072 "IncomingPacket() error: payload type found but "
2073 "corresponding codec is NULL");
2076 codecs_[index]->UpdateDecoderSampFreq(index);
2077 neteq_.set_received_stereo(stereo_receive_[index]);
2078 current_receive_codec_idx_ = index;
2080 // If we have a change in the expected number of channels, flush packet
2081 // buffers in NetEQ.
2082 if ((stereo_receive_[index] && (expected_channels_ == 1)) ||
2083 (!stereo_receive_[index] && (expected_channels_ == 2))) {
2084 neteq_.FlushBuffers();
2085 codecs_[index]->ResetDecoder(registered_pltypes_[index]);
2088 if (stereo_receive_[index] && (expected_channels_ == 1)) {
2089 // When switching from a mono to stereo codec reset the slave.
2090 if (InitStereoSlave() != 0)
2094 // Store number of channels we expect to receive for the current payload type.
2095 if (stereo_receive_[index]) {
2096 expected_channels_ = 2;
2098 expected_channels_ = 1;
2101 // Reset previous received channel.
2102 prev_received_channel_ = 0;
2106 bool AudioCodingModuleImpl::IsCodecForSlave(int index) const {
2107 return (registered_pltypes_[index] != -1 && stereo_receive_[index]);
2110 int AudioCodingModuleImpl::InitStereoSlave() {
2111 neteq_.RemoveSlaves();
2113 if (neteq_.AddSlave(ACMCodecDB::NetEQDecoders(),
2114 ACMCodecDB::kNumCodecs) < 0) {
2115 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2116 "Cannot add slave jitter buffer to NetEQ.");
2120 // Register all needed codecs with slave.
2121 for (int i = 0; i < ACMCodecDB::kNumCodecs; i++) {
2122 if (codecs_[i] != NULL && IsCodecForSlave(i)) {
2123 WebRtcACMCodecParams decoder_params;
2124 if (codecs_[i]->DecoderParams(&decoder_params, registered_pltypes_[i])) {
2125 if (RegisterRecCodecMSSafe(decoder_params.codec_inst,
2126 i, ACMCodecDB::MirrorID(i),
2127 ACMNetEQ::kSlaveJb) < 0) {
2128 WEBRTC_TRACE(kTraceError, kTraceAudioCoding, id_,
2129 "Cannot register slave codec.");
2138 int AudioCodingModuleImpl::SetMinimumPlayoutDelay(int time_ms) {
2140 CriticalSectionScoped lock(acm_crit_sect_);
2141 // Don't let the extra delay modified while accumulating buffers in NetEq.
2142 if (track_neteq_buffer_ && first_payload_received_)
2145 return neteq_.SetMinimumDelay(time_ms);
2148 int AudioCodingModuleImpl::SetMaximumPlayoutDelay(int time_ms) {
2149 return neteq_.SetMaximumDelay(time_ms);
2152 // Get Dtmf playout status.
2153 bool AudioCodingModuleImpl::DtmfPlayoutStatus() const {
2154 #ifndef WEBRTC_CODEC_AVT
2157 return neteq_.avt_playout();
2161 // Configure Dtmf playout status i.e on/off playout the incoming outband
2163 int32_t AudioCodingModuleImpl::SetDtmfPlayoutStatus(
2164 #ifndef WEBRTC_CODEC_AVT
2165 const bool /* enable */) {
2166 WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceAudioCoding, id_,
2167 "SetDtmfPlayoutStatus() failed: AVT is not supported.");
2170 const bool enable) {
2171 return neteq_.SetAVTPlayout(enable);
2175 // Estimate the Bandwidth based on the incoming stream, needed for one way
2176 // audio where the RTCP send the BW estimate.
2177 // This is also done in the RTP module.
2178 int32_t AudioCodingModuleImpl::DecoderEstimatedBandwidth() const {
2180 int16_t codec_id = -1;
2184 // Get iSAC settings.
2185 for (int id = 0; id < ACMCodecDB::kNumCodecs; id++) {
2186 // Store codec settings for codec number "codeCntr" in the output struct.
2187 ACMCodecDB::Codec(id, &codec);
2189 if (!STR_CASE_CMP(codec.plname, "isac")) {
2191 pltype_wb = codec.pltype;
2193 ACMCodecDB::Codec(id + 1, &codec);
2194 pltype_swb = codec.pltype;
2201 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2202 "DecoderEstimatedBandwidth failed");
2206 if ((last_recv_audio_codec_pltype_ == pltype_wb) ||
2207 (last_recv_audio_codec_pltype_ == pltype_swb)) {
2208 return codecs_[codec_id]->GetEstimatedBandwidth();
2214 // Set playout mode for: voice, fax, or streaming.
2215 int32_t AudioCodingModuleImpl::SetPlayoutMode(
2216 const AudioPlayoutMode mode) {
2217 if ((mode != voice) && (mode != fax) && (mode != streaming) &&
2219 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2220 "Invalid playout mode.");
2223 return neteq_.SetPlayoutMode(mode);
2226 // Get playout mode voice, fax.
2227 AudioPlayoutMode AudioCodingModuleImpl::PlayoutMode() const {
2228 return neteq_.playout_mode();
2231 // Get 10 milliseconds of raw audio data to play out.
2232 // Automatic resample to the requested frequency.
2233 int32_t AudioCodingModuleImpl::PlayoutData10Ms(
2234 int32_t desired_freq_hz, AudioFrame* audio_frame) {
2235 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "ACM::PlayoutData10Ms", this);
2238 if (GetSilence(desired_freq_hz, audio_frame)) {
2239 TRACE_EVENT_ASYNC_END1("webrtc", "ACM::PlayoutData10Ms", this,
2241 return 0; // Silence is generated, return.
2244 // RecOut always returns 10 ms.
2245 if (neteq_.RecOut(audio_frame_) != 0) {
2246 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2247 "PlayoutData failed, RecOut Failed");
2250 int decoded_seq_num;
2251 uint32_t decoded_timestamp;
2253 neteq_.DecodedRtpInfo(&decoded_seq_num, &decoded_timestamp);
2255 // Master and Slave samples are interleaved starting with Master.
2256 uint16_t receive_freq;
2257 bool tone_detected = false;
2258 int16_t last_detected_tone;
2261 // Limit the scope of ACM Critical section.
2263 CriticalSectionScoped lock(acm_crit_sect_);
2265 audio_frame->num_channels_ = audio_frame_.num_channels_;
2266 audio_frame->vad_activity_ = audio_frame_.vad_activity_;
2267 audio_frame->speech_type_ = audio_frame_.speech_type_;
2269 stereo_mode = (audio_frame_.num_channels_ > 1);
2271 receive_freq = static_cast<uint16_t>(audio_frame_.sample_rate_hz_);
2272 // Update call statistics.
2273 call_stats_.DecodedByNetEq(audio_frame->speech_type_);
2275 if (nack_enabled_ && update_nack) {
2276 assert(nack_.get());
2277 nack_->UpdateLastDecodedPacket(decoded_seq_num, decoded_timestamp);
2280 // If we are in AV-sync and have already received an audio packet, but the
2281 // latest packet is too late, then insert sync packet.
2282 if (av_sync_ && first_payload_received_ &&
2283 NowTimestamp(current_receive_codec_idx_) > 5 * last_timestamp_diff_ +
2284 last_receive_timestamp_) {
2285 if (!last_packet_was_sync_) {
2286 // If the last packet inserted has been a regular packet Skip two
2287 // packets to give room for PLC.
2288 last_incoming_send_timestamp_ += 2 * last_timestamp_diff_;
2289 last_sequence_number_ += 2;
2290 last_receive_timestamp_ += 2 * last_timestamp_diff_;
2294 if (PushSyncPacketSafe() < 0)
2298 if ((receive_freq != desired_freq_hz) && (desired_freq_hz != -1)) {
2299 TRACE_EVENT_ASYNC_END2("webrtc", "ACM::PlayoutData10Ms", this,
2300 "seqnum", decoded_seq_num,
2301 "now", clock_->TimeInMilliseconds());
2302 // Resample payload_data.
2303 int16_t temp_len = output_resampler_.Resample10Msec(
2304 audio_frame_.data_, receive_freq, audio_frame->data_,
2305 desired_freq_hz, audio_frame_.num_channels_);
2308 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2309 "PlayoutData failed, resampler failed");
2313 // Set the payload data length from the resampler.
2314 audio_frame->samples_per_channel_ = static_cast<uint16_t>(temp_len);
2315 // Set the sampling frequency.
2316 audio_frame->sample_rate_hz_ = desired_freq_hz;
2318 TRACE_EVENT_ASYNC_END2("webrtc", "ACM::PlayoutData10Ms", this,
2319 "seqnum", decoded_seq_num,
2320 "now", clock_->TimeInMilliseconds());
2321 memcpy(audio_frame->data_, audio_frame_.data_,
2322 audio_frame_.samples_per_channel_ * audio_frame->num_channels_
2324 // Set the payload length.
2325 audio_frame->samples_per_channel_ =
2326 audio_frame_.samples_per_channel_;
2327 // Set the sampling frequency.
2328 audio_frame->sample_rate_hz_ = receive_freq;
2331 // Tone detection done for master channel.
2332 if (dtmf_detector_ != NULL) {
2334 if (audio_frame->sample_rate_hz_ == 8000) {
2335 // Use audio_frame->data_ then Dtmf detector doesn't
2338 dtmf_detector_->Detect(audio_frame->data_,
2339 audio_frame->samples_per_channel_,
2340 audio_frame->sample_rate_hz_, tone_detected,
2343 // We are in 8 kHz so the master channel needs only 80 samples.
2344 int16_t master_channel[80];
2345 for (int n = 0; n < 80; n++) {
2346 master_channel[n] = audio_frame->data_[n << 1];
2348 dtmf_detector_->Detect(master_channel,
2349 audio_frame->samples_per_channel_,
2350 audio_frame->sample_rate_hz_, tone_detected,
2354 // Do the detection on the audio that we got from NetEQ (audio_frame_).
2356 dtmf_detector_->Detect(audio_frame_.data_,
2357 audio_frame_.samples_per_channel_,
2358 receive_freq, tone_detected, tone);
2360 int16_t master_channel[WEBRTC_10MS_PCM_AUDIO];
2361 for (int n = 0; n < audio_frame_.samples_per_channel_; n++) {
2362 master_channel[n] = audio_frame_.data_[n << 1];
2364 dtmf_detector_->Detect(master_channel,
2365 audio_frame_.samples_per_channel_,
2366 receive_freq, tone_detected, tone);
2371 // We want to do this while we are in acm_crit_sect_.
2372 // (Doesn't really need to initialize the following
2373 // variable but Linux complains if we don't.)
2374 last_detected_tone = kACMToneEnd;
2375 if (tone_detected) {
2376 last_detected_tone = last_detected_tone_;
2377 last_detected_tone_ = tone;
2381 if (tone_detected) {
2382 // We will deal with callback here, so enter callback critical section.
2383 CriticalSectionScoped lock(callback_crit_sect_);
2385 if (dtmf_callback_ != NULL) {
2386 if (tone != kACMToneEnd) {
2388 dtmf_callback_->IncomingDtmf(static_cast<uint8_t>(tone), false);
2389 } else if ((tone == kACMToneEnd) && (last_detected_tone != kACMToneEnd)) {
2390 // The tone is "END" and the previously detected tone is
2391 // not "END," so call fir an end.
2392 dtmf_callback_->IncomingDtmf(static_cast<uint8_t>(last_detected_tone),
2398 audio_frame->id_ = id_;
2399 audio_frame->energy_ = -1;
2400 audio_frame->timestamp_ = 0;
2405 /////////////////////////////////////////
2409 int32_t AudioCodingModuleImpl::NetworkStatistics(
2410 ACMNetworkStatistics* statistics) {
2412 status = neteq_.NetworkStatistics(statistics);
2416 void AudioCodingModuleImpl::DestructEncoderInst(void* inst) {
2417 WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceAudioCoding, id_,
2418 "DestructEncoderInst()");
2419 if (!HaveValidEncoder("DestructEncoderInst")) {
2423 codecs_[current_send_codec_idx_]->DestructEncoderInst(inst);
2426 int16_t AudioCodingModuleImpl::AudioBuffer(
2427 WebRtcACMAudioBuff& buffer) {
2428 WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceAudioCoding, id_,
2430 if (!HaveValidEncoder("AudioBuffer")) {
2433 buffer.last_in_timestamp = last_in_timestamp_;
2434 return codecs_[current_send_codec_idx_]->AudioBuffer(buffer);
2437 int16_t AudioCodingModuleImpl::SetAudioBuffer(
2438 WebRtcACMAudioBuff& buffer) {
2439 WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceAudioCoding, id_,
2440 "SetAudioBuffer()");
2441 if (!HaveValidEncoder("SetAudioBuffer")) {
2444 return codecs_[current_send_codec_idx_]->SetAudioBuffer(buffer);
2447 uint32_t AudioCodingModuleImpl::EarliestTimestamp() const {
2448 WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceAudioCoding, id_,
2449 "EarliestTimestamp()");
2450 if (!HaveValidEncoder("EarliestTimestamp")) {
2453 return codecs_[current_send_codec_idx_]->EarliestTimestamp();
2456 int32_t AudioCodingModuleImpl::RegisterVADCallback(
2457 ACMVADCallback* vad_callback) {
2458 WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceAudioCoding, id_,
2459 "RegisterVADCallback()");
2460 CriticalSectionScoped lock(callback_crit_sect_);
2461 vad_callback_ = vad_callback;
2465 // TODO(turajs): Remove this API if it is not used.
2466 // TODO(tlegrand): Modify this function to work for stereo, and add tests.
2467 // TODO(turajs): Receive timestamp in this method is incremented by frame-size
2468 // and does not reflect the true receive frame-size. Therefore, subsequent
2469 // jitter computations are not accurate.
2470 int32_t AudioCodingModuleImpl::IncomingPayload(
2471 const uint8_t* incoming_payload, const int32_t payload_length,
2472 const uint8_t payload_type, const uint32_t timestamp) {
2473 if (payload_length < 0) {
2474 // Log error in trace file.
2475 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2476 "IncomingPacket() Error, payload-length cannot be negative");
2480 if (dummy_rtp_header_ == NULL) {
2481 // This is the first time that we are using |dummy_rtp_header_|
2482 // so we have to create it.
2483 WebRtcACMCodecParams codec_params;
2484 dummy_rtp_header_ = new WebRtcRTPHeader;
2485 if (dummy_rtp_header_ == NULL) {
2486 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2487 "IncomingPayload() Error, out of memory");
2490 dummy_rtp_header_->header.payloadType = payload_type;
2491 // Don't matter in this case.
2492 dummy_rtp_header_->header.ssrc = 0;
2493 dummy_rtp_header_->header.markerBit = false;
2494 // Start with random numbers.
2495 dummy_rtp_header_->header.sequenceNumber = rand();
2496 dummy_rtp_header_->header.timestamp =
2497 (static_cast<uint32_t>(rand()) << 16) +
2498 static_cast<uint32_t>(rand());
2499 dummy_rtp_header_->type.Audio.channel = 1;
2501 if (DecoderParamByPlType(payload_type, codec_params) < 0) {
2502 // We didn't find a codec with the given payload.
2503 // Something is wrong we exit, but we delete |dummy_rtp_header_|
2504 // and set it to NULL to start clean next time.
2505 delete dummy_rtp_header_;
2506 dummy_rtp_header_ = NULL;
2509 recv_pl_frame_size_smpls_ = codec_params.codec_inst.pacsize;
2512 if (payload_type != dummy_rtp_header_->header.payloadType) {
2513 // Payload type has changed since the last time we might need to
2514 // update the frame-size.
2515 WebRtcACMCodecParams codec_params;
2516 if (DecoderParamByPlType(payload_type, codec_params) < 0) {
2517 // We didn't find a codec with the given payload.
2520 recv_pl_frame_size_smpls_ = codec_params.codec_inst.pacsize;
2521 dummy_rtp_header_->header.payloadType = payload_type;
2524 if (timestamp > 0) {
2525 dummy_rtp_header_->header.timestamp = timestamp;
2528 // Store the payload Type. this will be used to retrieve "received codec"
2529 // and "received frequency."
2530 last_recv_audio_codec_pltype_ = payload_type;
2532 last_receive_timestamp_ += recv_pl_frame_size_smpls_;
2534 if (neteq_.RecIn(incoming_payload, payload_length, *dummy_rtp_header_,
2535 last_receive_timestamp_) < 0) {
2539 // Get ready for the next payload.
2540 dummy_rtp_header_->header.sequenceNumber++;
2541 dummy_rtp_header_->header.timestamp += recv_pl_frame_size_smpls_;
2545 int16_t AudioCodingModuleImpl::DecoderParamByPlType(
2546 const uint8_t payload_type,
2547 WebRtcACMCodecParams& codec_params) const {
2548 CriticalSectionScoped lock(acm_crit_sect_);
2549 for (int16_t id = 0; id < ACMCodecDB::kMaxNumCodecs;
2551 if (codecs_[id] != NULL) {
2552 if (codecs_[id]->DecoderInitialized()) {
2553 if (codecs_[id]->DecoderParams(&codec_params, payload_type)) {
2559 // If we are here it means that we could not find a
2560 // codec with that payload type. reset the values to
2561 // not acceptable values and return -1.
2562 codec_params.codec_inst.plname[0] = '\0';
2563 codec_params.codec_inst.pacsize = 0;
2564 codec_params.codec_inst.rate = 0;
2565 codec_params.codec_inst.pltype = -1;
2569 int16_t AudioCodingModuleImpl::DecoderListIDByPlName(
2570 const char* name, const uint16_t frequency) const {
2571 WebRtcACMCodecParams codec_params;
2572 CriticalSectionScoped lock(acm_crit_sect_);
2573 for (int16_t id = 0; id < ACMCodecDB::kMaxNumCodecs; id++) {
2574 if ((codecs_[id] != NULL)) {
2575 if (codecs_[id]->DecoderInitialized()) {
2576 assert(registered_pltypes_[id] >= 0);
2577 assert(registered_pltypes_[id] <= 255);
2578 codecs_[id]->DecoderParams(
2579 &codec_params, static_cast<uint8_t>(registered_pltypes_[id]));
2580 if (!STR_CASE_CMP(codec_params.codec_inst.plname, name)) {
2581 // Check if the given sampling frequency matches.
2582 // A zero sampling frequency means we matching the names
2583 // is sufficient and we don't need to check for the
2585 // Currently it is only iSAC which has one name but two
2586 // sampling frequencies.
2587 if ((frequency == 0)||
2588 (codec_params.codec_inst.plfreq == frequency)) {
2595 // If we are here it means that we could not find a
2596 // codec with that payload type. return -1.
2600 int32_t AudioCodingModuleImpl::LastEncodedTimestamp(
2601 uint32_t& timestamp) const {
2602 CriticalSectionScoped lock(acm_crit_sect_);
2603 if (!HaveValidEncoder("LastEncodedTimestamp")) {
2606 timestamp = codecs_[current_send_codec_idx_]->LastEncodedTimestamp();
2610 int32_t AudioCodingModuleImpl::ReplaceInternalDTXWithWebRtc(
2611 bool use_webrtc_dtx) {
2612 CriticalSectionScoped lock(acm_crit_sect_);
2614 if (!HaveValidEncoder("ReplaceInternalDTXWithWebRtc")) {
2616 webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2617 "Cannot replace codec internal DTX when no send codec is registered.");
2621 int32_t res = codecs_[current_send_codec_idx_]->ReplaceInternalDTX(
2623 // Check if VAD is turned on, or if there is any error.
2625 vad_enabled_ = true;
2626 } else if (res < 0) {
2627 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2628 "Failed to set ReplaceInternalDTXWithWebRtc(%d)",
2636 int32_t AudioCodingModuleImpl::IsInternalDTXReplacedWithWebRtc(
2637 bool* uses_webrtc_dtx) {
2638 CriticalSectionScoped lock(acm_crit_sect_);
2640 if (!HaveValidEncoder("IsInternalDTXReplacedWithWebRtc")) {
2643 if (codecs_[current_send_codec_idx_]->IsInternalDTXReplaced(uses_webrtc_dtx)
2650 int AudioCodingModuleImpl::SetISACMaxRate(int max_bit_per_sec) {
2651 CriticalSectionScoped lock(acm_crit_sect_);
2653 if (!HaveValidEncoder("SetISACMaxRate")) {
2657 return codecs_[current_send_codec_idx_]->SetISACMaxRate(max_bit_per_sec);
2660 int AudioCodingModuleImpl::SetISACMaxPayloadSize(int max_size_bytes) {
2661 CriticalSectionScoped lock(acm_crit_sect_);
2663 if (!HaveValidEncoder("SetISACMaxPayloadSize")) {
2667 return codecs_[current_send_codec_idx_]->SetISACMaxPayloadSize(
2671 int32_t AudioCodingModuleImpl::ConfigISACBandwidthEstimator(
2673 int rate_bit_per_sec,
2674 bool enforce_frame_size) {
2675 CriticalSectionScoped lock(acm_crit_sect_);
2677 if (!HaveValidEncoder("ConfigISACBandwidthEstimator")) {
2681 return codecs_[current_send_codec_idx_]->ConfigISACBandwidthEstimator(
2682 frame_size_ms, rate_bit_per_sec, enforce_frame_size);
2685 int32_t AudioCodingModuleImpl::PlayoutTimestamp(
2686 uint32_t* timestamp) {
2687 WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceAudioCoding, id_,
2688 "PlayoutTimestamp()");
2690 CriticalSectionScoped lock(acm_crit_sect_);
2691 if (track_neteq_buffer_) {
2692 *timestamp = playout_ts_;
2696 return neteq_.PlayoutTimestamp(*timestamp);
2699 bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
2700 if ((!send_codec_registered_) || (current_send_codec_idx_ < 0) ||
2701 (current_send_codec_idx_ >= ACMCodecDB::kNumCodecs)) {
2702 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2703 "%s failed: No send codec is registered.", caller_name);
2706 if ((current_send_codec_idx_ < 0) ||
2707 (current_send_codec_idx_ >= ACMCodecDB::kNumCodecs)) {
2708 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2709 "%s failed: Send codec index out of range.", caller_name);
2712 if (codecs_[current_send_codec_idx_] == NULL) {
2713 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2714 "%s failed: Send codec is NULL pointer.", caller_name);
2720 int AudioCodingModuleImpl::UnregisterReceiveCodec(uint8_t payload_type) {
2721 CriticalSectionScoped lock(acm_crit_sect_);
2724 // Search through the list of registered payload types.
2725 for (id = 0; id < ACMCodecDB::kMaxNumCodecs; id++) {
2726 if (registered_pltypes_[id] == payload_type) {
2727 // We have found the id registered with the payload type.
2732 if (id >= ACMCodecDB::kNumCodecs) {
2733 // Payload type was not registered. No need to unregister.
2737 // Unregister the codec with the given payload type.
2738 return UnregisterReceiveCodecSafe(id);
2741 int32_t AudioCodingModuleImpl::UnregisterReceiveCodecSafe(
2742 const int16_t codec_id) {
2743 const WebRtcNetEQDecoder *neteq_decoder = ACMCodecDB::NetEQDecoders();
2744 int16_t mirror_id = ACMCodecDB::MirrorID(codec_id);
2745 bool stereo_receiver = false;
2747 if (codecs_[codec_id] != NULL) {
2748 if (registered_pltypes_[codec_id] != -1) {
2749 // Store stereo information for future use.
2750 stereo_receiver = stereo_receive_[codec_id];
2752 // Before deleting the decoder instance unregister from NetEQ.
2753 if (neteq_.RemoveCodec(neteq_decoder[codec_id],
2754 stereo_receive_[codec_id]) < 0) {
2756 ACMCodecDB::Codec(codec_id, &codec);
2757 WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_,
2758 "Unregistering %s-%d from NetEQ failed.", codec.plname,
2763 // CN is a special case for NetEQ, all three sampling frequencies
2764 // are unregistered if one is deleted.
2765 if (IsCodecCN(codec_id)) {
2766 for (int i = 0; i < ACMCodecDB::kNumCodecs; i++) {
2768 stereo_receive_[i] = false;
2769 registered_pltypes_[i] = -1;
2773 if (codec_id == mirror_id) {
2774 codecs_[codec_id]->DestructDecoder();
2775 if (stereo_receive_[codec_id]) {
2776 slave_codecs_[codec_id]->DestructDecoder();
2777 stereo_receive_[codec_id] = false;
2782 // Check if this is the last registered stereo receive codec.
2783 if (stereo_receiver) {
2784 bool no_stereo = true;
2786 for (int i = 0; i < ACMCodecDB::kNumCodecs; i++) {
2787 if (stereo_receive_[i]) {
2788 // We still have stereo codecs registered.
2794 // If we don't have any stereo codecs left, change status.
2796 neteq_.RemoveSlaves(); // No longer need the slave.
2797 stereo_receive_registered_ = false;
2803 if (registered_pltypes_[codec_id] == receive_red_pltype_) {
2804 // RED is going to be unregistered, set to an invalid value.
2805 receive_red_pltype_ = 255;
2807 registered_pltypes_[codec_id] = -1;
2812 int32_t AudioCodingModuleImpl::REDPayloadISAC(
2813 const int32_t isac_rate, const int16_t isac_bw_estimate,
2814 uint8_t* payload, int16_t* length_bytes) {
2815 if (!HaveValidEncoder("EncodeData")) {
2819 status = codecs_[current_send_codec_idx_]->REDPayloadISAC(isac_rate,
2826 void AudioCodingModuleImpl::ResetFragmentation(int vector_size) {
2827 for (int n = 0; n < kMaxNumFragmentationVectors; n++) {
2828 fragmentation_.fragmentationOffset[n] = n * MAX_PAYLOAD_SIZE_BYTE;
2830 memset(fragmentation_.fragmentationLength, 0, kMaxNumFragmentationVectors *
2831 sizeof(fragmentation_.fragmentationLength[0]));
2832 memset(fragmentation_.fragmentationTimeDiff, 0, kMaxNumFragmentationVectors *
2833 sizeof(fragmentation_.fragmentationTimeDiff[0]));
2834 memset(fragmentation_.fragmentationPlType, 0, kMaxNumFragmentationVectors *
2835 sizeof(fragmentation_.fragmentationPlType[0]));
2836 fragmentation_.fragmentationVectorSize =
2837 static_cast<uint16_t>(vector_size);
2840 // TODO(turajs): Add second parameter to enable/disable AV-sync.
2841 int AudioCodingModuleImpl::SetInitialPlayoutDelay(int delay_ms) {
2842 if (delay_ms < 0 || delay_ms > 10000) {
2846 CriticalSectionScoped lock(acm_crit_sect_);
2848 // Receiver should be initialized before this call processed.
2849 if (!receiver_initialized_) {
2850 InitializeReceiverSafe();
2853 if (first_payload_received_) {
2854 // Too late for this API. Only works before a call is started.
2857 initial_delay_ms_ = delay_ms;
2859 // If initial delay is zero, NetEq buffer should not be tracked, also we
2860 // don't want to be in AV-sync mode.
2861 track_neteq_buffer_ = delay_ms > 0;
2862 av_sync_ = delay_ms > 0;
2864 neteq_.EnableAVSync(av_sync_);
2865 return neteq_.SetMinimumDelay(delay_ms);
2868 bool AudioCodingModuleImpl::GetSilence(int desired_sample_rate_hz,
2869 AudioFrame* frame) {
2870 CriticalSectionScoped lock(acm_crit_sect_);
2871 if (initial_delay_ms_ == 0 || !track_neteq_buffer_) {
2875 if (accumulated_audio_ms_ >= initial_delay_ms_) {
2876 // We have enough data stored that match our initial delay target.
2877 track_neteq_buffer_ = false;
2881 // Record call to silence generator.
2882 call_stats_.DecodedBySilenceGenerator();
2884 // We stop accumulating packets, if the number of packets or the total size
2885 // exceeds a threshold.
2886 int max_num_packets;
2887 int buffer_size_bytes;
2888 int per_payload_overhead_bytes;
2889 neteq_.BufferSpec(max_num_packets, buffer_size_bytes,
2890 per_payload_overhead_bytes);
2891 int total_bytes_accumulated = num_bytes_accumulated_ +
2892 num_packets_accumulated_ * per_payload_overhead_bytes;
2893 if (num_packets_accumulated_ > max_num_packets * 0.9 ||
2894 total_bytes_accumulated > buffer_size_bytes * 0.9) {
2895 WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceAudioCoding, id_,
2896 "GetSilence: Initial delay couldn't be achieved."
2897 " num_packets_accumulated=%d, total_bytes_accumulated=%d",
2898 num_packets_accumulated_, num_bytes_accumulated_);
2899 track_neteq_buffer_ = false;
2903 if (desired_sample_rate_hz > 0) {
2904 frame->sample_rate_hz_ = desired_sample_rate_hz;
2906 frame->sample_rate_hz_ = 0;
2907 if (current_receive_codec_idx_ >= 0) {
2908 frame->sample_rate_hz_ =
2909 ACMCodecDB::database_[current_receive_codec_idx_].plfreq;
2911 // No payload received yet, use the default sampling rate of NetEq.
2912 frame->sample_rate_hz_ = neteq_.CurrentSampFreqHz();
2915 frame->num_channels_ = expected_channels_;
2916 frame->samples_per_channel_ = frame->sample_rate_hz_ / 100; // Always 10 ms.
2917 frame->speech_type_ = AudioFrame::kCNG;
2918 frame->vad_activity_ = AudioFrame::kVadPassive;
2920 int samples = frame->samples_per_channel_ * frame->num_channels_;
2921 memset(frame->data_, 0, samples * sizeof(int16_t));
2925 // Must be called within the scope of ACM critical section.
2926 int AudioCodingModuleImpl::PushSyncPacketSafe() {
2928 last_sequence_number_++;
2929 last_incoming_send_timestamp_ += last_timestamp_diff_;
2930 last_receive_timestamp_ += last_timestamp_diff_;
2932 WebRtcRTPHeader rtp_info;
2933 rtp_info.header.payloadType = last_recv_audio_codec_pltype_;
2934 rtp_info.header.ssrc = last_ssrc_;
2935 rtp_info.header.markerBit = false;
2936 rtp_info.header.sequenceNumber = last_sequence_number_;
2937 rtp_info.header.timestamp = last_incoming_send_timestamp_;
2938 rtp_info.type.Audio.channel = stereo_receive_[current_receive_codec_idx_] ?
2940 last_packet_was_sync_ = true;
2941 int payload_len_bytes = neteq_.RecIn(rtp_info, last_receive_timestamp_);
2943 if (payload_len_bytes < 0)
2946 // This is to account for sync packets inserted during the buffering phase.
2947 if (track_neteq_buffer_)
2948 UpdateBufferingSafe(rtp_info, payload_len_bytes);
2953 // Must be called within the scope of ACM critical section.
2954 void AudioCodingModuleImpl::UpdateBufferingSafe(const WebRtcRTPHeader& rtp_info,
2955 int payload_len_bytes) {
2956 const int in_sample_rate_khz =
2957 (ACMCodecDB::database_[current_receive_codec_idx_].plfreq / 1000);
2958 if (first_payload_received_ &&
2959 rtp_info.header.timestamp > last_incoming_send_timestamp_ &&
2960 in_sample_rate_khz > 0) {
2961 accumulated_audio_ms_ += (rtp_info.header.timestamp -
2962 last_incoming_send_timestamp_) / in_sample_rate_khz;
2965 num_packets_accumulated_++;
2966 num_bytes_accumulated_ += payload_len_bytes;
2968 playout_ts_ = static_cast<uint32_t>(
2969 rtp_info.header.timestamp - static_cast<uint32_t>(
2970 initial_delay_ms_ * in_sample_rate_khz));
2973 uint32_t AudioCodingModuleImpl::NowTimestamp(int codec_id) {
2974 // Down-cast the time to (32-6)-bit since we only care about
2975 // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms.
2976 // we masked 6 most significant bits of 32-bit so we don't lose resolution
2977 // when do the following multiplication.
2978 int sample_rate_khz = ACMCodecDB::database_[codec_id].plfreq / 1000;
2979 const uint32_t now_in_ms = static_cast<uint32_t>(
2980 clock_->TimeInMilliseconds() & kMaskTimestamp);
2981 return static_cast<uint32_t>(sample_rate_khz * now_in_ms);
2984 std::vector<uint16_t> AudioCodingModuleImpl::GetNackList(
2985 int round_trip_time_ms) const {
2986 CriticalSectionScoped lock(acm_crit_sect_);
2987 if (round_trip_time_ms < 0) {
2988 WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceAudioCoding, id_,
2989 "GetNackList: round trip time cannot be negative."
2990 " round_trip_time_ms=%d", round_trip_time_ms);
2992 if (nack_enabled_ && round_trip_time_ms >= 0) {
2993 assert(nack_.get());
2994 return nack_->GetNackList(round_trip_time_ms);
2996 std::vector<uint16_t> empty_list;
3000 int AudioCodingModuleImpl::LeastRequiredDelayMs() const {
3001 return std::max(neteq_.LeastRequiredDelayMs(), initial_delay_ms_);
3004 int AudioCodingModuleImpl::EnableNack(size_t max_nack_list_size) {
3005 // Don't do anything if |max_nack_list_size| is out of range.
3006 if (max_nack_list_size == 0 ||
3007 max_nack_list_size > acm2::Nack::kNackListSizeLimit)
3010 CriticalSectionScoped lock(acm_crit_sect_);
3011 if (!nack_enabled_) {
3012 nack_.reset(acm2::Nack::Create(kNackThresholdPackets));
3013 nack_enabled_ = true;
3015 // Sampling rate might need to be updated if we change from disable to
3016 // enable. Do it if the receive codec is valid.
3017 if (current_receive_codec_idx_ >= 0) {
3018 nack_->UpdateSampleRate(
3019 ACMCodecDB::database_[current_receive_codec_idx_].plfreq);
3022 return nack_->SetMaxNackListSize(max_nack_list_size);
3025 void AudioCodingModuleImpl::DisableNack() {
3026 CriticalSectionScoped lock(acm_crit_sect_);
3027 nack_.reset(); // Memory is released.
3028 nack_enabled_ = false;
3031 void AudioCodingModuleImpl::GetDecodingCallStatistics(
3032 AudioDecodingCallStats* call_stats) const {
3033 CriticalSectionScoped lock(acm_crit_sect_);
3034 *call_stats = call_stats_.GetDecodingStatistics();
3039 } // namespace webrtc