// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-#include "base/logging.h"
#include "media/cast/audio_receiver/audio_decoder.h"
-#include "third_party/webrtc/modules/audio_coding/main/interface/audio_coding_module.h"
-#include "third_party/webrtc/modules/interface/module_common_types.h"
+#include "base/bind.h"
+#include "base/bind_helpers.h"
+#include "base/location.h"
+#include "base/logging.h"
+#include "base/memory/ref_counted.h"
+#include "base/stl_util.h"
+#include "base/sys_byteorder.h"
+#include "media/cast/cast_defines.h"
+#include "third_party/opus/src/include/opus.h"
namespace media {
namespace cast {
-AudioDecoder::AudioDecoder(scoped_refptr<CastEnvironment> cast_environment,
- const AudioReceiverConfig& audio_config,
- RtpPayloadFeedback* incoming_payload_feedback)
- : cast_environment_(cast_environment),
- audio_decoder_(webrtc::AudioCodingModule::Create(0)),
- cast_message_builder_(cast_environment->Clock(),
- incoming_payload_feedback,
- &frame_id_map_,
- audio_config.incoming_ssrc,
- true,
- 0),
- have_received_packets_(false),
- last_played_out_timestamp_(0) {
- audio_decoder_->InitializeReceiver();
-
- webrtc::CodecInst receive_codec;
- switch (audio_config.codec) {
- case transport::kPcm16:
- receive_codec.pltype = audio_config.rtp_payload_type;
- strncpy(receive_codec.plname, "L16", 4);
- receive_codec.plfreq = audio_config.frequency;
- receive_codec.pacsize = -1;
- receive_codec.channels = audio_config.channels;
- receive_codec.rate = -1;
- break;
- case transport::kOpus:
- receive_codec.pltype = audio_config.rtp_payload_type;
- strncpy(receive_codec.plname, "opus", 5);
- receive_codec.plfreq = audio_config.frequency;
- receive_codec.pacsize = -1;
- receive_codec.channels = audio_config.channels;
- receive_codec.rate = -1;
- break;
- case transport::kExternalAudio:
- NOTREACHED() << "Codec must be specified for audio decoder";
- break;
- }
- if (audio_decoder_->RegisterReceiveCodec(receive_codec) != 0) {
- NOTREACHED() << "Failed to register receive codec";
+// Base class that handles the common problem of detecting dropped frames, and
+// then invoking the Decode() method implemented by the subclasses to convert
+// the encoded payload data into usable audio data.
+class AudioDecoder::ImplBase
+ : public base::RefCountedThreadSafe<AudioDecoder::ImplBase> {
+ public:
+ ImplBase(const scoped_refptr<CastEnvironment>& cast_environment,
+ transport::AudioCodec codec,
+ int num_channels,
+ int sampling_rate)
+ : cast_environment_(cast_environment),
+ codec_(codec),
+ num_channels_(num_channels),
+ cast_initialization_status_(STATUS_AUDIO_UNINITIALIZED),
+ seen_first_frame_(false) {
+ if (num_channels_ <= 0 || sampling_rate <= 0 || sampling_rate % 100 != 0)
+ cast_initialization_status_ = STATUS_INVALID_AUDIO_CONFIGURATION;
}
- audio_decoder_->SetMaximumPlayoutDelay(audio_config.rtp_max_delay_ms);
- audio_decoder_->SetPlayoutMode(webrtc::streaming);
-}
+ CastInitializationStatus InitializationResult() const {
+ return cast_initialization_status_;
+ }
-AudioDecoder::~AudioDecoder() {}
+ void DecodeFrame(scoped_ptr<transport::EncodedAudioFrame> encoded_frame,
+ const DecodeFrameCallback& callback) {
+ DCHECK_EQ(cast_initialization_status_, STATUS_AUDIO_INITIALIZED);
-bool AudioDecoder::GetRawAudioFrame(int number_of_10ms_blocks,
- int desired_frequency,
- PcmAudioFrame* audio_frame,
- uint32* rtp_timestamp) {
- DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::AUDIO));
- // We don't care about the race case where a packet arrives at the same time
- // as this function in called. The data will be there the next time this
- // function is called.
- lock_.Acquire();
- // Get a local copy under lock.
- bool have_received_packets = have_received_packets_;
- lock_.Release();
-
- if (!have_received_packets)
- return false;
-
- audio_frame->samples.clear();
-
- for (int i = 0; i < number_of_10ms_blocks; ++i) {
- webrtc::AudioFrame webrtc_audio_frame;
- if (0 != audio_decoder_->PlayoutData10Ms(desired_frequency,
- &webrtc_audio_frame)) {
- return false;
+ scoped_ptr<AudioBus> decoded_audio;
+ if (encoded_frame->codec != codec_) {
+ NOTREACHED();
+ cast_environment_->PostTask(CastEnvironment::MAIN,
+ FROM_HERE,
+ base::Bind(callback,
+ base::Passed(&decoded_audio),
+ false));
}
- if (webrtc_audio_frame.speech_type_ == webrtc::AudioFrame::kPLCCNG ||
- webrtc_audio_frame.speech_type_ == webrtc::AudioFrame::kUndefined) {
- // We are only interested in real decoded audio.
- return false;
- }
- audio_frame->frequency = webrtc_audio_frame.sample_rate_hz_;
- audio_frame->channels = webrtc_audio_frame.num_channels_;
- if (i == 0) {
- // Use the timestamp from the first 10ms block.
- if (0 != audio_decoder_->PlayoutTimestamp(rtp_timestamp)) {
- return false;
+ COMPILE_ASSERT(sizeof(encoded_frame->frame_id) == sizeof(last_frame_id_),
+ size_of_frame_id_types_do_not_match);
+ bool is_continuous = true;
+ if (seen_first_frame_) {
+ const uint32 frames_ahead = encoded_frame->frame_id - last_frame_id_;
+ if (frames_ahead > 1) {
+ RecoverBecauseFramesWereDropped();
+ is_continuous = false;
}
- lock_.Acquire();
- last_played_out_timestamp_ = *rtp_timestamp;
- lock_.Release();
+ } else {
+ seen_first_frame_ = true;
}
- int samples_per_10ms = webrtc_audio_frame.samples_per_channel_;
+ last_frame_id_ = encoded_frame->frame_id;
- audio_frame->samples.insert(
- audio_frame->samples.end(),
- &webrtc_audio_frame.data_[0],
- &webrtc_audio_frame.data_[samples_per_10ms * audio_frame->channels]);
+ decoded_audio = Decode(
+ reinterpret_cast<uint8*>(string_as_array(&encoded_frame->data)),
+ static_cast<int>(encoded_frame->data.size()));
+ cast_environment_->PostTask(CastEnvironment::MAIN,
+ FROM_HERE,
+ base::Bind(callback,
+ base::Passed(&decoded_audio),
+ is_continuous));
}
- return true;
-}
-void AudioDecoder::IncomingParsedRtpPacket(const uint8* payload_data,
- size_t payload_size,
- const RtpCastHeader& rtp_header) {
- DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
- DCHECK_LE(payload_size, kMaxIpPacketSize);
- audio_decoder_->IncomingPacket(
- payload_data, static_cast<int32>(payload_size), rtp_header.webrtc);
- lock_.Acquire();
- have_received_packets_ = true;
- uint32 last_played_out_timestamp = last_played_out_timestamp_;
- lock_.Release();
-
- PacketType packet_type = frame_id_map_.InsertPacket(rtp_header);
- if (packet_type != kNewPacketCompletingFrame)
- return;
+ protected:
+ friend class base::RefCountedThreadSafe<ImplBase>;
+ virtual ~ImplBase() {}
- cast_message_builder_.CompleteFrameReceived(rtp_header.frame_id,
- rtp_header.is_key_frame);
+ virtual void RecoverBecauseFramesWereDropped() {}
- frame_id_rtp_timestamp_map_[rtp_header.frame_id] =
- rtp_header.webrtc.header.timestamp;
+ // Note: Implementation of Decode() is allowed to mutate |data|.
+ virtual scoped_ptr<AudioBus> Decode(uint8* data, int len) = 0;
- if (last_played_out_timestamp == 0)
- return; // Nothing is played out yet.
+ const scoped_refptr<CastEnvironment> cast_environment_;
+ const transport::AudioCodec codec_;
+ const int num_channels_;
- uint32 latest_frame_id_to_remove = 0;
- bool frame_to_remove = false;
+ // Subclass' ctor is expected to set this to STATUS_AUDIO_INITIALIZED.
+ CastInitializationStatus cast_initialization_status_;
- FrameIdRtpTimestampMap::iterator it = frame_id_rtp_timestamp_map_.begin();
- while (it != frame_id_rtp_timestamp_map_.end()) {
- if (IsNewerRtpTimestamp(it->second, last_played_out_timestamp)) {
- break;
+ private:
+ bool seen_first_frame_;
+ uint32 last_frame_id_;
+
+ DISALLOW_COPY_AND_ASSIGN(ImplBase);
+};
+
+class AudioDecoder::OpusImpl : public AudioDecoder::ImplBase {
+ public:
+ OpusImpl(const scoped_refptr<CastEnvironment>& cast_environment,
+ int num_channels,
+ int sampling_rate)
+ : ImplBase(cast_environment,
+ transport::kOpus,
+ num_channels,
+ sampling_rate),
+ decoder_memory_(new uint8[opus_decoder_get_size(num_channels)]),
+ opus_decoder_(reinterpret_cast<OpusDecoder*>(decoder_memory_.get())),
+ max_samples_per_frame_(
+ kOpusMaxFrameDurationMillis * sampling_rate / 1000),
+ buffer_(new float[max_samples_per_frame_ * num_channels]) {
+ if (ImplBase::cast_initialization_status_ != STATUS_AUDIO_UNINITIALIZED)
+ return;
+ if (opus_decoder_init(opus_decoder_, sampling_rate, num_channels) !=
+ OPUS_OK) {
+ ImplBase::cast_initialization_status_ =
+ STATUS_INVALID_AUDIO_CONFIGURATION;
+ return;
}
- frame_to_remove = true;
- latest_frame_id_to_remove = it->first;
- frame_id_rtp_timestamp_map_.erase(it);
- it = frame_id_rtp_timestamp_map_.begin();
+ ImplBase::cast_initialization_status_ = STATUS_AUDIO_INITIALIZED;
}
- if (!frame_to_remove)
- return;
- frame_id_map_.RemoveOldFrames(latest_frame_id_to_remove);
+ private:
+ virtual ~OpusImpl() {}
+
+ virtual void RecoverBecauseFramesWereDropped() OVERRIDE {
+ // Passing NULL for the input data notifies the decoder of frame loss.
+ const opus_int32 result =
+ opus_decode_float(
+ opus_decoder_, NULL, 0, buffer_.get(), max_samples_per_frame_, 0);
+ DCHECK_GE(result, 0);
+ }
+
+ virtual scoped_ptr<AudioBus> Decode(uint8* data, int len) OVERRIDE {
+ scoped_ptr<AudioBus> audio_bus;
+ const opus_int32 num_samples_decoded = opus_decode_float(
+ opus_decoder_, data, len, buffer_.get(), max_samples_per_frame_, 0);
+ if (num_samples_decoded <= 0)
+ return audio_bus.Pass(); // Decode error.
+
+ // Copy interleaved samples from |buffer_| into a new AudioBus (where
+ // samples are stored in planar format, for each channel).
+ audio_bus = AudioBus::Create(num_channels_, num_samples_decoded).Pass();
+ // TODO(miu): This should be moved into AudioBus::FromInterleaved().
+ for (int ch = 0; ch < num_channels_; ++ch) {
+ const float* src = buffer_.get() + ch;
+ const float* const src_end = src + num_samples_decoded * num_channels_;
+ float* dest = audio_bus->channel(ch);
+ for (; src < src_end; src += num_channels_, ++dest)
+ *dest = *src;
+ }
+ return audio_bus.Pass();
+ }
+
+ const scoped_ptr<uint8[]> decoder_memory_;
+ OpusDecoder* const opus_decoder_;
+ const int max_samples_per_frame_;
+ const scoped_ptr<float[]> buffer_;
+
+ // According to documentation in third_party/opus/src/include/opus.h, we must
+ // provide enough space in |buffer_| to contain 120ms of samples. At 48 kHz,
+ // then, that means 5760 samples times the number of channels.
+ static const int kOpusMaxFrameDurationMillis = 120;
+
+ DISALLOW_COPY_AND_ASSIGN(OpusImpl);
+};
+
+class AudioDecoder::Pcm16Impl : public AudioDecoder::ImplBase {
+ public:
+ Pcm16Impl(const scoped_refptr<CastEnvironment>& cast_environment,
+ int num_channels,
+ int sampling_rate)
+ : ImplBase(cast_environment,
+ transport::kPcm16,
+ num_channels,
+ sampling_rate) {
+ if (ImplBase::cast_initialization_status_ != STATUS_AUDIO_UNINITIALIZED)
+ return;
+ ImplBase::cast_initialization_status_ = STATUS_AUDIO_INITIALIZED;
+ }
+
+ private:
+ virtual ~Pcm16Impl() {}
+
+ virtual scoped_ptr<AudioBus> Decode(uint8* data, int len) OVERRIDE {
+ scoped_ptr<AudioBus> audio_bus;
+ const int num_samples = len / sizeof(int16) / num_channels_;
+ if (num_samples <= 0)
+ return audio_bus.Pass();
+
+ int16* const pcm_data = reinterpret_cast<int16*>(data);
+#if defined(ARCH_CPU_LITTLE_ENDIAN)
+ // Convert endianness.
+ const int num_elements = num_samples * num_channels_;
+ for (int i = 0; i < num_elements; ++i)
+ pcm_data[i] = static_cast<int16>(base::NetToHost16(pcm_data[i]));
+#endif
+ audio_bus = AudioBus::Create(num_channels_, num_samples).Pass();
+ audio_bus->FromInterleaved(pcm_data, num_samples, sizeof(int16));
+ return audio_bus.Pass();
+ }
+
+ DISALLOW_COPY_AND_ASSIGN(Pcm16Impl);
+};
+
+AudioDecoder::AudioDecoder(
+ const scoped_refptr<CastEnvironment>& cast_environment,
+ const AudioReceiverConfig& audio_config)
+ : cast_environment_(cast_environment) {
+ switch (audio_config.codec) {
+ case transport::kOpus:
+ impl_ = new OpusImpl(cast_environment,
+ audio_config.channels,
+ audio_config.frequency);
+ break;
+ case transport::kPcm16:
+ impl_ = new Pcm16Impl(cast_environment,
+ audio_config.channels,
+ audio_config.frequency);
+ break;
+ default:
+ NOTREACHED() << "Unknown or unspecified codec.";
+ break;
+ }
}
-bool AudioDecoder::TimeToSendNextCastMessage(base::TimeTicks* time_to_send) {
- DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
- return cast_message_builder_.TimeToSendNextCastMessage(time_to_send);
+AudioDecoder::~AudioDecoder() {}
+
+CastInitializationStatus AudioDecoder::InitializationResult() const {
+ if (impl_)
+ return impl_->InitializationResult();
+ return STATUS_UNSUPPORTED_AUDIO_CODEC;
}
-void AudioDecoder::SendCastMessage() {
- DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
- cast_message_builder_.UpdateCastMessage();
+void AudioDecoder::DecodeFrame(
+ scoped_ptr<transport::EncodedAudioFrame> encoded_frame,
+ const DecodeFrameCallback& callback) {
+ DCHECK(encoded_frame.get());
+ DCHECK(!callback.is_null());
+ if (!impl_ || impl_->InitializationResult() != STATUS_AUDIO_INITIALIZED) {
+ callback.Run(make_scoped_ptr<AudioBus>(NULL), false);
+ return;
+ }
+ cast_environment_->PostTask(CastEnvironment::AUDIO,
+ FROM_HERE,
+ base::Bind(&AudioDecoder::ImplBase::DecodeFrame,
+ impl_,
+ base::Passed(&encoded_frame),
+ callback));
}
} // namespace cast
projects / platform / framework / web / crosswalk.git / blobdiff