src/media/cast/audio_receiver/audio_receiver.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/audio_receiver/audio_receiver.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/message_loop/message_loop.h"
  10 #include "base/strings/string_piece.h"
  11 #include "media/cast/audio_receiver/audio_decoder.h"
  12 #include "media/cast/framer/framer.h"
  13 #include "media/cast/rtcp/receiver_rtcp_event_subscriber.h"
  14 #include "media/cast/rtcp/rtcp.h"
  15 #include "media/cast/rtp_receiver/rtp_receiver.h"
  16 #include "media/cast/transport/cast_transport_defines.h"
  17
  18 namespace {
  19
  20 using media::cast::kMaxIpPacketSize;
  21 using media::cast::kRtcpCastLogHeaderSize;
  22 using media::cast::kRtcpReceiverEventLogSize;
  23
  24 // Max time we wait until an audio frame is due to be played out is released.
  25 static const int64 kMaxAudioFrameWaitMs = 20;
  26 static const int64 kMinSchedulingDelayMs = 1;
  27
  28 // This is an upper bound on number of events that can fit into a single RTCP
  29 // packet.
  30 static const int64 kMaxEventSubscriberEntries =
  31     (kMaxIpPacketSize - kRtcpCastLogHeaderSize) / kRtcpReceiverEventLogSize;
  32
  33 }  // namespace
  34
  35 namespace media {
  36 namespace cast {
  37
  38 DecodedAudioCallbackData::DecodedAudioCallbackData()
  39     : number_of_10ms_blocks(0), desired_frequency(0), callback() {}
  40
  41 DecodedAudioCallbackData::~DecodedAudioCallbackData() {}
  42
  43 // Local implementation of RtpData (defined in rtp_rtcp_defines.h).
  44 // Used to pass payload data into the audio receiver.
  45 class LocalRtpAudioData : public RtpData {
  46  public:
  47   explicit LocalRtpAudioData(AudioReceiver* audio_receiver)
  48       : audio_receiver_(audio_receiver) {}
  49
  50   virtual void OnReceivedPayloadData(const uint8* payload_data,
  51                                      size_t payload_size,
  52                                      const RtpCastHeader* rtp_header) OVERRIDE {
  53     audio_receiver_->IncomingParsedRtpPacket(payload_data, payload_size,
  54                                              *rtp_header);
  55   }
  56
  57  private:
  58   AudioReceiver* audio_receiver_;
  59 };
  60
  61 // Local implementation of RtpPayloadFeedback (defined in rtp_defines.h)
  62 // Used to convey cast-specific feedback from receiver to sender.
  63 class LocalRtpAudioFeedback : public RtpPayloadFeedback {
  64  public:
  65   explicit LocalRtpAudioFeedback(AudioReceiver* audio_receiver)
  66       : audio_receiver_(audio_receiver) {}
  67
  68   virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE {
  69     audio_receiver_->CastFeedback(cast_message);
  70   }
  71
  72  private:
  73   AudioReceiver* audio_receiver_;
  74 };
  75
  76 class LocalRtpReceiverStatistics : public RtpReceiverStatistics {
  77  public:
  78   explicit LocalRtpReceiverStatistics(RtpReceiver* rtp_receiver)
  79       : rtp_receiver_(rtp_receiver) {}
  80
  81   virtual void GetStatistics(uint8* fraction_lost,
  82                              uint32* cumulative_lost,  // 24 bits valid.
  83                              uint32* extended_high_sequence_number,
  84                              uint32* jitter) OVERRIDE {
  85     rtp_receiver_->GetStatistics(fraction_lost, cumulative_lost,
  86                                  extended_high_sequence_number, jitter);
  87   }
  88
  89  private:
  90   RtpReceiver* rtp_receiver_;
  91 };
  92
  93 AudioReceiver::AudioReceiver(scoped_refptr<CastEnvironment> cast_environment,
  94                              const AudioReceiverConfig& audio_config,
  95                              transport::PacedPacketSender* const packet_sender)
  96     : cast_environment_(cast_environment),
  97       event_subscriber_(
  98           kMaxEventSubscriberEntries,
  99           ReceiverRtcpEventSubscriber::kAudioEventSubscriber),
 100       codec_(audio_config.codec),
 101       frequency_(audio_config.frequency),
 102       audio_buffer_(),
 103       audio_decoder_(),
 104       time_offset_(),
 105       weak_factory_(this) {
 106   target_delay_delta_ =
 107       base::TimeDelta::FromMilliseconds(audio_config.rtp_max_delay_ms);
 108   incoming_payload_callback_.reset(new LocalRtpAudioData(this));
 109   incoming_payload_feedback_.reset(new LocalRtpAudioFeedback(this));
 110   if (audio_config.use_external_decoder) {
 111     audio_buffer_.reset(new Framer(cast_environment->Clock(),
 112                                    incoming_payload_feedback_.get(),
 113                                    audio_config.incoming_ssrc, true, 0));
 114   } else {
 115     audio_decoder_.reset(new AudioDecoder(cast_environment, audio_config,
 116                                           incoming_payload_feedback_.get()));
 117   }
 118   decryptor_.Initialize(audio_config.aes_key, audio_config.aes_iv_mask);
 119   rtp_receiver_.reset(new RtpReceiver(cast_environment->Clock(),
 120                                       &audio_config,
 121                                       NULL,
 122                                       incoming_payload_callback_.get()));
 123   rtp_audio_receiver_statistics_.reset(
 124       new LocalRtpReceiverStatistics(rtp_receiver_.get()));
 125   base::TimeDelta rtcp_interval_delta =
 126       base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval);
 127   rtcp_.reset(new Rtcp(cast_environment, NULL, NULL, packet_sender, NULL,
 128                        rtp_audio_receiver_statistics_.get(),
 129                        audio_config.rtcp_mode, rtcp_interval_delta,
 130                        audio_config.feedback_ssrc, audio_config.incoming_ssrc,
 131                        audio_config.rtcp_c_name));
 132   cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_);
 133 }
 134
 135 AudioReceiver::~AudioReceiver() {
 136   cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_);
 137 }
 138
 139 void AudioReceiver::InitializeTimers() {
 140   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 141   ScheduleNextRtcpReport();
 142   ScheduleNextCastMessage();
 143 }
 144
 145 void AudioReceiver::IncomingParsedRtpPacket(const uint8* payload_data,
 146                                             size_t payload_size,
 147                                             const RtpCastHeader& rtp_header) {
 148   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 149   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 150
 151   cast_environment_->Logging()->InsertPacketEvent(
 152       now, kAudioPacketReceived, rtp_header.webrtc.header.timestamp,
 153       rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id,
 154       payload_size);
 155
 156   // TODO(pwestin): update this as video to refresh over time.
 157   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 158   if (time_first_incoming_packet_.is_null()) {
 159     InitializeTimers();
 160     first_incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp;
 161     time_first_incoming_packet_ = now;
 162   }
 163
 164   if (audio_decoder_) {
 165     DCHECK(!audio_buffer_) << "Invalid internal state";
 166     std::string plaintext;
 167     if (decryptor_.initialized()) {
 168       if (!decryptor_.Decrypt(
 169                rtp_header.frame_id,
 170                base::StringPiece(reinterpret_cast<const char*>(payload_data),
 171                                  payload_size),
 172                &plaintext))
 173         return;
 174     } else {
 175       plaintext.append(reinterpret_cast<const char*>(payload_data),
 176                        payload_size);
 177     }
 178     audio_decoder_->IncomingParsedRtpPacket(
 179         reinterpret_cast<const uint8*>(plaintext.data()), plaintext.size(),
 180         rtp_header);
 181     if (!queued_decoded_callbacks_.empty()) {
 182       DecodedAudioCallbackData decoded_data = queued_decoded_callbacks_.front();
 183       queued_decoded_callbacks_.pop_front();
 184       cast_environment_->PostTask(
 185           CastEnvironment::AUDIO_DECODER, FROM_HERE,
 186           base::Bind(&AudioReceiver::DecodeAudioFrameThread,
 187                      base::Unretained(this), decoded_data.number_of_10ms_blocks,
 188                      decoded_data.desired_frequency, decoded_data.callback));
 189     }
 190     return;
 191   }
 192
 193   DCHECK(audio_buffer_) << "Invalid internal state";
 194   DCHECK(!audio_decoder_) << "Invalid internal state";
 195
 196   bool duplicate = false;
 197   bool complete = audio_buffer_->InsertPacket(payload_data, payload_size,
 198                                               rtp_header, &duplicate);
 199   if (duplicate) {
 200     cast_environment_->Logging()->InsertPacketEvent(
 201         now, kDuplicateAudioPacketReceived, rtp_header.webrtc.header.timestamp,
 202         rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id,
 203         payload_size);
 204     // Duplicate packets are ignored.
 205     return;
 206   }
 207   if (!complete) return;  // Audio frame not complete; wait for more packets.
 208   if (queued_encoded_callbacks_.empty()) return;
 209   AudioFrameEncodedCallback callback = queued_encoded_callbacks_.front();
 210   queued_encoded_callbacks_.pop_front();
 211   cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 212                               base::Bind(&AudioReceiver::GetEncodedAudioFrame,
 213                                          weak_factory_.GetWeakPtr(), callback));
 214 }
 215
 216 void AudioReceiver::GetRawAudioFrame(
 217     int number_of_10ms_blocks, int desired_frequency,
 218     const AudioFrameDecodedCallback& callback) {
 219   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 220   DCHECK(audio_decoder_) << "Invalid function call in this configuration";
 221   // TODO(pwestin): we can skip this function by posting direct to the decoder.
 222   cast_environment_->PostTask(
 223       CastEnvironment::AUDIO_DECODER, FROM_HERE,
 224       base::Bind(&AudioReceiver::DecodeAudioFrameThread, base::Unretained(this),
 225                  number_of_10ms_blocks, desired_frequency, callback));
 226 }
 227
 228 void AudioReceiver::DecodeAudioFrameThread(
 229     int number_of_10ms_blocks, int desired_frequency,
 230     const AudioFrameDecodedCallback callback) {
 231   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::AUDIO_DECODER));
 232   // TODO(mikhal): Allow the application to allocate this memory.
 233   scoped_ptr<PcmAudioFrame> audio_frame(new PcmAudioFrame());
 234
 235   uint32 rtp_timestamp = 0;
 236   if (!audio_decoder_->GetRawAudioFrame(number_of_10ms_blocks,
 237                                         desired_frequency, audio_frame.get(),
 238                                         &rtp_timestamp)) {
 239     DecodedAudioCallbackData callback_data;
 240     callback_data.number_of_10ms_blocks = number_of_10ms_blocks;
 241     callback_data.desired_frequency = desired_frequency;
 242     callback_data.callback = callback;
 243     queued_decoded_callbacks_.push_back(callback_data);
 244     return;
 245   }
 246
 247   cast_environment_->PostTask(
 248       CastEnvironment::MAIN, FROM_HERE,
 249       base::Bind(&AudioReceiver::ReturnDecodedFrameWithPlayoutDelay,
 250                  base::Unretained(this), base::Passed(&audio_frame),
 251                  rtp_timestamp, callback));
 252 }
 253
 254 void AudioReceiver::ReturnDecodedFrameWithPlayoutDelay(
 255     scoped_ptr<PcmAudioFrame> audio_frame, uint32 rtp_timestamp,
 256     const AudioFrameDecodedCallback callback) {
 257   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 258   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 259   cast_environment_->Logging()->InsertFrameEvent(
 260       now, kAudioFrameDecoded, rtp_timestamp, kFrameIdUnknown);
 261
 262   base::TimeTicks playout_time = GetPlayoutTime(now, rtp_timestamp);
 263
 264   cast_environment_->Logging()->InsertFrameEventWithDelay(
 265       now, kAudioPlayoutDelay, rtp_timestamp, kFrameIdUnknown,
 266       playout_time - now);
 267
 268   // Frame is ready - Send back to the caller.
 269   cast_environment_->PostTask(
 270       CastEnvironment::MAIN, FROM_HERE,
 271       base::Bind(callback, base::Passed(&audio_frame), playout_time));
 272 }
 273
 274 void AudioReceiver::PlayoutTimeout() {
 275   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 276   DCHECK(audio_buffer_) << "Invalid function call in this configuration";
 277   if (queued_encoded_callbacks_.empty()) {
 278     // Already released by incoming packet.
 279     return;
 280   }
 281   bool next_frame = false;
 282   scoped_ptr<transport::EncodedAudioFrame> encoded_frame(
 283       new transport::EncodedAudioFrame());
 284
 285   if (!audio_buffer_->GetEncodedAudioFrame(encoded_frame.get(), &next_frame)) {
 286     // We have no audio frames. Wait for new packet(s).
 287     // Since the application can post multiple AudioFrameEncodedCallback and
 288     // we only check the next frame to play out we might have multiple timeout
 289     // events firing after each other; however this should be a rare event.
 290     VLOG(1) << "Failed to retrieved a complete frame at this point in time";
 291     return;
 292   }
 293
 294   if (decryptor_.initialized() && !DecryptAudioFrame(&encoded_frame)) {
 295     // Logging already done.
 296     return;
 297   }
 298
 299   if (PostEncodedAudioFrame(
 300           queued_encoded_callbacks_.front(), next_frame, &encoded_frame)) {
 301     // Call succeed remove callback from list.
 302     queued_encoded_callbacks_.pop_front();
 303   }
 304 }
 305
 306 void AudioReceiver::GetEncodedAudioFrame(
 307     const AudioFrameEncodedCallback& callback) {
 308   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 309   DCHECK(audio_buffer_) << "Invalid function call in this configuration";
 310
 311   bool next_frame = false;
 312   scoped_ptr<transport::EncodedAudioFrame> encoded_frame(
 313       new transport::EncodedAudioFrame());
 314
 315   if (!audio_buffer_->GetEncodedAudioFrame(encoded_frame.get(), &next_frame)) {
 316     // We have no audio frames. Wait for new packet(s).
 317     VLOG(1) << "Wait for more audio packets in frame";
 318     queued_encoded_callbacks_.push_back(callback);
 319     return;
 320   }
 321   if (decryptor_.initialized() && !DecryptAudioFrame(&encoded_frame)) {
 322     // Logging already done.
 323     queued_encoded_callbacks_.push_back(callback);
 324     return;
 325   }
 326   if (!PostEncodedAudioFrame(callback, next_frame, &encoded_frame)) {
 327     // We have an audio frame; however we are missing packets and we have time
 328     // to wait for new packet(s).
 329     queued_encoded_callbacks_.push_back(callback);
 330   }
 331 }
 332
 333 bool AudioReceiver::PostEncodedAudioFrame(
 334     const AudioFrameEncodedCallback& callback,
 335     bool next_frame,
 336     scoped_ptr<transport::EncodedAudioFrame>* encoded_frame) {
 337   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 338   DCHECK(audio_buffer_) << "Invalid function call in this configuration";
 339   DCHECK(encoded_frame) << "Invalid encoded_frame";
 340
 341   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 342   base::TimeTicks playout_time =
 343       GetPlayoutTime(now, (*encoded_frame)->rtp_timestamp);
 344   base::TimeDelta time_until_playout = playout_time - now;
 345   base::TimeDelta min_wait_delta =
 346       base::TimeDelta::FromMilliseconds(kMaxAudioFrameWaitMs);
 347
 348   if (!next_frame && (time_until_playout > min_wait_delta)) {
 349     base::TimeDelta time_until_release = time_until_playout - min_wait_delta;
 350     cast_environment_->PostDelayedTask(
 351         CastEnvironment::MAIN, FROM_HERE,
 352         base::Bind(&AudioReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()),
 353         time_until_release);
 354     VLOG(1) << "Wait until time to playout:"
 355             << time_until_release.InMilliseconds();
 356     return false;
 357   }
 358   (*encoded_frame)->codec = codec_;
 359   audio_buffer_->ReleaseFrame((*encoded_frame)->frame_id);
 360
 361   cast_environment_->PostTask(
 362       CastEnvironment::MAIN, FROM_HERE,
 363       base::Bind(callback, base::Passed(encoded_frame), playout_time));
 364   return true;
 365 }
 366
 367 void AudioReceiver::IncomingPacket(scoped_ptr<Packet> packet) {
 368   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 369   bool rtcp_packet = Rtcp::IsRtcpPacket(&packet->front(), packet->size());
 370   if (!rtcp_packet) {
 371     rtp_receiver_->ReceivedPacket(&packet->front(), packet->size());
 372   } else {
 373     rtcp_->IncomingRtcpPacket(&packet->front(), packet->size());
 374   }
 375 }
 376
 377 void AudioReceiver::CastFeedback(const RtcpCastMessage& cast_message) {
 378   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 379   cast_environment_->Logging()->InsertGenericEvent(now, kAudioAckSent,
 380                                                    cast_message.ack_frame_id_);
 381
 382   rtcp_->SendRtcpFromRtpReceiver(&cast_message, &event_subscriber_);
 383 }
 384
 385 base::TimeTicks AudioReceiver::GetPlayoutTime(base::TimeTicks now,
 386                                               uint32 rtp_timestamp) {
 387   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 388   // Senders time in ms when this frame was recorded.
 389   // Note: the senders clock and our local clock might not be synced.
 390   base::TimeTicks rtp_timestamp_in_ticks;
 391   base::TimeTicks playout_time;
 392   if (time_offset_ == base::TimeDelta()) {
 393     if (rtcp_->RtpTimestampInSenderTime(frequency_,
 394                                         first_incoming_rtp_timestamp_,
 395                                         &rtp_timestamp_in_ticks)) {
 396       time_offset_ = time_first_incoming_packet_ - rtp_timestamp_in_ticks;
 397     } else {
 398       // We have not received any RTCP to sync the stream play it out as soon as
 399       // possible.
 400       uint32 rtp_timestamp_diff = rtp_timestamp - first_incoming_rtp_timestamp_;
 401
 402       int frequency_khz = frequency_ / 1000;
 403       base::TimeDelta rtp_time_diff_delta =
 404           base::TimeDelta::FromMilliseconds(rtp_timestamp_diff / frequency_khz);
 405       base::TimeDelta time_diff_delta = now - time_first_incoming_packet_;
 406
 407       playout_time = now + std::max(rtp_time_diff_delta - time_diff_delta,
 408                                     base::TimeDelta());
 409     }
 410   }
 411   if (playout_time.is_null()) {
 412     // This can fail if we have not received any RTCP packets in a long time.
 413     if (rtcp_->RtpTimestampInSenderTime(frequency_, rtp_timestamp,
 414                                         &rtp_timestamp_in_ticks)) {
 415       playout_time =
 416           rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_;
 417     } else {
 418       playout_time = now;
 419     }
 420   }
 421   // Don't allow the playout time to go backwards.
 422   if (last_playout_time_ > playout_time) playout_time = last_playout_time_;
 423   last_playout_time_ = playout_time;
 424   return playout_time;
 425 }
 426
 427 bool AudioReceiver::DecryptAudioFrame(
 428     scoped_ptr<transport::EncodedAudioFrame>* audio_frame) {
 429   if (!decryptor_.initialized())
 430     return false;
 431
 432   std::string decrypted_audio_data;
 433   if (!decryptor_.Decrypt((*audio_frame)->frame_id,
 434                           (*audio_frame)->data,
 435                           &decrypted_audio_data)) {
 436     // Give up on this frame, release it from the jitter buffer.
 437     audio_buffer_->ReleaseFrame((*audio_frame)->frame_id);
 438     return false;
 439   }
 440   (*audio_frame)->data.swap(decrypted_audio_data);
 441   return true;
 442 }
 443
 444 void AudioReceiver::ScheduleNextRtcpReport() {
 445   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 446   base::TimeDelta time_to_send = rtcp_->TimeToSendNextRtcpReport() -
 447                                  cast_environment_->Clock()->NowTicks();
 448
 449   time_to_send = std::max(
 450       time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 451
 452   cast_environment_->PostDelayedTask(
 453       CastEnvironment::MAIN, FROM_HERE,
 454       base::Bind(&AudioReceiver::SendNextRtcpReport,
 455                  weak_factory_.GetWeakPtr()),
 456       time_to_send);
 457 }
 458
 459 void AudioReceiver::SendNextRtcpReport() {
 460   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 461   // TODO(pwestin): add logging.
 462   rtcp_->SendRtcpFromRtpReceiver(NULL, NULL);
 463   ScheduleNextRtcpReport();
 464 }
 465
 466 // Cast messages should be sent within a maximum interval. Schedule a call
 467 // if not triggered elsewhere, e.g. by the cast message_builder.
 468 void AudioReceiver::ScheduleNextCastMessage() {
 469   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 470   base::TimeTicks send_time;
 471   if (audio_buffer_) {
 472     audio_buffer_->TimeToSendNextCastMessage(&send_time);
 473   } else if (audio_decoder_) {
 474     audio_decoder_->TimeToSendNextCastMessage(&send_time);
 475   } else {
 476     NOTREACHED();
 477   }
 478   base::TimeDelta time_to_send =
 479       send_time - cast_environment_->Clock()->NowTicks();
 480   time_to_send = std::max(
 481       time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 482   cast_environment_->PostDelayedTask(
 483       CastEnvironment::MAIN, FROM_HERE,
 484       base::Bind(&AudioReceiver::SendNextCastMessage,
 485                  weak_factory_.GetWeakPtr()),
 486       time_to_send);
 487 }
 488
 489 void AudioReceiver::SendNextCastMessage() {
 490   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 491
 492   if (audio_buffer_) {
 493     // Will only send a message if it is time.
 494     audio_buffer_->SendCastMessage();
 495   }
 496   if (audio_decoder_) {
 497     // Will only send a message if it is time.
 498     audio_decoder_->SendCastMessage();
 499   }
 500   ScheduleNextCastMessage();
 501 }
 502
 503 }  // namespace cast
 504 }  // namespace media