src/media/base/android/media_decoder_job.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/base/android/media_decoder_job.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback_helpers.h"
   9 #include "base/debug/trace_event.h"
  10 #include "base/message_loop/message_loop_proxy.h"
  11 #include "media/base/android/media_codec_bridge.h"
  12 #include "media/base/bind_to_current_loop.h"
  13 #include "media/base/buffers.h"
  14
  15 namespace media {
  16
  17 // Timeout value for media codec operations. Because the first
  18 // DequeInputBuffer() can take about 150 milliseconds, use 250 milliseconds
  19 // here. See http://b/9357571.
  20 static const int kMediaCodecTimeoutInMilliseconds = 250;
  21
  22 MediaDecoderJob::MediaDecoderJob(
  23     const scoped_refptr<base::SingleThreadTaskRunner>& decoder_task_runner,
  24     MediaCodecBridge* media_codec_bridge,
  25     const base::Closure& request_data_cb)
  26     : ui_task_runner_(base::MessageLoopProxy::current()),
  27       decoder_task_runner_(decoder_task_runner),
  28       media_codec_bridge_(media_codec_bridge),
  29       needs_flush_(false),
  30       input_eos_encountered_(false),
  31       output_eos_encountered_(false),
  32       skip_eos_enqueue_(true),
  33       prerolling_(true),
  34       request_data_cb_(request_data_cb),
  35       current_demuxer_data_index_(0),
  36       input_buf_index_(-1),
  37       stop_decode_pending_(false),
  38       destroy_pending_(false),
  39       is_requesting_demuxer_data_(false),
  40       is_incoming_data_invalid_(false),
  41       weak_factory_(this) {
  42   InitializeReceivedData();
  43 }
  44
  45 MediaDecoderJob::~MediaDecoderJob() {}
  46
  47 void MediaDecoderJob::OnDataReceived(const DemuxerData& data) {
  48   DVLOG(1) << __FUNCTION__ << ": " << data.access_units.size() << " units";
  49   DCHECK(ui_task_runner_->BelongsToCurrentThread());
  50   DCHECK(NoAccessUnitsRemainingInChunk(false));
  51
  52   TRACE_EVENT_ASYNC_END2(
  53       "media", "MediaDecoderJob::RequestData", this,
  54       "Data type", data.type == media::DemuxerStream::AUDIO ? "AUDIO" : "VIDEO",
  55       "Units read", data.access_units.size());
  56
  57   if (is_incoming_data_invalid_) {
  58     is_incoming_data_invalid_ = false;
  59
  60     // If there is a pending callback, need to request the data again to get
  61     // valid data.
  62     if (!on_data_received_cb_.is_null())
  63       request_data_cb_.Run();
  64     else
  65       is_requesting_demuxer_data_ = false;
  66     return;
  67   }
  68
  69   size_t next_demuxer_data_index = inactive_demuxer_data_index();
  70   received_data_[next_demuxer_data_index] = data;
  71   access_unit_index_[next_demuxer_data_index] = 0;
  72   is_requesting_demuxer_data_ = false;
  73
  74   base::Closure done_cb = base::ResetAndReturn(&on_data_received_cb_);
  75
  76   // If this data request is for the inactive chunk, or |on_data_received_cb_|
  77   // was set to null by ClearData() or Release(), do nothing.
  78   if (done_cb.is_null())
  79     return;
  80
  81   if (stop_decode_pending_) {
  82     DCHECK(is_decoding());
  83     OnDecodeCompleted(MEDIA_CODEC_STOPPED, kNoTimestamp(), 0);
  84     return;
  85   }
  86
  87   done_cb.Run();
  88 }
  89
  90 void MediaDecoderJob::Prefetch(const base::Closure& prefetch_cb) {
  91   DCHECK(ui_task_runner_->BelongsToCurrentThread());
  92   DCHECK(on_data_received_cb_.is_null());
  93   DCHECK(decode_cb_.is_null());
  94
  95   if (HasData()) {
  96     DVLOG(1) << __FUNCTION__ << " : using previously received data";
  97     ui_task_runner_->PostTask(FROM_HERE, prefetch_cb);
  98     return;
  99   }
 100
 101   DVLOG(1) << __FUNCTION__ << " : requesting data";
 102   RequestData(prefetch_cb);
 103 }
 104
 105 bool MediaDecoderJob::Decode(
 106     base::TimeTicks start_time_ticks,
 107     base::TimeDelta start_presentation_timestamp,
 108     const DecoderCallback& callback) {
 109   DCHECK(decode_cb_.is_null());
 110   DCHECK(on_data_received_cb_.is_null());
 111   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 112
 113   decode_cb_ = callback;
 114
 115   if (!HasData()) {
 116     RequestData(base::Bind(&MediaDecoderJob::DecodeCurrentAccessUnit,
 117                            base::Unretained(this),
 118                            start_time_ticks,
 119                            start_presentation_timestamp));
 120     return true;
 121   }
 122
 123   if (DemuxerStream::kConfigChanged == CurrentAccessUnit().status) {
 124     // Clear received data because we need to handle a config change.
 125     decode_cb_.Reset();
 126     ClearData();
 127     return false;
 128   }
 129
 130   DecodeCurrentAccessUnit(start_time_ticks, start_presentation_timestamp);
 131   return true;
 132 }
 133
 134 void MediaDecoderJob::StopDecode() {
 135   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 136   DCHECK(is_decoding());
 137   stop_decode_pending_ = true;
 138 }
 139
 140 void MediaDecoderJob::Flush() {
 141   DCHECK(decode_cb_.is_null());
 142
 143   // Do nothing, flush when the next Decode() happens.
 144   needs_flush_ = true;
 145   ClearData();
 146 }
 147
 148 void MediaDecoderJob::BeginPrerolling(base::TimeDelta preroll_timestamp) {
 149   DVLOG(1) << __FUNCTION__ << "(" << preroll_timestamp.InSecondsF() << ")";
 150   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 151   DCHECK(!is_decoding());
 152
 153   preroll_timestamp_ = preroll_timestamp;
 154   prerolling_ = true;
 155 }
 156
 157 void MediaDecoderJob::Release() {
 158   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 159   DVLOG(1) << __FUNCTION__;
 160
 161   // If the decoder job is not waiting for data, and is still decoding, we
 162   // cannot delete the job immediately.
 163   destroy_pending_ = on_data_received_cb_.is_null() && is_decoding();
 164
 165   request_data_cb_.Reset();
 166   on_data_received_cb_.Reset();
 167   decode_cb_.Reset();
 168
 169   if (destroy_pending_) {
 170     DVLOG(1) << __FUNCTION__ << " : delete is pending decode completion";
 171     return;
 172   }
 173
 174   delete this;
 175 }
 176
 177 MediaCodecStatus MediaDecoderJob::QueueInputBuffer(const AccessUnit& unit) {
 178   DVLOG(1) << __FUNCTION__;
 179   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
 180   TRACE_EVENT0("media", __FUNCTION__);
 181
 182   int input_buf_index = input_buf_index_;
 183   input_buf_index_ = -1;
 184
 185   // TODO(xhwang): Hide DequeueInputBuffer() and the index in MediaCodecBridge.
 186   if (input_buf_index == -1) {
 187     base::TimeDelta timeout = base::TimeDelta::FromMilliseconds(
 188         kMediaCodecTimeoutInMilliseconds);
 189     MediaCodecStatus status =
 190         media_codec_bridge_->DequeueInputBuffer(timeout, &input_buf_index);
 191     if (status != MEDIA_CODEC_OK) {
 192       DVLOG(1) << "DequeueInputBuffer fails: " << status;
 193       return status;
 194     }
 195   }
 196
 197   // TODO(qinmin): skip frames if video is falling far behind.
 198   DCHECK_GE(input_buf_index, 0);
 199   if (unit.end_of_stream || unit.data.empty()) {
 200     media_codec_bridge_->QueueEOS(input_buf_index);
 201     return MEDIA_CODEC_INPUT_END_OF_STREAM;
 202   }
 203
 204   if (unit.key_id.empty() || unit.iv.empty()) {
 205     DCHECK(unit.iv.empty() || !unit.key_id.empty());
 206     return media_codec_bridge_->QueueInputBuffer(
 207         input_buf_index, &unit.data[0], unit.data.size(), unit.timestamp);
 208   }
 209
 210   MediaCodecStatus status = media_codec_bridge_->QueueSecureInputBuffer(
 211       input_buf_index,
 212       &unit.data[0], unit.data.size(),
 213       reinterpret_cast<const uint8*>(&unit.key_id[0]), unit.key_id.size(),
 214       reinterpret_cast<const uint8*>(&unit.iv[0]), unit.iv.size(),
 215       unit.subsamples.empty() ? NULL : &unit.subsamples[0],
 216       unit.subsamples.size(),
 217       unit.timestamp);
 218
 219   // In case of MEDIA_CODEC_NO_KEY, we must reuse the |input_buf_index_|.
 220   // Otherwise MediaDrm will report errors.
 221   if (status == MEDIA_CODEC_NO_KEY)
 222     input_buf_index_ = input_buf_index;
 223
 224   return status;
 225 }
 226
 227 bool MediaDecoderJob::HasData() const {
 228   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 229   // When |input_eos_encountered_| is set, |access_unit_index_| and
 230   // |current_demuxer_data_index_| must be pointing to an EOS unit.
 231   // We'll reuse this unit to flush the decoder until we hit output EOS.
 232   DCHECK(!input_eos_encountered_ || !NoAccessUnitsRemainingInChunk(true));
 233   return !NoAccessUnitsRemainingInChunk(true) ||
 234       !NoAccessUnitsRemainingInChunk(false);
 235 }
 236
 237 void MediaDecoderJob::RequestData(const base::Closure& done_cb) {
 238   DVLOG(1) << __FUNCTION__;
 239   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 240   DCHECK(on_data_received_cb_.is_null());
 241   DCHECK(!input_eos_encountered_);
 242   DCHECK(NoAccessUnitsRemainingInChunk(false));
 243
 244   TRACE_EVENT_ASYNC_BEGIN0("media", "MediaDecoderJob::RequestData", this);
 245
 246   on_data_received_cb_ = done_cb;
 247
 248   // If we are already expecting new data, just set the callback and do
 249   // nothing.
 250   if (is_requesting_demuxer_data_)
 251     return;
 252
 253   // The new incoming data will be stored as the next demuxer data chunk, since
 254   // the decoder might still be decoding the current one.
 255   size_t next_demuxer_data_index = inactive_demuxer_data_index();
 256   received_data_[next_demuxer_data_index] = DemuxerData();
 257   access_unit_index_[next_demuxer_data_index] = 0;
 258   is_requesting_demuxer_data_ = true;
 259
 260   request_data_cb_.Run();
 261 }
 262
 263 void MediaDecoderJob::DecodeCurrentAccessUnit(
 264     base::TimeTicks start_time_ticks,
 265     base::TimeDelta start_presentation_timestamp) {
 266   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 267   DCHECK(!decode_cb_.is_null());
 268
 269   RequestCurrentChunkIfEmpty();
 270   const AccessUnit& access_unit = CurrentAccessUnit();
 271   // If the first access unit is a config change, request the player to dequeue
 272   // the input buffer again so that it can request config data.
 273   if (access_unit.status == DemuxerStream::kConfigChanged) {
 274     ui_task_runner_->PostTask(FROM_HERE,
 275                               base::Bind(&MediaDecoderJob::OnDecodeCompleted,
 276                                          base::Unretained(this),
 277                                          MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER,
 278                                          kNoTimestamp(),
 279                                          0));
 280     return;
 281   }
 282
 283   decoder_task_runner_->PostTask(FROM_HERE, base::Bind(
 284       &MediaDecoderJob::DecodeInternal, base::Unretained(this),
 285       access_unit,
 286       start_time_ticks, start_presentation_timestamp, needs_flush_,
 287       media::BindToCurrentLoop(base::Bind(
 288           &MediaDecoderJob::OnDecodeCompleted, base::Unretained(this)))));
 289   needs_flush_ = false;
 290 }
 291
 292 void MediaDecoderJob::DecodeInternal(
 293     const AccessUnit& unit,
 294     base::TimeTicks start_time_ticks,
 295     base::TimeDelta start_presentation_timestamp,
 296     bool needs_flush,
 297     const MediaDecoderJob::DecoderCallback& callback) {
 298   DVLOG(1) << __FUNCTION__;
 299   DCHECK(decoder_task_runner_->BelongsToCurrentThread());
 300   TRACE_EVENT0("media", __FUNCTION__);
 301
 302   if (needs_flush) {
 303     DVLOG(1) << "DecodeInternal needs flush.";
 304     input_eos_encountered_ = false;
 305     output_eos_encountered_ = false;
 306     MediaCodecStatus reset_status = media_codec_bridge_->Reset();
 307     if (MEDIA_CODEC_OK != reset_status) {
 308       callback.Run(reset_status, kNoTimestamp(), 0);
 309       return;
 310     }
 311   }
 312
 313   // Once output EOS has occurred, we should not be asked to decode again.
 314   // MediaCodec has undefined behavior if similarly asked to decode after output
 315   // EOS.
 316   DCHECK(!output_eos_encountered_);
 317
 318   // For aborted access unit, just skip it and inform the player.
 319   if (unit.status == DemuxerStream::kAborted) {
 320     // TODO(qinmin): use a new enum instead of MEDIA_CODEC_STOPPED.
 321     callback.Run(MEDIA_CODEC_STOPPED, kNoTimestamp(), 0);
 322     return;
 323   }
 324
 325   if (skip_eos_enqueue_) {
 326     if (unit.end_of_stream || unit.data.empty()) {
 327       input_eos_encountered_ = true;
 328       output_eos_encountered_ = true;
 329       callback.Run(MEDIA_CODEC_OUTPUT_END_OF_STREAM, kNoTimestamp(), 0);
 330       return;
 331     }
 332
 333     skip_eos_enqueue_ = false;
 334   }
 335
 336   MediaCodecStatus input_status = MEDIA_CODEC_INPUT_END_OF_STREAM;
 337   if (!input_eos_encountered_) {
 338     input_status = QueueInputBuffer(unit);
 339     if (input_status == MEDIA_CODEC_INPUT_END_OF_STREAM) {
 340       input_eos_encountered_ = true;
 341     } else if (input_status != MEDIA_CODEC_OK) {
 342       callback.Run(input_status, kNoTimestamp(), 0);
 343       return;
 344     }
 345   }
 346
 347   int buffer_index = 0;
 348   size_t offset = 0;
 349   size_t size = 0;
 350   base::TimeDelta presentation_timestamp;
 351
 352   base::TimeDelta timeout = base::TimeDelta::FromMilliseconds(
 353       kMediaCodecTimeoutInMilliseconds);
 354
 355   MediaCodecStatus status =
 356       media_codec_bridge_->DequeueOutputBuffer(timeout,
 357                                                &buffer_index,
 358                                                &offset,
 359                                                &size,
 360                                                &presentation_timestamp,
 361                                                &output_eos_encountered_,
 362                                                NULL);
 363
 364   if (status != MEDIA_CODEC_OK) {
 365     if (status == MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED &&
 366         !media_codec_bridge_->GetOutputBuffers()) {
 367       status = MEDIA_CODEC_ERROR;
 368     }
 369     callback.Run(status, kNoTimestamp(), 0);
 370     return;
 371   }
 372
 373   // TODO(xhwang/qinmin): This logic is correct but strange. Clean it up.
 374   if (output_eos_encountered_)
 375     status = MEDIA_CODEC_OUTPUT_END_OF_STREAM;
 376   else if (input_status == MEDIA_CODEC_INPUT_END_OF_STREAM)
 377     status = MEDIA_CODEC_INPUT_END_OF_STREAM;
 378
 379   bool render_output  = presentation_timestamp >= preroll_timestamp_ &&
 380       (status != MEDIA_CODEC_OUTPUT_END_OF_STREAM || size != 0u);
 381   base::TimeDelta time_to_render;
 382   DCHECK(!start_time_ticks.is_null());
 383   if (render_output && ComputeTimeToRender()) {
 384     time_to_render = presentation_timestamp - (base::TimeTicks::Now() -
 385         start_time_ticks + start_presentation_timestamp);
 386   }
 387
 388   if (time_to_render > base::TimeDelta()) {
 389     decoder_task_runner_->PostDelayedTask(
 390         FROM_HERE,
 391         base::Bind(&MediaDecoderJob::ReleaseOutputBuffer,
 392                    weak_factory_.GetWeakPtr(),
 393                    buffer_index,
 394                    size,
 395                    render_output,
 396                    base::Bind(callback, status, presentation_timestamp)),
 397         time_to_render);
 398     return;
 399   }
 400
 401   // TODO(qinmin): The codec is lagging behind, need to recalculate the
 402   // |start_presentation_timestamp_| and |start_time_ticks_| in
 403   // media_source_player.cc.
 404   DVLOG(1) << "codec is lagging behind :" << time_to_render.InMicroseconds();
 405   if (render_output) {
 406     // The player won't expect a timestamp smaller than the
 407     // |start_presentation_timestamp|. However, this could happen due to decoder
 408     // errors.
 409     presentation_timestamp = std::max(
 410         presentation_timestamp, start_presentation_timestamp);
 411   } else {
 412     presentation_timestamp = kNoTimestamp();
 413   }
 414   ReleaseOutputCompletionCallback completion_callback = base::Bind(
 415       callback, status, presentation_timestamp);
 416   ReleaseOutputBuffer(buffer_index, size, render_output, completion_callback);
 417 }
 418
 419 void MediaDecoderJob::OnDecodeCompleted(
 420     MediaCodecStatus status, base::TimeDelta presentation_timestamp,
 421     size_t audio_output_bytes) {
 422   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 423
 424   if (destroy_pending_) {
 425     DVLOG(1) << __FUNCTION__ << " : completing pending deletion";
 426     delete this;
 427     return;
 428   }
 429
 430   DCHECK(!decode_cb_.is_null());
 431
 432   // If output was queued for rendering, then we have completed prerolling.
 433   if (presentation_timestamp != kNoTimestamp())
 434     prerolling_ = false;
 435
 436   switch (status) {
 437     case MEDIA_CODEC_OK:
 438     case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
 439     case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
 440     case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:
 441     case MEDIA_CODEC_OUTPUT_END_OF_STREAM:
 442       if (!input_eos_encountered_)
 443         access_unit_index_[current_demuxer_data_index_]++;
 444       break;
 445
 446     case MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER:
 447     case MEDIA_CODEC_INPUT_END_OF_STREAM:
 448     case MEDIA_CODEC_NO_KEY:
 449     case MEDIA_CODEC_STOPPED:
 450     case MEDIA_CODEC_ERROR:
 451       // Do nothing.
 452       break;
 453   };
 454
 455   stop_decode_pending_ = false;
 456   base::ResetAndReturn(&decode_cb_).Run(status, presentation_timestamp,
 457                                         audio_output_bytes);
 458 }
 459
 460 const AccessUnit& MediaDecoderJob::CurrentAccessUnit() const {
 461   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 462   DCHECK(HasData());
 463   int index = NoAccessUnitsRemainingInChunk(true) ?
 464       inactive_demuxer_data_index() : current_demuxer_data_index_;
 465   return received_data_[index].access_units[access_unit_index_[index]];
 466 }
 467
 468 bool MediaDecoderJob::NoAccessUnitsRemainingInChunk(
 469     bool is_active_chunk) const {
 470   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 471   size_t index = is_active_chunk ? current_demuxer_data_index_ :
 472       inactive_demuxer_data_index();
 473   return received_data_[index].access_units.size() <= access_unit_index_[index];
 474 }
 475
 476 void MediaDecoderJob::ClearData() {
 477   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 478   current_demuxer_data_index_ = 0;
 479   InitializeReceivedData();
 480   on_data_received_cb_.Reset();
 481   if (is_requesting_demuxer_data_)
 482     is_incoming_data_invalid_ = true;
 483   input_eos_encountered_ = false;
 484 }
 485
 486 void MediaDecoderJob::RequestCurrentChunkIfEmpty() {
 487   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 488   DCHECK(HasData());
 489   if (!NoAccessUnitsRemainingInChunk(true))
 490     return;
 491
 492   // Requests new data if the the last access unit of the next chunk is not EOS.
 493   current_demuxer_data_index_ = inactive_demuxer_data_index();
 494   const AccessUnit last_access_unit =
 495       received_data_[current_demuxer_data_index_].access_units.back();
 496   if (!last_access_unit.end_of_stream &&
 497       last_access_unit.status != DemuxerStream::kConfigChanged &&
 498       last_access_unit.status != DemuxerStream::kAborted) {
 499     RequestData(base::Closure());
 500   }
 501 }
 502
 503 void MediaDecoderJob::InitializeReceivedData() {
 504   for (size_t i = 0; i < 2; ++i) {
 505     received_data_[i] = DemuxerData();
 506     access_unit_index_[i] = 0;
 507   }
 508 }
 509
 510 }  // namespace media