src/media/cast/test/fake_media_source.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/test/fake_media_source.h"
   6
   7 #include "base/files/memory_mapped_file.h"
   8 #include "base/files/scoped_file.h"
   9 #include "base/logging.h"
  10 #include "base/strings/string_number_conversions.h"
  11 #include "media/audio/audio_parameters.h"
  12 #include "media/base/audio_buffer.h"
  13 #include "media/base/audio_bus.h"
  14 #include "media/base/audio_fifo.h"
  15 #include "media/base/audio_timestamp_helper.h"
  16 #include "media/base/media.h"
  17 #include "media/base/multi_channel_resampler.h"
  18 #include "media/base/video_frame.h"
  19 #include "media/base/video_util.h"
  20 #include "media/cast/cast_sender.h"
  21 #include "media/cast/test/utility/audio_utility.h"
  22 #include "media/cast/test/utility/video_utility.h"
  23 #include "media/ffmpeg/ffmpeg_common.h"
  24 #include "media/ffmpeg/ffmpeg_deleters.h"
  25 #include "media/filters/audio_renderer_algorithm.h"
  26 #include "media/filters/ffmpeg_demuxer.h"
  27 #include "media/filters/ffmpeg_glue.h"
  28 #include "media/filters/in_memory_url_protocol.h"
  29 #include "ui/gfx/size.h"
  30
  31 namespace {
  32
  33 static const int kAudioChannels = 2;
  34 static const int kAudioSamplingFrequency = 48000;
  35 static const int kSoundFrequency = 1234;  // Frequency of sinusoid wave.
  36 static const float kSoundVolume = 0.5f;
  37 static const int kAudioFrameMs = 10;  // Each audio frame is exactly 10ms.
  38 static const int kAudioPacketsPerSecond = 1000 / kAudioFrameMs;
  39
  40 void AVFreeFrame(AVFrame* frame) {
  41   av_frame_free(&frame);
  42 }
  43
  44 }  // namespace
  45
  46 namespace media {
  47 namespace cast {
  48
  49 FakeMediaSource::FakeMediaSource(
  50     scoped_refptr<base::SingleThreadTaskRunner> task_runner,
  51     base::TickClock* clock,
  52     const VideoSenderConfig& video_config)
  53     : task_runner_(task_runner),
  54       video_config_(video_config),
  55       synthetic_count_(0),
  56       clock_(clock),
  57       audio_frame_count_(0),
  58       video_frame_count_(0),
  59       weak_factory_(this),
  60       av_format_context_(NULL),
  61       audio_stream_index_(-1),
  62       playback_rate_(1.0),
  63       video_stream_index_(-1),
  64       video_frame_rate_numerator_(video_config.max_frame_rate),
  65       video_frame_rate_denominator_(1),
  66       video_first_pts_(0),
  67       video_first_pts_set_(false) {
  68   audio_bus_factory_.reset(new TestAudioBusFactory(kAudioChannels,
  69                                                    kAudioSamplingFrequency,
  70                                                    kSoundFrequency,
  71                                                    kSoundVolume));
  72 }
  73
  74 FakeMediaSource::~FakeMediaSource() {
  75 }
  76
  77 void FakeMediaSource::SetSourceFile(const base::FilePath& video_file,
  78                                     int override_fps) {
  79   DCHECK(!video_file.empty());
  80
  81   if (override_fps) {
  82     video_config_.max_frame_rate = override_fps;
  83     video_frame_rate_numerator_ = override_fps;
  84   }
  85
  86   LOG(INFO) << "Source: " << video_file.value();
  87   if (!file_data_.Initialize(video_file)) {
  88     LOG(ERROR) << "Cannot load file.";
  89     return;
  90   }
  91   protocol_.reset(
  92       new InMemoryUrlProtocol(file_data_.data(), file_data_.length(), false));
  93   glue_.reset(new FFmpegGlue(protocol_.get()));
  94
  95   if (!glue_->OpenContext()) {
  96     LOG(ERROR) << "Cannot open file.";
  97     return;
  98   }
  99
 100   // AVFormatContext is owned by the glue.
 101   av_format_context_ = glue_->format_context();
 102   if (avformat_find_stream_info(av_format_context_, NULL) < 0) {
 103     LOG(ERROR) << "Cannot find stream information.";
 104     return;
 105   }
 106
 107   // Prepare FFmpeg decoders.
 108   for (unsigned int i = 0; i < av_format_context_->nb_streams; ++i) {
 109     AVStream* av_stream = av_format_context_->streams[i];
 110     AVCodecContext* av_codec_context = av_stream->codec;
 111     AVCodec* av_codec = avcodec_find_decoder(av_codec_context->codec_id);
 112
 113     if (!av_codec) {
 114       LOG(ERROR) << "Cannot find decoder for the codec: "
 115                  << av_codec_context->codec_id;
 116       continue;
 117     }
 118
 119     // Number of threads for decoding.
 120     av_codec_context->thread_count = 2;
 121     av_codec_context->error_concealment = FF_EC_GUESS_MVS | FF_EC_DEBLOCK;
 122     av_codec_context->request_sample_fmt = AV_SAMPLE_FMT_S16;
 123
 124     if (avcodec_open2(av_codec_context, av_codec, NULL) < 0) {
 125       LOG(ERROR) << "Cannot open AVCodecContext for the codec: "
 126                  << av_codec_context->codec_id;
 127       return;
 128     }
 129
 130     if (av_codec->type == AVMEDIA_TYPE_AUDIO) {
 131       if (av_codec_context->sample_fmt == AV_SAMPLE_FMT_S16P) {
 132         LOG(ERROR) << "Audio format not supported.";
 133         continue;
 134       }
 135       ChannelLayout layout = ChannelLayoutToChromeChannelLayout(
 136           av_codec_context->channel_layout,
 137           av_codec_context->channels);
 138       if (layout == CHANNEL_LAYOUT_UNSUPPORTED) {
 139         LOG(ERROR) << "Unsupported audio channels layout.";
 140         continue;
 141       }
 142       if (audio_stream_index_ != -1) {
 143         LOG(WARNING) << "Found multiple audio streams.";
 144       }
 145       audio_stream_index_ = static_cast<int>(i);
 146       audio_params_.Reset(
 147           AudioParameters::AUDIO_PCM_LINEAR,
 148           layout,
 149           av_codec_context->channels,
 150           av_codec_context->sample_rate,
 151           8 * av_get_bytes_per_sample(av_codec_context->sample_fmt),
 152           av_codec_context->sample_rate / kAudioPacketsPerSecond);
 153       LOG(INFO) << "Source file has audio.";
 154     } else if (av_codec->type == AVMEDIA_TYPE_VIDEO) {
 155       VideoFrame::Format format =
 156           PixelFormatToVideoFormat(av_codec_context->pix_fmt);
 157       if (format != VideoFrame::YV12) {
 158         LOG(ERROR) << "Cannot handle non YV12 video format: " << format;
 159         continue;
 160       }
 161       if (video_stream_index_ != -1) {
 162         LOG(WARNING) << "Found multiple video streams.";
 163       }
 164       video_stream_index_ = static_cast<int>(i);
 165       if (!override_fps) {
 166         video_frame_rate_numerator_ = av_stream->r_frame_rate.num;
 167         video_frame_rate_denominator_ = av_stream->r_frame_rate.den;
 168         // Max frame rate is rounded up.
 169         video_config_.max_frame_rate =
 170             video_frame_rate_denominator_ +
 171             video_frame_rate_numerator_ - 1;
 172         video_config_.max_frame_rate /= video_frame_rate_denominator_;
 173       } else {
 174         // If video is played at a manual speed audio needs to match.
 175         playback_rate_ = 1.0 * override_fps *
 176             av_stream->r_frame_rate.den /  av_stream->r_frame_rate.num;
 177       }
 178       LOG(INFO) << "Source file has video.";
 179     } else {
 180       LOG(ERROR) << "Unknown stream type; ignore.";
 181     }
 182   }
 183
 184   Rewind();
 185 }
 186
 187 void FakeMediaSource::Start(scoped_refptr<AudioFrameInput> audio_frame_input,
 188                             scoped_refptr<VideoFrameInput> video_frame_input) {
 189   audio_frame_input_ = audio_frame_input;
 190   video_frame_input_ = video_frame_input;
 191
 192   LOG(INFO) << "Max Frame rate: " << video_config_.max_frame_rate;
 193   LOG(INFO) << "Real Frame rate: "
 194             << video_frame_rate_numerator_ << "/"
 195             << video_frame_rate_denominator_ << " fps.";
 196   LOG(INFO) << "Audio playback rate: " << playback_rate_;
 197
 198   if (!is_transcoding_audio() && !is_transcoding_video()) {
 199     // Send fake patterns.
 200     task_runner_->PostTask(
 201         FROM_HERE,
 202         base::Bind(
 203             &FakeMediaSource::SendNextFakeFrame,
 204             base::Unretained(this)));
 205     return;
 206   }
 207
 208   // Send transcoding streams.
 209   audio_algo_.Initialize(audio_params_);
 210   audio_algo_.FlushBuffers();
 211   audio_fifo_input_bus_ =
 212       AudioBus::Create(
 213           audio_params_.channels(), audio_params_.frames_per_buffer());
 214   // Audio FIFO can carry all data fron AudioRendererAlgorithm.
 215   audio_fifo_.reset(
 216       new AudioFifo(audio_params_.channels(),
 217                     audio_algo_.QueueCapacity()));
 218   audio_resampler_.reset(new media::MultiChannelResampler(
 219       audio_params_.channels(),
 220       static_cast<double>(audio_params_.sample_rate()) /
 221       kAudioSamplingFrequency,
 222       audio_params_.frames_per_buffer(),
 223       base::Bind(&FakeMediaSource::ProvideData, base::Unretained(this))));
 224   task_runner_->PostTask(
 225       FROM_HERE,
 226       base::Bind(
 227           &FakeMediaSource::SendNextFrame,
 228           base::Unretained(this)));
 229 }
 230
 231 void FakeMediaSource::SendNextFakeFrame() {
 232   gfx::Size size(video_config_.width, video_config_.height);
 233   scoped_refptr<VideoFrame> video_frame =
 234       VideoFrame::CreateBlackFrame(size);
 235   PopulateVideoFrame(video_frame.get(), synthetic_count_);
 236   ++synthetic_count_;
 237
 238   base::TimeTicks now = clock_->NowTicks();
 239   if (start_time_.is_null())
 240     start_time_ = now;
 241
 242   base::TimeDelta video_time = VideoFrameTime(++video_frame_count_);
 243   video_frame->set_timestamp(video_time);
 244   video_frame_input_->InsertRawVideoFrame(video_frame,
 245                                           start_time_ + video_time);
 246
 247   // Send just enough audio data to match next video frame's time.
 248   base::TimeDelta audio_time = AudioFrameTime(audio_frame_count_);
 249   while (audio_time < video_time) {
 250     if (is_transcoding_audio()) {
 251       Decode(true);
 252       CHECK(!audio_bus_queue_.empty()) << "No audio decoded.";
 253       scoped_ptr<AudioBus> bus(audio_bus_queue_.front());
 254       audio_bus_queue_.pop();
 255       audio_frame_input_->InsertAudio(
 256           bus.Pass(), start_time_ + audio_time);
 257     } else {
 258       audio_frame_input_->InsertAudio(
 259           audio_bus_factory_->NextAudioBus(
 260               base::TimeDelta::FromMilliseconds(kAudioFrameMs)),
 261           start_time_ + audio_time);
 262     }
 263     audio_time = AudioFrameTime(++audio_frame_count_);
 264   }
 265
 266   // This is the time since the stream started.
 267   const base::TimeDelta elapsed_time = now - start_time_;
 268
 269   // Handle the case when frame generation cannot keep up.
 270   // Move the time ahead to match the next frame.
 271   while (video_time < elapsed_time) {
 272     LOG(WARNING) << "Skipping one frame.";
 273     video_time = VideoFrameTime(++video_frame_count_);
 274   }
 275
 276   task_runner_->PostDelayedTask(
 277       FROM_HERE,
 278       base::Bind(&FakeMediaSource::SendNextFakeFrame,
 279                  weak_factory_.GetWeakPtr()),
 280       video_time - elapsed_time);
 281 }
 282
 283 bool FakeMediaSource::SendNextTranscodedVideo(base::TimeDelta elapsed_time) {
 284   if (!is_transcoding_video())
 285     return false;
 286
 287   Decode(false);
 288   if (video_frame_queue_.empty())
 289     return false;
 290
 291   scoped_refptr<VideoFrame> decoded_frame =
 292       video_frame_queue_.front();
 293   if (elapsed_time < decoded_frame->timestamp())
 294     return false;
 295
 296   gfx::Size size(video_config_.width, video_config_.height);
 297   scoped_refptr<VideoFrame> video_frame =
 298       VideoFrame::CreateBlackFrame(size);
 299   video_frame_queue_.pop();
 300   media::CopyPlane(VideoFrame::kYPlane,
 301                    decoded_frame->data(VideoFrame::kYPlane),
 302                    decoded_frame->stride(VideoFrame::kYPlane),
 303                    decoded_frame->rows(VideoFrame::kYPlane),
 304                    video_frame.get());
 305   media::CopyPlane(VideoFrame::kUPlane,
 306                    decoded_frame->data(VideoFrame::kUPlane),
 307                    decoded_frame->stride(VideoFrame::kUPlane),
 308                    decoded_frame->rows(VideoFrame::kUPlane),
 309                    video_frame.get());
 310   media::CopyPlane(VideoFrame::kVPlane,
 311                    decoded_frame->data(VideoFrame::kVPlane),
 312                    decoded_frame->stride(VideoFrame::kVPlane),
 313                    decoded_frame->rows(VideoFrame::kVPlane),
 314                    video_frame.get());
 315
 316   base::TimeDelta video_time;
 317   // Use the timestamp from the file if we're transcoding.
 318   video_time = ScaleTimestamp(decoded_frame->timestamp());
 319   video_frame_input_->InsertRawVideoFrame(
 320       video_frame, start_time_ + video_time);
 321
 322   // Make sure queue is not empty.
 323   Decode(false);
 324   return true;
 325 }
 326
 327 bool FakeMediaSource::SendNextTranscodedAudio(base::TimeDelta elapsed_time) {
 328   if (!is_transcoding_audio())
 329     return false;
 330
 331   Decode(true);
 332   if (audio_bus_queue_.empty())
 333     return false;
 334
 335   base::TimeDelta audio_time = audio_sent_ts_->GetTimestamp();
 336   if (elapsed_time < audio_time)
 337     return false;
 338   scoped_ptr<AudioBus> bus(audio_bus_queue_.front());
 339   audio_bus_queue_.pop();
 340   audio_sent_ts_->AddFrames(bus->frames());
 341   audio_frame_input_->InsertAudio(
 342       bus.Pass(), start_time_ + audio_time);
 343
 344   // Make sure queue is not empty.
 345   Decode(true);
 346   return true;
 347 }
 348
 349 void FakeMediaSource::SendNextFrame() {
 350   if (start_time_.is_null())
 351     start_time_ = clock_->NowTicks();
 352   if (start_time_.is_null())
 353     start_time_ = clock_->NowTicks();
 354
 355   // Send as much as possible. Audio is sent according to
 356   // system time.
 357   while (SendNextTranscodedAudio(clock_->NowTicks() - start_time_));
 358
 359   // Video is sync'ed to audio.
 360   while (SendNextTranscodedVideo(audio_sent_ts_->GetTimestamp()));
 361
 362   if (audio_bus_queue_.empty() && video_frame_queue_.empty()) {
 363     // Both queues are empty can only mean that we have reached
 364     // the end of the stream.
 365     LOG(INFO) << "Rewind.";
 366     Rewind();
 367     start_time_ = base::TimeTicks();
 368     audio_sent_ts_.reset();
 369     video_first_pts_set_ = false;
 370   }
 371
 372   // Send next send.
 373   task_runner_->PostDelayedTask(
 374       FROM_HERE,
 375       base::Bind(
 376           &FakeMediaSource::SendNextFrame,
 377           base::Unretained(this)),
 378       base::TimeDelta::FromMilliseconds(kAudioFrameMs));
 379 }
 380
 381 base::TimeDelta FakeMediaSource::VideoFrameTime(int frame_number) {
 382   return frame_number * base::TimeDelta::FromSeconds(1) *
 383       video_frame_rate_denominator_ / video_frame_rate_numerator_;
 384 }
 385
 386 base::TimeDelta FakeMediaSource::ScaleTimestamp(base::TimeDelta timestamp) {
 387   return base::TimeDelta::FromMicroseconds(
 388       timestamp.InMicroseconds() / playback_rate_);
 389 }
 390
 391 base::TimeDelta FakeMediaSource::AudioFrameTime(int frame_number) {
 392   return frame_number * base::TimeDelta::FromMilliseconds(kAudioFrameMs);
 393 }
 394
 395 void FakeMediaSource::Rewind() {
 396   CHECK(av_seek_frame(av_format_context_, -1, 0, AVSEEK_FLAG_BACKWARD) >= 0)
 397       << "Failed to rewind to the beginning.";
 398 }
 399
 400 ScopedAVPacket FakeMediaSource::DemuxOnePacket(bool* audio) {
 401   ScopedAVPacket packet(new AVPacket());
 402   if (av_read_frame(av_format_context_, packet.get()) < 0) {
 403     LOG(ERROR) << "Failed to read one AVPacket.";
 404     packet.reset();
 405     return packet.Pass();
 406   }
 407
 408   int stream_index = static_cast<int>(packet->stream_index);
 409   if (stream_index == audio_stream_index_) {
 410     *audio = true;
 411   } else if (stream_index == video_stream_index_) {
 412     *audio = false;
 413   } else {
 414     // Ignore unknown packet.
 415     LOG(INFO) << "Unknown packet.";
 416     packet.reset();
 417   }
 418   return packet.Pass();
 419 }
 420
 421 void FakeMediaSource::DecodeAudio(ScopedAVPacket packet) {
 422   // Audio.
 423   AVFrame* avframe = av_frame_alloc();
 424
 425   // Make a shallow copy of packet so we can slide packet.data as frames are
 426   // decoded from the packet; otherwise av_free_packet() will corrupt memory.
 427   AVPacket packet_temp = *packet.get();
 428
 429   do {
 430     int frame_decoded = 0;
 431     int result = avcodec_decode_audio4(
 432         av_audio_context(), avframe, &frame_decoded, &packet_temp);
 433     CHECK(result >= 0) << "Failed to decode audio.";
 434     packet_temp.size -= result;
 435     packet_temp.data += result;
 436     if (!frame_decoded)
 437       continue;
 438
 439     int frames_read = avframe->nb_samples;
 440     if (frames_read < 0)
 441       break;
 442
 443     if (!audio_sent_ts_) {
 444       // Initialize the base time to the first packet in the file.
 445       // This is set to the frequency we send to the receiver.
 446       // Not the frequency of the source file. This is because we
 447       // increment the frame count by samples we sent.
 448       audio_sent_ts_.reset(
 449           new AudioTimestampHelper(kAudioSamplingFrequency));
 450       // For some files this is an invalid value.
 451       base::TimeDelta base_ts;
 452       audio_sent_ts_->SetBaseTimestamp(base_ts);
 453     }
 454
 455     scoped_refptr<AudioBuffer> buffer =
 456         AudioBuffer::CopyFrom(
 457             AVSampleFormatToSampleFormat(
 458                 av_audio_context()->sample_fmt),
 459             ChannelLayoutToChromeChannelLayout(
 460                 av_audio_context()->channel_layout,
 461                 av_audio_context()->channels),
 462             av_audio_context()->channels,
 463             av_audio_context()->sample_rate,
 464             frames_read,
 465             &avframe->data[0],
 466             // Note: Not all files have correct values for pkt_pts.
 467             base::TimeDelta::FromMilliseconds(avframe->pkt_pts));
 468     audio_algo_.EnqueueBuffer(buffer);
 469     av_frame_unref(avframe);
 470   } while (packet_temp.size > 0);
 471   av_frame_free(&avframe);
 472
 473   const int frames_needed_to_scale =
 474       playback_rate_ * av_audio_context()->sample_rate /
 475       kAudioPacketsPerSecond;
 476   while (frames_needed_to_scale <= audio_algo_.frames_buffered()) {
 477     if (!audio_algo_.FillBuffer(audio_fifo_input_bus_.get(),
 478                                 audio_fifo_input_bus_->frames(),
 479                                 playback_rate_)) {
 480       // Nothing can be scaled. Decode some more.
 481       return;
 482     }
 483
 484     // Prevent overflow of audio data in the FIFO.
 485     if (audio_fifo_input_bus_->frames() + audio_fifo_->frames()
 486         <= audio_fifo_->max_frames()) {
 487       audio_fifo_->Push(audio_fifo_input_bus_.get());
 488     } else {
 489       LOG(WARNING) << "Audio FIFO full; dropping samples.";
 490     }
 491
 492     // Make sure there's enough data to resample audio.
 493     if (audio_fifo_->frames() <
 494         2 * audio_params_.sample_rate() / kAudioPacketsPerSecond) {
 495       continue;
 496     }
 497
 498     scoped_ptr<media::AudioBus> resampled_bus(
 499         media::AudioBus::Create(
 500             audio_params_.channels(),
 501             kAudioSamplingFrequency / kAudioPacketsPerSecond));
 502     audio_resampler_->Resample(resampled_bus->frames(),
 503                                resampled_bus.get());
 504     audio_bus_queue_.push(resampled_bus.release());
 505   }
 506 }
 507
 508 void FakeMediaSource::DecodeVideo(ScopedAVPacket packet) {
 509   // Video.
 510   int got_picture;
 511   AVFrame* avframe = av_frame_alloc();
 512   // Tell the decoder to reorder for us.
 513   avframe->reordered_opaque =
 514       av_video_context()->reordered_opaque = packet->pts;
 515   CHECK(avcodec_decode_video2(
 516       av_video_context(), avframe, &got_picture, packet.get()) >= 0)
 517       << "Video decode error.";
 518   if (!got_picture) {
 519     av_frame_free(&avframe);
 520     return;
 521   }
 522   gfx::Size size(av_video_context()->width, av_video_context()->height);
 523   if (!video_first_pts_set_ ||
 524       avframe->reordered_opaque < video_first_pts_) {
 525     video_first_pts_set_ = true;
 526     video_first_pts_ = avframe->reordered_opaque;
 527   }
 528   int64 pts = avframe->reordered_opaque - video_first_pts_;
 529   video_frame_queue_.push(
 530       VideoFrame::WrapExternalYuvData(
 531           media::VideoFrame::YV12,
 532           size,
 533           gfx::Rect(size),
 534           size,
 535           avframe->linesize[0],
 536           avframe->linesize[1],
 537           avframe->linesize[2],
 538           avframe->data[0],
 539           avframe->data[1],
 540           avframe->data[2],
 541           base::TimeDelta::FromMilliseconds(pts),
 542           base::Bind(&AVFreeFrame, avframe)));
 543 }
 544
 545 void FakeMediaSource::Decode(bool decode_audio) {
 546   // Read the stream until one video frame can be decoded.
 547   while (true) {
 548     if (decode_audio && !audio_bus_queue_.empty())
 549       return;
 550     if (!decode_audio && !video_frame_queue_.empty())
 551       return;
 552
 553     bool audio_packet = false;
 554     ScopedAVPacket packet = DemuxOnePacket(&audio_packet);
 555     if (!packet) {
 556       LOG(INFO) << "End of stream.";
 557       return;
 558     }
 559
 560     if (audio_packet)
 561       DecodeAudio(packet.Pass());
 562     else
 563       DecodeVideo(packet.Pass());
 564   }
 565 }
 566
 567 void FakeMediaSource::ProvideData(int frame_delay,
 568                                   media::AudioBus* output_bus) {
 569   if (audio_fifo_->frames() >= output_bus->frames()) {
 570     audio_fifo_->Consume(output_bus, 0, output_bus->frames());
 571   } else {
 572     LOG(WARNING) << "Not enough audio data for resampling.";
 573     output_bus->Zero();
 574   }
 575 }
 576
 577 AVStream* FakeMediaSource::av_audio_stream() {
 578   return av_format_context_->streams[audio_stream_index_];
 579 }
 580
 581 AVStream* FakeMediaSource::av_video_stream() {
 582   return av_format_context_->streams[video_stream_index_];
 583 }
 584
 585 AVCodecContext* FakeMediaSource::av_audio_context() {
 586   return av_audio_stream()->codec;
 587 }
 588
 589 AVCodecContext* FakeMediaSource::av_video_context() {
 590   return av_video_stream()->codec;
 591 }
 592
 593 }  // namespace cast
 594 }  // namespace media