media/renderers/renderer_impl.h

   1 // Copyright 2014 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #ifndef MEDIA_RENDERERS_RENDERER_IMPL_H_
   6 #define MEDIA_RENDERERS_RENDERER_IMPL_H_
   7
   8 #include <memory>
   9 #include <vector>
  10
  11 #include "base/cancelable_callback.h"
  12 #include "base/memory/raw_ptr.h"
  13 #include "base/memory/scoped_refptr.h"
  14 #include "base/memory/weak_ptr.h"
  15 #include "base/synchronization/lock.h"
  16 #include "base/task/sequenced_task_runner.h"
  17 #include "base/time/clock.h"
  18 #include "base/time/default_tick_clock.h"
  19 #include "base/time/time.h"
  20 #include "media/base/audio_decoder_config.h"
  21 #include "media/base/buffering_state.h"
  22 #include "media/base/decryptor.h"
  23 #include "media/base/demuxer_stream.h"
  24 #include "media/base/media_export.h"
  25 #include "media/base/pipeline_status.h"
  26 #include "media/base/renderer.h"
  27 #include "media/base/tuneable.h"
  28 #include "media/base/video_decoder_config.h"
  29 #include "media/base/waiting.h"
  30 #include "ui/gfx/geometry/size.h"
  31
  32 namespace media {
  33
  34 class AudioRenderer;
  35 class MediaResource;
  36 class TimeSource;
  37 class VideoRenderer;
  38 class WallClockTimeSource;
  39
  40 class MEDIA_EXPORT RendererImpl final : public Renderer {
  41  public:
  42   // Renders audio/video streams using |audio_renderer| and |video_renderer|
  43   // provided. All methods except for GetMediaTime() run on the |task_runner|.
  44   // GetMediaTime() runs on the render main thread because it's part of JS sync
  45   // API.
  46   RendererImpl(const scoped_refptr<base::SequencedTaskRunner>& task_runner,
  47                std::unique_ptr<AudioRenderer> audio_renderer,
  48                std::unique_ptr<VideoRenderer> video_renderer);
  49
  50   RendererImpl(const RendererImpl&) = delete;
  51   RendererImpl& operator=(const RendererImpl&) = delete;
  52
  53   ~RendererImpl() final;
  54
  55   // Renderer implementation.
  56   void Initialize(MediaResource* media_resource,
  57                   RendererClient* client,
  58                   PipelineStatusCallback init_cb) final;
  59   void SetCdm(CdmContext* cdm_context, CdmAttachedCB cdm_attached_cb) final;
  60   void SetLatencyHint(absl::optional<base::TimeDelta> latency_hint) final;
  61   void SetPreservesPitch(bool preserves_pitch) final;
  62   void SetWasPlayedWithUserActivation(
  63       bool was_played_with_user_activation) final;
  64   void Flush(base::OnceClosure flush_cb) final;
  65 #if defined(TIZEN_MULTIMEDIA)
  66   void Seek(base::TimeDelta time, base::OnceClosure seek_cb) final {}
  67 #endif
  68   void StartPlayingFrom(base::TimeDelta time) final;
  69   void SetPlaybackRate(double playback_rate) final;
  70   void SetVolume(float volume) final;
  71   base::TimeDelta GetMediaTime() final;
  72   void OnSelectedVideoTracksChanged(
  73       const std::vector<DemuxerStream*>& enabled_tracks,
  74       base::OnceClosure change_completed_cb) final;
  75   void OnEnabledAudioTracksChanged(
  76       const std::vector<DemuxerStream*>& enabled_tracks,
  77       base::OnceClosure change_completed_cb) final;
  78   RendererType GetRendererType() final;
  79
  80   // Helper functions for testing purposes. Must be called before Initialize().
  81   void DisableUnderflowForTesting();
  82   void EnableClocklessVideoPlaybackForTesting();
  83   void set_time_source_for_testing(TimeSource* time_source) {
  84     time_source_ = time_source;
  85   }
  86   void set_video_underflow_threshold_for_testing(base::TimeDelta threshold) {
  87     video_underflow_threshold_.set_for_testing(threshold);
  88   }
  89
  90  private:
  91   class RendererClientInternal;
  92
  93   enum State {
  94     STATE_UNINITIALIZED,
  95     STATE_INIT_PENDING_CDM,  // Initialization is waiting for the CDM to be set.
  96     STATE_INITIALIZING,      // Initializing audio/video renderers.
  97     STATE_FLUSHING,          // Flushing is in progress.
  98     STATE_FLUSHED,           // After initialization or after flush completed.
  99     STATE_PLAYING,           // After StartPlayingFrom has been called.
 100     STATE_ERROR
 101   };
 102
 103   bool GetWallClockTimes(const std::vector<base::TimeDelta>& media_timestamps,
 104                          std::vector<base::TimeTicks>* wall_clock_times);
 105
 106   bool HasEncryptedStream();
 107
 108   void FinishInitialization(PipelineStatus status);
 109   void FinishFlush();
 110
 111   // Helper functions and callbacks for Initialize().
 112   void InitializeAudioRenderer();
 113   void OnAudioRendererInitializeDone(PipelineStatus status);
 114   void InitializeVideoRenderer();
 115   void OnVideoRendererInitializeDone(PipelineStatus status);
 116
 117   // Helper functions and callbacks for Flush().
 118   void FlushInternal();
 119   void FlushAudioRenderer();
 120   void OnAudioRendererFlushDone();
 121   void FlushVideoRenderer();
 122   void OnVideoRendererFlushDone();
 123
 124   // Reinitialize audio/video renderer during a demuxer stream switching. The
 125   // renderer must be flushed first, and when the re-init is completed the
 126   // corresponding callback will be invoked to restart playback.
 127   // The |stream| parameter specifies the new demuxer stream, and the |time|
 128   // parameter specifies the time on media timeline where the switch occured.
 129   void ReinitializeAudioRenderer(DemuxerStream* stream,
 130                                  base::TimeDelta time,
 131                                  base::OnceClosure reinitialize_completed_cb);
 132   void OnAudioRendererReinitialized(DemuxerStream* stream,
 133                                     base::TimeDelta time,
 134                                     base::OnceClosure reinitialize_completed_cb,
 135                                     PipelineStatus status);
 136   void ReinitializeVideoRenderer(DemuxerStream* stream,
 137                                  base::TimeDelta time,
 138                                  base::OnceClosure restart_completed_cb);
 139   void OnVideoRendererReinitialized(DemuxerStream* stream,
 140                                     base::TimeDelta time,
 141                                     base::OnceClosure restart_completed_cb,
 142                                     PipelineStatus status);
 143
 144   // Restart audio/video renderer playback after a demuxer stream switch or
 145   // after a demuxer stream has been disabled and re-enabled. The |stream|
 146   // parameter specifies which stream needs to be restarted. The |time|
 147   // parameter specifies the position on the media timeline where the playback
 148   // needs to be restarted. It is necessary for demuxers with independent
 149   // streams (e.g. MSE / ChunkDemuxer) to synchronize data reading between those
 150   // streams.
 151   void RestartAudioRenderer(DemuxerStream* stream,
 152                             base::TimeDelta time,
 153                             base::OnceClosure restart_completed_cb);
 154   void RestartVideoRenderer(DemuxerStream* stream,
 155                             base::TimeDelta time,
 156                             base::OnceClosure restart_completed_cb);
 157
 158   // Fix state booleans after the stream switching is finished.
 159   void CleanUpTrackChange(base::OnceClosure on_finished,
 160                           bool* ended,
 161                           bool* playing);
 162
 163   // Callback executed by filters to update statistics.
 164   void OnStatisticsUpdate(const PipelineStatistics& stats);
 165
 166   // Collection of callback methods and helpers for tracking changes in
 167   // buffering state and transition from paused/underflow states and playing
 168   // states.
 169   //
 170   // While in the kPlaying state:
 171   //   - A waiting to non-waiting transition indicates preroll has completed
 172   //     and StartPlayback() should be called
 173   //   - A non-waiting to waiting transition indicates underflow has occurred
 174   //     and PausePlayback() should be called
 175   void OnBufferingStateChange(DemuxerStream::Type type,
 176                               BufferingState new_buffering_state,
 177                               BufferingStateChangeReason reason);
 178
 179   // Handles the buffering notifications that we might get while an audio or a
 180   // video stream is being restarted. In those cases we don't want to report
 181   // underflows immediately and instead give decoders a chance to catch up with
 182   // currently playing stream. Returns true if the buffering nofication has been
 183   // handled and no further processing is necessary, returns false to indicate
 184   // that we should fall back to the regular OnBufferingStateChange logic.
 185   bool HandleRestartedStreamBufferingChanges(
 186       DemuxerStream::Type type,
 187       BufferingState new_buffering_state);
 188
 189   bool WaitingForEnoughData() const;
 190   void PausePlayback();
 191   void StartPlayback();
 192
 193   // Callbacks executed when a renderer has ended.
 194   void OnRendererEnded(DemuxerStream::Type type);
 195   bool PlaybackHasEnded() const;
 196   void RunEndedCallbackIfNeeded();
 197
 198   // Callback executed when a runtime error happens.
 199   void OnError(PipelineStatus error);
 200
 201   // Callback executed when there is a fallback somewhere in the pipeline which
 202   // should be recorded for metrics analysis.
 203   void OnFallback(PipelineStatus fallback);
 204
 205   void OnWaiting(WaitingReason reason);
 206   void OnVideoNaturalSizeChange(const gfx::Size& size);
 207   void OnAudioConfigChange(const AudioDecoderConfig& config);
 208   void OnVideoConfigChange(const VideoDecoderConfig& config);
 209   void OnVideoOpacityChange(bool opaque);
 210   void OnVideoFrameRateChange(absl::optional<int> fps);
 211
 212   void OnStreamRestartCompleted();
 213
 214   State state_;
 215
 216   // Task runner used to execute pipeline tasks.
 217   scoped_refptr<base::SequencedTaskRunner> task_runner_;
 218
 219   raw_ptr<MediaResource> media_resource_;
 220   raw_ptr<RendererClient> client_;
 221
 222   // Temporary callback used for Initialize() and Flush().
 223   PipelineStatusCallback init_cb_;
 224   base::OnceClosure flush_cb_;
 225
 226   std::unique_ptr<RendererClientInternal> audio_renderer_client_;
 227   std::unique_ptr<RendererClientInternal> video_renderer_client_;
 228   std::unique_ptr<AudioRenderer> audio_renderer_;
 229   std::unique_ptr<VideoRenderer> video_renderer_;
 230
 231   raw_ptr<DemuxerStream> current_audio_stream_;
 232   raw_ptr<DemuxerStream> current_video_stream_;
 233
 234   // Renderer-provided time source used to control playback.
 235   raw_ptr<TimeSource, DanglingUntriaged> time_source_;
 236   std::unique_ptr<WallClockTimeSource> wall_clock_time_source_;
 237   bool time_ticking_;
 238   double playback_rate_;
 239
 240   // The time to start playback from after starting/seeking has completed.
 241   base::TimeDelta start_time_;
 242
 243   BufferingState audio_buffering_state_;
 244   BufferingState video_buffering_state_;
 245
 246   // Whether we've received the audio/video ended events.
 247   bool audio_ended_;
 248   bool video_ended_;
 249   bool audio_playing_;
 250   bool video_playing_;
 251
 252   raw_ptr<CdmContext> cdm_context_;
 253
 254   bool underflow_disabled_for_testing_;
 255   bool clockless_video_playback_enabled_for_testing_;
 256
 257   // Used to defer underflow for video when audio is present.
 258   base::CancelableOnceClosure deferred_video_underflow_cb_;
 259
 260   // We cannot use `!deferred_video_underflow_cb_.IsCancelled()` as that changes
 261   // when the callback is run, even if not explicitly cancelled.
 262   bool has_deferred_buffering_state_change_ = false;
 263
 264   // The amount of time to wait before declaring underflow if the video renderer
 265   // runs out of data but the audio renderer still has enough.
 266   Tuneable<base::TimeDelta> video_underflow_threshold_ = {
 267       "MediaVideoUnderflowThreshold", base::Milliseconds(1000),
 268       base::Milliseconds(3000), base::Milliseconds(8000)};
 269
 270   // Lock used to protect access to the |restarting_audio_| flag and
 271   // |restarting_audio_time_|.
 272   // TODO(servolk): Get rid of the lock and replace restarting_audio_ with
 273   // std::atomic<bool> when atomics are unbanned in Chromium.
 274   base::Lock restarting_audio_lock_;
 275   bool pending_audio_track_change_ = false;
 276   base::TimeDelta restarting_audio_time_ = kNoTimestamp;
 277
 278   bool pending_video_track_change_ = false;
 279
 280   base::WeakPtr<RendererImpl> weak_this_;
 281   base::WeakPtrFactory<RendererImpl> weak_factory_{this};
 282 };
 283
 284 }  // namespace media
 285
 286 #endif  // MEDIA_RENDERERS_RENDERER_IMPL_H_