1 // Copyright 2013 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef THIRD_PARTY_BLINK_RENDERER_PLATFORM_MEDIA_WEB_MEDIA_PLAYER_IMPL_H_
6 #define THIRD_PARTY_BLINK_RENDERER_PLATFORM_MEDIA_WEB_MEDIA_PLAYER_IMPL_H_
14 #include "base/cancelable_callback.h"
15 #include "base/compiler_specific.h"
16 #include "base/memory/memory_pressure_listener.h"
17 #include "base/memory/ref_counted.h"
18 #include "base/memory/scoped_refptr.h"
19 #include "base/memory/weak_ptr.h"
20 #include "base/threading/thread.h"
21 #include "base/time/default_tick_clock.h"
22 #include "base/time/time.h"
23 #include "base/timer/elapsed_timer.h"
24 #include "base/timer/timer.h"
25 #include "build/build_config.h"
26 #include "media/base/cdm_config.h"
27 #include "media/base/encryption_scheme.h"
28 #include "media/base/media_observer.h"
29 #include "media/base/media_tracks.h"
30 #include "media/base/overlay_info.h"
31 #include "media/base/pipeline_impl.h"
32 #include "media/base/renderer_factory_selector.h"
33 #include "media/base/routing_token_callback.h"
34 #include "media/base/simple_watch_timer.h"
35 #include "media/base/text_track.h"
36 #include "media/mojo/mojom/media_metrics_provider.mojom.h"
37 #include "media/mojo/mojom/playback_events_recorder.mojom.h"
38 #include "media/renderers/paint_canvas_video_renderer.h"
39 #include "mojo/public/cpp/bindings/pending_remote.h"
40 #include "mojo/public/cpp/bindings/remote.h"
41 #include "services/media_session/public/cpp/media_position.h"
42 #include "third_party/abseil-cpp/absl/types/optional.h"
43 #include "third_party/blink/public/platform/media/video_frame_compositor.h"
44 #include "third_party/blink/public/platform/media/web_media_player_builder.h"
45 #include "third_party/blink/public/platform/media/webmediaplayer_delegate.h"
46 #include "third_party/blink/public/platform/web_audio_source_provider.h"
47 #include "third_party/blink/public/platform/web_content_decryption_module_result.h"
48 #include "third_party/blink/public/platform/web_media_player.h"
49 #include "third_party/blink/public/platform/web_surface_layer_bridge.h"
50 #include "third_party/blink/public/web/modules/media/webmediaplayer_util.h"
51 #include "third_party/blink/renderer/platform/media/learning_experiment_helper.h"
52 #include "third_party/blink/renderer/platform/media/multi_buffer_data_source.h"
53 #include "third_party/blink/renderer/platform/media/smoothness_helper.h"
54 #include "third_party/blink/renderer/platform/platform_export.h"
58 class SingleThreadTaskRunner;
71 class LearningTaskController;
80 class MemoryDumpProviderProxy;
81 class PipelineController;
82 class SwitchableAudioRendererSink;
86 class RasterContextProvider;
90 class BufferedDataSourceHostImpl;
91 class PowerStatusHelper;
92 class ThreadSafeBrowserInterfaceBrokerProxy;
94 class VideoDecodeStatsReporter;
95 class VideoFrameCompositor;
96 class WatchTimeReporter;
97 class WebAudioSourceProviderImpl;
98 class WebContentDecryptionModule;
100 class WebMediaPlayerClient;
101 class WebMediaPlayerEncryptedMediaClient;
103 // The canonical implementation of WebMediaPlayer that's backed by
104 // Pipeline. Handles normal resource loading, Media Source, and
106 class PLATFORM_EXPORT WebMediaPlayerImpl
107 : public WebMediaPlayer,
108 public WebMediaPlayerDelegate::Observer,
109 public media::Pipeline::Client,
110 public media::MediaObserverClient,
111 public WebSurfaceLayerBridgeObserver,
112 public SmoothnessHelper::Client {
114 // Constructs a WebMediaPlayer implementation using Chromium's media stack.
115 // |delegate| and |renderer_factory_selector| must not be null.
117 WebLocalFrame* frame,
118 WebMediaPlayerClient* client,
119 WebMediaPlayerEncryptedMediaClient* encrypted_client,
120 WebMediaPlayerDelegate* delegate,
121 std::unique_ptr<media::RendererFactorySelector> renderer_factory_selector,
123 std::unique_ptr<VideoFrameCompositor> compositor,
124 std::unique_ptr<media::MediaLog> media_log,
125 media::MediaPlayerLoggingID player_id,
126 WebMediaPlayerBuilder::DeferLoadCB defer_load_cb,
127 scoped_refptr<media::SwitchableAudioRendererSink> audio_renderer_sink,
128 scoped_refptr<base::SingleThreadTaskRunner> media_task_runner,
129 scoped_refptr<base::TaskRunner> worker_task_runner,
130 scoped_refptr<base::SingleThreadTaskRunner> compositor_task_runner,
131 scoped_refptr<base::SingleThreadTaskRunner>
132 video_frame_compositor_task_runner,
133 WebMediaPlayerBuilder::AdjustAllocatedMemoryCB adjust_allocated_memory_cb,
134 WebContentDecryptionModule* initial_cdm,
135 media::RequestRoutingTokenCallback request_routing_token_cb,
136 base::WeakPtr<media::MediaObserver> media_observer,
137 bool enable_instant_source_buffer_gc,
138 bool embedded_media_experience_enabled,
139 mojo::PendingRemote<media::mojom::MediaMetricsProvider> metrics_provider,
140 CreateSurfaceLayerBridgeCB create_bridge_callback,
141 scoped_refptr<viz::RasterContextProvider> raster_context_provider,
142 #if defined(TIZEN_VIDEO_HOLE)
145 bool use_surface_layer,
146 bool is_background_suspend_enabled,
147 bool is_background_video_play_enabled,
148 bool is_background_video_track_optimization_supported,
149 std::unique_ptr<media::Demuxer> demuxer_override,
150 scoped_refptr<ThreadSafeBrowserInterfaceBrokerProxy> remote_interfaces);
151 WebMediaPlayerImpl(const WebMediaPlayerImpl&) = delete;
152 WebMediaPlayerImpl& operator=(const WebMediaPlayerImpl&) = delete;
153 ~WebMediaPlayerImpl() override;
155 // WebSurfaceLayerBridgeObserver implementation.
156 void OnWebLayerUpdated() override;
157 void RegisterContentsLayer(cc::Layer* layer) override;
158 void UnregisterContentsLayer(cc::Layer* layer) override;
159 void OnSurfaceIdUpdated(viz::SurfaceId surface_id) override;
161 WebMediaPlayer::LoadTiming Load(LoadType load_type,
162 const WebMediaPlayerSource& source,
164 bool is_cache_disabled) override;
166 // Playback controls.
167 void Play() override;
168 void Pause() override;
169 void Seek(double seconds) override;
170 void SetRate(double rate) override;
171 void SetVolume(double volume) override;
172 void SetLatencyHint(double seconds) override;
173 void SetPreservesPitch(bool preserves_pitch) override;
174 void SetWasPlayedWithUserActivation(
175 bool was_played_with_user_activation) override;
176 void OnRequestPictureInPicture() override;
177 void OnTimeUpdate() override;
178 bool SetSinkId(const WebString& sink_id,
179 WebSetSinkIdCompleteCallback completion_callback) override;
180 void SetPoster(const WebURL& poster) override;
181 void SetPreload(WebMediaPlayer::Preload preload) override;
182 WebTimeRanges Buffered() const override;
183 WebTimeRanges Seekable() const override;
184 #if defined(TIZEN_MULTIMEDIA)
185 void Suspend() override;
186 void Resume() override;
189 void OnFrozen() override;
191 // paint() the current video frame into |canvas|. This is used to support
192 // various APIs and functionalities, including but not limited to: <canvas>,
193 // ImageBitmap, printing and capturing capabilities.
194 void Paint(cc::PaintCanvas* canvas,
195 const gfx::Rect& rect,
196 cc::PaintFlags& flags) override;
197 scoped_refptr<media::VideoFrame> GetCurrentFrameThenUpdate() override;
198 absl::optional<int> CurrentFrameId() const override;
199 media::PaintCanvasVideoRenderer* GetPaintCanvasVideoRenderer() override;
201 // True if the loaded media has a playable video/audio track.
202 bool HasVideo() const override;
203 bool HasAudio() const override;
205 void EnabledAudioTracksChanged(
206 const WebVector<WebMediaPlayer::TrackId>& enabledTrackIds) override;
207 void SelectedVideoTrackChanged(
208 WebMediaPlayer::TrackId* selectedTrackId) override;
210 // Dimensions of the video.
211 gfx::Size NaturalSize() const override;
213 gfx::Size VisibleSize() const override;
215 // Getters of playback state.
216 bool Paused() const override;
217 bool Seeking() const override;
218 double Duration() const override;
219 virtual double timelineOffset() const;
220 double CurrentTime() const override;
221 bool IsEnded() const override;
223 bool PausedWhenHidden() const override;
225 // Internal states of loading and network.
226 // TODO(hclam): Ask the pipeline about the state rather than having reading
227 // them from members which would cause race conditions.
228 WebMediaPlayer::NetworkState GetNetworkState() const override;
229 WebMediaPlayer::ReadyState GetReadyState() const override;
231 WebString GetErrorMessage() const override;
232 bool DidLoadingProgress() override;
233 bool WouldTaintOrigin() const override;
235 double MediaTimeForTimeValue(double timeValue) const override;
237 unsigned DecodedFrameCount() const override;
238 unsigned DroppedFrameCount() const override;
239 uint64_t AudioDecodedByteCount() const override;
240 uint64_t VideoDecodedByteCount() const override;
242 bool PassedTimingAllowOriginCheck() const override;
244 void SetVolumeMultiplier(double multiplier) override;
245 void SetPersistentState(bool persistent) override;
246 void SetPowerExperimentState(bool state) override;
247 void SuspendForFrameClosed() override;
249 bool HasAvailableVideoFrame() const override;
251 scoped_refptr<WebAudioSourceProviderImpl> GetAudioSourceProvider() override;
253 void SetContentDecryptionModule(
254 WebContentDecryptionModule* cdm,
255 WebContentDecryptionModuleResult result) override;
257 void EnteredFullscreen() override;
258 void ExitedFullscreen() override;
259 void BecameDominantVisibleContent(bool is_dominant) override;
260 void SetIsEffectivelyFullscreen(
261 WebFullscreenVideoStatus fullscreen_video_status) override;
262 void OnHasNativeControlsChanged(bool) override;
263 void OnDisplayTypeChanged(DisplayType display_type) override;
265 // WebMediaPlayerDelegate::Observer implementation.
266 void OnFrameHidden() override;
267 void OnFrameShown() override;
268 void OnIdleTimeout() override;
269 void RequestRemotePlaybackDisabled(bool disabled) override;
271 #if BUILDFLAG(IS_ANDROID)
272 // TODO(https://crbug.com/839651): Rename Flinging[Started/Stopped] to
273 // RemotePlayback[Started/Stopped] once the other RemotePlayback methods have
275 void FlingingStarted() override;
276 void FlingingStopped() override;
278 // Called when the play/pause state of media playing on a remote cast device
279 // changes, and WMPI wasn't the originator of that change (e.g. a phone on the
280 // same network paused the cast device via the casting notification).
281 // This is only used by the FlingingRenderer/FlingingRendererClient, when we
282 // are flinging media (a.k.a. RemotePlayback).
283 // The consistency between the WMPI state and the cast device state is not
284 // guaranteed, and it a best effort, which can always be fixed by the user by
285 // tapping play/pause once. Attempts to enfore stronger consistency guarantees
286 // have lead to unstable states, and a worse user experience.
287 void OnRemotePlayStateChange(media::MediaStatus::State state);
290 // media::MediaObserverClient implementation.
291 void SwitchToRemoteRenderer(
292 const std::string& remote_device_friendly_name) override;
293 void SwitchToLocalRenderer(
294 media::MediaObserverClient::ReasonToSwitchToLocal reason) override;
295 void UpdateRemotePlaybackCompatibility(bool is_compatible) override;
297 // Test helper methods for exercising media suspension. Once called, when
298 // |target_state| is reached or exceeded the stale flag will be set when
299 // computing the play state, which will trigger suspend if the player is
300 // paused; see UpdatePlayState_ComputePlayState() for the exact details.
301 void ForceStaleStateForTesting(ReadyState target_state) override;
302 bool IsSuspendedForTesting() override;
303 bool DidLazyLoad() const override;
304 void OnBecameVisible() override;
305 bool IsOpaque() const override;
306 int GetDelegateId() override;
307 absl::optional<viz::SurfaceId> GetSurfaceId() override;
308 GURL GetSrcAfterRedirects() override;
309 void RequestVideoFrameCallback() override;
310 std::unique_ptr<WebMediaPlayer::VideoFramePresentationMetadata>
311 GetVideoFramePresentationMetadata() override;
312 void UpdateFrameIfStale() override;
314 base::WeakPtr<WebMediaPlayer> AsWeakPtr() override;
315 void RegisterFrameSinkHierarchy() override;
316 void UnregisterFrameSinkHierarchy() override;
318 bool IsBackgroundMediaSuspendEnabled() const {
319 return is_background_suspend_enabled_;
322 // Distinct states that |delegate_| can be in. (Public for testing.)
323 enum class DelegateState {
329 // Playback state variables computed together in UpdatePlayState().
330 // (Public for testing.)
332 DelegateState delegate_state;
334 bool is_memory_reporting_enabled;
338 // Allow background video tracks with ~5 second keyframes (rounding down) to
339 // be disabled to save resources.
340 enum { kMaxKeyframeDistanceToDisableBackgroundVideoMs = 5500 };
343 friend class WebMediaPlayerImplTest;
344 friend class WebMediaPlayerImplBackgroundBehaviorTest;
346 void EnableOverlay();
347 void DisableOverlay();
349 // Do we have overlay information? For CVV, this is a surface id. For
350 // AndroidOverlay, this is the routing token.
351 bool HaveOverlayInfo();
353 // Send OverlayInfo to the decoder.
355 // If we've requested but not yet received the surface id or routing token, or
356 // if there's no decoder-provided callback to send the overlay info, then this
357 // call will do nothing.
358 void MaybeSendOverlayInfoToDecoder();
360 void OnPipelineSuspended();
361 void OnBeforePipelineResume();
362 void OnPipelineResumed();
363 void OnPipelineSeeked(bool time_updated);
364 void OnDemuxerOpened();
366 // media::Pipeline::Client overrides.
367 void OnError(media::PipelineStatus status) override;
368 void OnFallback(media::PipelineStatus status) override;
369 void OnEnded() override;
370 void OnMetadata(const media::PipelineMetadata& metadata) override;
371 void OnBufferingStateChange(
372 media::BufferingState state,
373 media::BufferingStateChangeReason reason) override;
374 void OnDurationChange() override;
375 void OnAddTextTrack(const media::TextTrackConfig& config,
376 media::AddTextTrackDoneCB done_cb) override;
377 void OnWaiting(media::WaitingReason reason) override;
378 void OnAudioConfigChange(const media::AudioDecoderConfig& config) override;
379 void OnVideoConfigChange(const media::VideoDecoderConfig& config) override;
380 void OnVideoNaturalSizeChange(const gfx::Size& size) override;
381 void OnVideoOpacityChange(bool opaque) override;
382 void OnVideoFrameRateChange(absl::optional<int> fps) override;
383 void OnVideoAverageKeyframeDistanceUpdate() override;
384 void OnAudioPipelineInfoChange(const media::AudioPipelineInfo& info) override;
385 void OnVideoPipelineInfoChange(const media::VideoPipelineInfo& info) override;
386 #if defined(TIZEN_MULTIMEDIA)
387 // Called if a player in the browser process is suspended.
388 void OnRequestSuspend() override;
391 // Simplified watch time reporting.
392 void OnSimpleWatchTimerTick();
394 // Actually seek. Avoids causing |should_notify_time_changed_| to be set when
395 // |time_updated| is false.
396 void DoSeek(base::TimeDelta time, bool time_updated);
398 // Called after |defer_load_cb_| has decided to allow the load. If
399 // |defer_load_cb_| is null this is called immediately.
400 void DoLoad(LoadType load_type,
403 bool is_cache_disabled);
405 // Called after asynchronous initialization of a data source completed.
406 void DataSourceInitialized(bool success);
408 // Called if the |MultiBufferDataSource| is redirected.
409 void OnDataSourceRedirected();
411 // Called when the data source is downloading or paused.
412 void NotifyDownloading(bool is_downloading);
414 // Called by RenderFrameImpl with the overlay routing token, if we request it.
415 void OnOverlayRoutingToken(const base::UnguessableToken& token);
417 // Called by GpuVideoDecoder on Android to request a surface to render to (if
419 void OnOverlayInfoRequested(
420 bool decoder_requires_restart_for_overlay,
421 media::ProvideOverlayInfoCB provide_overlay_info_cb);
423 // Creates a Renderer via the |renderer_factory_selector_|. If the
424 // |renderer_type| is absl::nullopt, create the base Renderer. Otherwise, set
425 // the base type to be |renderer_type| and create a Renderer of that type.
426 std::unique_ptr<media::Renderer> CreateRenderer(
427 absl::optional<media::RendererType> renderer_type);
429 // Finishes starting the pipeline due to a call to load().
430 void StartPipeline();
432 // Restart the player/pipeline as soon as possible. This will destroy the
433 // current renderer, if any, and create a new one via the RendererFactory; and
434 // then seek to resume playback at the current position.
435 void ScheduleRestart();
437 // Helpers that set the network/ready state and notifies the client if
439 void SetNetworkState(WebMediaPlayer::NetworkState state);
440 void SetReadyState(WebMediaPlayer::ReadyState state);
442 // Returns the current video frame from |compositor_|, and asks the compositor
443 // to update its frame if it is stale.
444 // Can return a nullptr.
445 scoped_refptr<media::VideoFrame> GetCurrentFrameFromCompositor() const;
447 // Called when the demuxer encounters encrypted streams.
448 void OnEncryptedMediaInitData(media::EmeInitDataType init_data_type,
449 const std::vector<uint8_t>& init_data);
451 // Called when the FFmpegDemuxer encounters new media tracks. This is only
452 // invoked when using FFmpegDemuxer, since MSE/ChunkDemuxer handle media
453 // tracks separately in WebSourceBufferImpl.
454 void OnFFmpegMediaTracksUpdated(std::unique_ptr<media::MediaTracks> tracks);
456 // Sets CdmContext from |cdm| on the pipeline and calls OnCdmAttached()
458 void SetCdmInternal(WebContentDecryptionModule* cdm);
460 // Called when a CDM has been attached to the |pipeline_|.
461 void OnCdmAttached(bool success);
463 // Inspects the current playback state and:
464 // - notifies |delegate_|,
465 // - toggles the memory usage reporting timer, and
466 // - toggles suspend/resume as necessary.
468 // This method should be called any time its dependent values change. These
472 // - delegate_->IsHidden(),
473 // - network_state_, ready_state_,
474 // - is_idle_, must_suspend_,
475 // - paused_, ended_,
476 // - pending_suspend_resume_cycle_,
477 // - enter_pip_callback_,
478 void UpdatePlayState();
480 // Methods internal to UpdatePlayState().
481 PlayState UpdatePlayState_ComputePlayState(bool is_flinging,
482 bool can_auto_suspend,
484 bool is_backgrounded,
485 bool is_in_picture_in_picture);
486 void SetDelegateState(DelegateState new_state, bool is_idle);
487 void SetMemoryReportingState(bool is_memory_reporting_enabled);
488 void SetSuspendState(bool is_suspended);
490 void SetDemuxer(std::unique_ptr<media::Demuxer> demuxer);
492 // Called at low frequency to tell external observers how much memory we're
493 // using for video playback. Called by |memory_usage_reporting_timer_|.
494 // Memory usage reporting is done in two steps, because |demuxer_| must be
495 // accessed on the media thread.
496 void ReportMemoryUsage();
497 void FinishMemoryUsageReport(int64_t demuxer_memory_usage);
499 void OnMainThreadMemoryDump(int32_t id,
500 const base::trace_event::MemoryDumpArgs& args,
501 base::trace_event::ProcessMemoryDump* pmd);
502 static void OnMediaThreadMemoryDump(
504 media::Demuxer* demuxer,
505 const base::trace_event::MemoryDumpArgs& args,
506 base::trace_event::ProcessMemoryDump* pmd);
508 void OnMemoryPressure(
509 base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level);
511 // Called during OnHidden() when we want a suspended player to enter the
512 // paused state after some idle timeout.
513 void ScheduleIdlePauseTimer();
515 // Returns |true| before HaveFutureData whenever there has been loading
516 // progress and we have not been resumed for at least kLoadingToIdleTimeout
519 // This is used to delay suspension long enough for preroll to complete, which
520 // is necessay because play() will not be called before HaveFutureData (and
521 // thus we think we are idle forever).
522 bool IsPrerollAttemptNeeded();
524 void CreateWatchTimeReporter();
525 void UpdateSecondaryProperties();
527 void CreateVideoDecodeStatsReporter();
529 // Returns true if the player is hidden.
530 bool IsHidden() const;
532 // Returns true if the player's source is streaming.
533 bool IsStreaming() const;
535 // Return whether |pipeline_metadata_| is compatible with an overlay. This
536 // is intended for android.
537 bool DoesOverlaySupportMetadata() const;
539 // Whether the playback should be paused when hidden. Uses metadata so has
540 // meaning only after the pipeline has started, otherwise returns false.
541 // Doesn't check if the playback can actually be paused depending on the
543 bool ShouldPausePlaybackWhenHidden() const;
545 // Whether the video track should be disabled when hidden. Uses metadata so
546 // has meaning only after the pipeline has started, otherwise returns false.
547 // Doesn't check if the video track can actually be disabled depending on the
549 bool ShouldDisableVideoWhenHidden() const;
551 // Whether the video is suitable for background playback optimizations (either
552 // pausing it or disabling the video track). Uses metadata so has meaning only
553 // after the pipeline has started, otherwise returns false.
554 // The logical OR between the two methods above that is also used as their
555 // common implementation.
556 bool IsBackgroundOptimizationCandidate() const;
558 // If enabling or disabling background video optimization has been delayed,
559 // because of the pipeline not running, seeking or resuming, this method
560 // needs to be called to update the optimization state.
561 void UpdateBackgroundVideoOptimizationState();
563 // Pauses a hidden video only player to save power if possible.
564 // Must be called when either of the following happens:
565 // - right after the video was hidden,
566 // - right ater the pipeline has resumed if the video is hidden.
567 void PauseVideoIfNeeded();
569 // Disables the video track to save power if possible.
570 // Must be called when either of the following happens:
571 // - right after the video was hidden,
572 // - right after the pipeline has started (|seeking_| is used to detect the
573 // when pipeline started) if the video is hidden,
574 // - right ater the pipeline has resumed if the video is hidden.
575 void DisableVideoTrackIfNeeded();
577 // Enables the video track if it was disabled before to save power.
578 // Must be called when either of the following happens:
579 // - right after the video was shown,
580 // - right before the pipeline is requested to resume
581 // (see https://crbug.com/678374),
582 // - right after the pipeline has resumed if the video is not hidden.
583 void EnableVideoTrackIfNeeded();
585 // Overrides the pipeline statistics returned by GetPiplineStatistics() for
587 void SetPipelineStatisticsForTest(const media::PipelineStatistics& stats);
589 // Returns the pipeline statistics or the value overridden by tests.
590 media::PipelineStatistics GetPipelineStatistics() const;
592 // Overrides the pipeline media duration returned by
593 // GetPipelineMediaDuration() for tests.
594 void SetPipelineMediaDurationForTest(base::TimeDelta duration);
596 // Returns the pipeline media duration or the value overridden by tests.
597 base::TimeDelta GetPipelineMediaDuration() const;
599 media::MediaContentType GetMediaContentType() const;
601 // Records |duration| to the appropriate metric based on whether we're
602 // handling a src= or MSE based playback.
603 void RecordUnderflowDuration(base::TimeDelta duration);
605 // Called by the data source (for src=) or demuxer (for mse) when loading
607 // Can be called quite often.
610 // Returns true when we estimate that we can play the rest of the media
611 // without buffering.
612 bool CanPlayThrough();
614 // Internal implementation of Pipeline::Client::OnBufferingStateChange(). When
615 // |for_suspended_start| is true, the given state will be set even if the
616 // pipeline is not currently stable.
617 void OnBufferingStateChangeInternal(media::BufferingState state,
618 media::BufferingStateChangeReason reason,
619 bool for_suspended_start = false);
621 // Records |natural_size| to MediaLog and video height to UMA.
622 void RecordVideoNaturalSize(const gfx::Size& natural_size);
624 void SetTickClockForTest(const base::TickClock* tick_clock);
626 // Returns the current time without clamping to Duration() as required by
627 // HTMLMediaElement for handling ended. This method will never return a
628 // negative or kInfiniteDuration value. See http://crbug.com/409280,
629 // http://crbug.com/645998, and http://crbug.com/751823 for reasons why.
630 base::TimeDelta GetCurrentTimeInternal() const;
632 // Called by the compositor the very first time a frame is received.
633 void OnFirstFrame(base::TimeTicks frame_time);
635 // Records timing metrics for three UMA metrics: #key.SRC, #key.MSE, and
636 // #key.EME. The SRC and MSE ones are mutually exclusive based on the presence
637 // of |chunk_demuxer_|, while the EME one is only recorded if |is_encrypted_|.
638 void RecordTimingUMA(const std::string& key, base::TimeDelta elapsed);
640 // Records the encryption scheme used by the stream |stream_name|. This is
641 // only recorded when metadata is available.
642 void RecordEncryptionScheme(const std::string& stream_name,
643 media::EncryptionScheme encryption_scheme);
645 // Returns whether the player is currently displayed in Picture-in-Picture.
646 // It will return true even if the player is in AutoPIP mode.
647 // The player MUST have a `client_` when this call happen.
648 bool IsInPictureInPicture() const;
650 // Sets the UKM container name if needed.
651 void MaybeSetContainerNameForMetrics();
653 // Switch to SurfaceLayer, either initially or from VideoLayer.
654 void ActivateSurfaceLayerForVideo();
656 // Called by |compositor_| upon presenting a frame, after
657 // RequestAnimationFrame() is called.
658 void OnNewFramePresentedCallback();
660 // Notifies |mb_data_source_| of playback and rate changes which may increase
661 // the amount of data the DataSource buffers. Does nothing prior to reaching
662 // kReadyStateHaveEnoughData for the first time.
663 void MaybeUpdateBufferSizesForPlayback();
665 // Create / recreate |smoothness_helper_|, with current features. Will take
666 // no action if we already have a smoothness helper with the same features
667 // that we want now. Will destroy the helper if we shouldn't be measuring
668 // smoothness right now.
669 void UpdateSmoothnessHelper();
671 // Get the LearningTaskController for |task_name|.
672 std::unique_ptr<media::learning::LearningTaskController>
673 GetLearningTaskController(const char* task_name);
675 // Returns whether the player has an audio track and whether it should be
676 // allowed to play it.
677 bool HasUnmutedAudio() const;
679 // Returns true if the video frame from this player are being captured.
680 bool IsVideoBeingCaptured() const;
682 #if defined(TIZEN_VIDEO_HOLE)
683 bool ShouldUseVideoHole() const;
684 void CreateVideoHoleFrame();
685 void OnDrawableContentRectChanged(gfx::Rect rect, bool is_video);
686 bool UpdateBoundaryRectangle();
687 void StartLayerBoundUpdateTimer();
688 void StopLayerBoundUpdateTimer();
689 void OnLayerBoundUpdateTimerFired();
692 // Report UMAs when this object instance is destroyed.
693 void ReportSessionUMAs() const;
695 WebLocalFrame* const frame_;
697 WebMediaPlayer::NetworkState network_state_ =
698 WebMediaPlayer::kNetworkStateEmpty;
699 WebMediaPlayer::ReadyState ready_state_ =
700 WebMediaPlayer::kReadyStateHaveNothing;
701 WebMediaPlayer::ReadyState highest_ready_state_ =
702 WebMediaPlayer::kReadyStateHaveNothing;
704 // Preload state for when |data_source_| is created after setPreload().
705 MultiBufferDataSource::Preload preload_ = MultiBufferDataSource::METADATA;
707 // Poster state (for UMA reporting).
708 bool has_poster_ = false;
710 // Task runner for posting tasks on Chrome's main thread. Also used
711 // for DCHECKs so methods calls won't execute in the wrong thread.
712 const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
713 const scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
714 const scoped_refptr<base::TaskRunner> worker_task_runner_;
716 // This is the ID that is used within the internals of the media element
717 // primarily for correlating logs.
718 const media::MediaPlayerLoggingID media_player_id_;
720 std::unique_ptr<media::MediaLog> media_log_;
722 // |pipeline_controller_| owns an instance of Pipeline.
723 std::unique_ptr<media::PipelineController> pipeline_controller_;
725 // The LoadType passed in the |load_type| parameter of the load() call.
726 LoadType load_type_ = kLoadTypeURL;
728 // Cache of metadata for answering hasAudio(), hasVideo(), and naturalSize().
729 media::PipelineMetadata pipeline_metadata_;
731 // Whether the video is known to be opaque or not.
732 bool opaque_ = false;
736 // TODO(scherkus): we have these because Pipeline favours the simplicity of a
737 // single "playback rate" over worrying about paused/stopped etc... It forces
738 // all clients to manage the pause+playback rate externally, but is that
739 // really a bad thing?
741 // TODO(scherkus): since SetPlaybackRate(0) is asynchronous and we don't want
742 // to hang the render thread during pause(), we record the time at the same
743 // time we pause and then return that value in currentTime(). Otherwise our
744 // clock can creep forward a little bit while the asynchronous
745 // SetPlaybackRate(0) is being executed.
746 double playback_rate_ = 0.0;
748 // Counter that limits spam to |media_log_| of |playback_rate_| changes.
749 int num_playback_rate_logs_ = 0;
751 // Set while paused. |paused_time_| is only valid when |paused_| is true.
753 base::TimeDelta paused_time_;
755 // Set if paused automatically when hidden and need to resume when visible.
756 // Reset if paused for any other reason.
757 bool paused_when_hidden_ = false;
759 // Set when starting, seeking, and resuming (all of which require a Pipeline
760 // seek). |seek_time_| is only valid when |seeking_| is true.
761 bool seeking_ = false;
762 base::TimeDelta seek_time_;
764 // Set when doing a restart (a suspend and resume in sequence) of the pipeline
765 // in order to destruct and reinitialize the decoders. This is separate from
766 // |pending_resume_| and |pending_suspend_| because they can be elided in
767 // certain cases, whereas for a restart they must happen.
768 // TODO(sandersd,watk): Create a simpler interface for a pipeline restart.
769 bool pending_suspend_resume_cycle_ = false;
771 // TODO(scherkus): Replace with an explicit ended signal to HTMLMediaElement,
772 // see http://crbug.com/409280
775 // Tracks whether to issue time changed notifications during buffering state
777 bool should_notify_time_changed_ = false;
779 bool overlay_enabled_ = false;
781 // Whether the current decoder requires a restart on overlay transitions.
782 bool decoder_requires_restart_for_overlay_ = false;
784 WebMediaPlayerClient* const client_;
785 WebMediaPlayerEncryptedMediaClient* const encrypted_client_;
787 // WebMediaPlayer notifies the |delegate_| of playback state changes using
788 // |delegate_id_|; an id provided after registering with the delegate. The
789 // WebMediaPlayer may also receive directives (play, pause) from the delegate
790 // via the WebMediaPlayerDelegate::Observer interface after registration.
792 // NOTE: HTMLMediaElement is a ExecutionContextLifecycleObserver, and
793 // will receive a call to contextDestroyed() when Document::shutdown()
794 // is called. Document::shutdown() is called before the frame detaches (and
795 // before the frame is destroyed). RenderFrameImpl owns |delegate_| and is
796 // guaranteed to outlive |this|; thus it is safe to store |delegate_| as a raw
798 WebMediaPlayerDelegate* delegate_;
799 int delegate_id_ = 0;
801 // The playback state last reported to |delegate_|, to avoid setting duplicate
803 // TODO(sandersd): The delegate should be implementing deduplication.
804 DelegateState delegate_state_ = DelegateState::GONE;
805 bool delegate_has_audio_ = false;
807 WebMediaPlayerBuilder::DeferLoadCB defer_load_cb_;
809 // Members for notifying upstream clients about internal memory usage. The
810 // |adjust_allocated_memory_cb_| must only be called on |main_task_runner_|.
811 base::RepeatingTimer memory_usage_reporting_timer_;
812 WebMediaPlayerBuilder::AdjustAllocatedMemoryCB adjust_allocated_memory_cb_;
813 int64_t last_reported_memory_usage_ = 0;
814 std::unique_ptr<media::MemoryDumpProviderProxy> main_thread_mem_dumper_;
815 std::unique_ptr<media::MemoryDumpProviderProxy> media_thread_mem_dumper_;
817 // Routes audio playback to either AudioRendererSink or WebAudio.
818 scoped_refptr<WebAudioSourceProviderImpl> audio_source_provider_;
820 // These two are mutually exclusive:
821 // |data_source_| is used for regular resource loads.
822 // |chunk_demuxer_| is used for Media Source resource loads.
824 // |demuxer_| will contain the appropriate demuxer based on which resource
825 // load strategy we're using.
826 MultiBufferDataSource* mb_data_source_ = nullptr;
827 std::unique_ptr<media::DataSource> data_source_;
828 std::unique_ptr<media::Demuxer> demuxer_;
829 media::ChunkDemuxer* chunk_demuxer_ = nullptr;
831 std::unique_ptr<base::MemoryPressureListener> memory_pressure_listener_;
833 const base::TickClock* tick_clock_ = nullptr;
835 std::unique_ptr<BufferedDataSourceHostImpl> buffered_data_source_host_;
836 UrlIndex* const url_index_;
837 scoped_refptr<viz::RasterContextProvider> raster_context_provider_;
839 // Video rendering members.
840 // The |compositor_| runs on the compositor thread, or if
841 // kEnableSurfaceLayerForVideo is enabled, the media thread. This task runner
842 // posts tasks for the |compositor_| on the correct thread.
843 scoped_refptr<base::SingleThreadTaskRunner> vfc_task_runner_;
844 std::unique_ptr<VideoFrameCompositor>
845 compositor_; // Deleted on |vfc_task_runner_|.
846 media::PaintCanvasVideoRenderer video_renderer_;
848 // The compositor layer for displaying the video content when using composited
850 scoped_refptr<cc::VideoLayer> video_layer_;
852 std::unique_ptr<WebContentDecryptionModuleResult> set_cdm_result_;
854 // If a CdmContext is attached keep a reference to the CdmContextRef, so that
855 // it is not destroyed until after the pipeline is done with it.
856 std::unique_ptr<media::CdmContextRef> cdm_context_ref_;
858 // Keep track of the CdmContextRef while it is in the process of attaching to
860 std::unique_ptr<media::CdmContextRef> pending_cdm_context_ref_;
862 // True when encryption is detected, either by demuxer or by presence of a
863 // ContentDecyprtionModule (CDM).
864 bool is_encrypted_ = false;
866 // Captured once the cdm is provided to SetCdmInternal(). Used in creation of
867 // |video_decode_stats_reporter_|.
868 absl::optional<media::CdmConfig> cdm_config_;
870 // Tracks if we are currently flinging a video (e.g. in a RemotePlayback
871 // session). Used to prevent videos from being paused when hidden.
872 // TODO(https://crbug.com/839651): remove or rename this flag, when removing
874 bool is_flinging_ = false;
876 // Tracks if we are currently using a remote renderer. See
877 // SwitchToRemoteRenderer().
878 bool is_remote_rendering_ = false;
880 // The last volume received by setVolume() and the last volume multiplier from
881 // SetVolumeMultiplier(). The multiplier is typical 1.0, but may be less
882 // if the WebMediaPlayerDelegate has requested a volume reduction (ducking)
883 // for a transient sound. Playout volume is derived by volume * multiplier.
884 double volume_ = 1.0;
885 double volume_multiplier_ = 1.0;
887 std::unique_ptr<media::RendererFactorySelector> renderer_factory_selector_;
889 // For canceling AndroidOverlay routing token requests.
890 base::CancelableOnceCallback<void(const base::UnguessableToken&)>
893 // If overlay info is requested before we have it, then the request is saved
894 // and satisfied once the overlay info is available. If the decoder does not
895 // require restart to change surfaces, this is callback is kept until cleared
897 media::ProvideOverlayInfoCB provide_overlay_info_cb_;
899 // On Android an overlay surface means using
900 // SurfaceView instead of SurfaceTexture.
902 // Allow overlays for all video on android.
903 bool always_enable_overlays_ = false;
905 // Suppresses calls to OnPipelineError() after destruction / shutdown has been
906 // started; prevents us from spuriously logging errors that are transient or
908 bool suppress_destruction_errors_ = false;
910 // TODO(dalecurtis): The following comment is inaccurate as this value is also
911 // used for, for example, data URLs.
912 // Used for HLS playback and in certain fallback paths (e.g. on older devices
913 // that can't support the unified media pipeline).
916 // NOTE: |using_media_player_renderer_| is set based on the usage of a
917 // MediaResource::Type::URL in StartPipeline(). This works because
918 // MediaPlayerRendererClientFactory is the only factory that uses
919 // MediaResource::Type::URL for now.
920 bool using_media_player_renderer_ = false;
922 #if BUILDFLAG(IS_ANDROID)
923 // Set during the initial DoLoad() call. Used to determine whether to allow
924 // credentials or not for MediaPlayerRenderer.
925 bool allow_media_player_renderer_credentials_ = false;
928 // Stores the current position state of the media.
929 media_session::MediaPosition media_position_state_;
931 // Set whenever the demuxer encounters an HLS file.
932 // This flag is distinct from |using_media_player_renderer_|, because on older
933 // devices we might use MediaPlayerRenderer for non HLS playback.
934 bool demuxer_found_hls_ = false;
936 // Called sometime after the media is suspended in a playing state in
937 // OnFrameHidden(), causing the state to change to paused.
938 base::OneShotTimer background_pause_timer_;
940 // Monitors the watch time of the played content.
941 std::unique_ptr<WatchTimeReporter> watch_time_reporter_;
942 media::AudioDecoderType audio_decoder_type_ =
943 media::AudioDecoderType::kUnknown;
944 media::VideoDecoderType video_decoder_type_ =
945 media::VideoDecoderType::kUnknown;
947 // The time at which DoLoad() is executed.
948 base::TimeTicks load_start_time_;
950 // Time elapsed time from |load_start_time_| to OnMetadata(). Used to later
951 // adjust |load_start_time_| if a suspended startup occurred.
952 base::TimeDelta time_to_metadata_;
953 bool skip_metrics_due_to_startup_suspend_ = false;
955 bool have_reported_time_to_play_ready_ = false;
957 // Records pipeline statistics for describing media capabilities.
958 std::unique_ptr<VideoDecodeStatsReporter> video_decode_stats_reporter_;
960 // Elapsed time since we've last reached BUFFERING_HAVE_NOTHING.
961 std::unique_ptr<base::ElapsedTimer> underflow_timer_;
963 // Used to track loading progress, used by IsPrerollAttemptNeeded().
964 // |preroll_attempt_pending_| indicates that the clock has been reset
965 // (awaiting a resume to start), while |preroll_attempt_start_time_| tracks
966 // when a preroll attempt began.
967 bool preroll_attempt_pending_ = false;
968 base::TimeTicks preroll_attempt_start_time_;
970 // Monitors the player events.
971 base::WeakPtr<media::MediaObserver> observer_;
973 // Owns the weblayer and obtains/maintains SurfaceIds.
974 std::unique_ptr<WebSurfaceLayerBridge> bridge_;
976 // The maximum video keyframe distance that allows triggering background
977 // playback optimizations (non-MSE).
978 base::TimeDelta max_keyframe_distance_to_disable_background_video_;
980 // The maximum video keyframe distance that allows triggering background
981 // playback optimizations (MSE).
982 base::TimeDelta max_keyframe_distance_to_disable_background_video_mse_;
984 // When MSE memory pressure based garbage collection is enabled, the
985 // |enable_instant_source_buffer_gc| controls whether the GC is done
986 // immediately on memory pressure notification or during the next SourceBuffer
987 // append (slower, but MSE spec compliant).
988 bool enable_instant_source_buffer_gc_ = false;
990 // Whether disabled the video track as an optimization.
991 bool video_track_disabled_ = false;
993 // Whether the pipeline is being resumed at the moment.
994 bool is_pipeline_resuming_ = false;
996 // When this is true, pipeline will not be auto suspended.
997 bool disable_pipeline_auto_suspend_ = false;
999 // Pipeline statistics overridden by tests.
1000 absl::optional<media::PipelineStatistics> pipeline_statistics_for_test_;
1002 // Pipeline media duration overridden by tests.
1003 absl::optional<base::TimeDelta> pipeline_media_duration_for_test_;
1005 // Whether the video requires a user gesture to resume after it was paused in
1006 // the background. Affects the value of ShouldPausePlaybackWhenHidden().
1007 bool video_locked_when_paused_when_hidden_ = false;
1009 // Whether embedded media experience is currently enabled.
1010 bool embedded_media_experience_enabled_ = false;
1012 // When should we use SurfaceLayer for video?
1013 bool use_surface_layer_ = false;
1015 // Whether surface layer is currently in use to display frames.
1016 bool surface_layer_for_video_enabled_ = false;
1018 CreateSurfaceLayerBridgeCB create_bridge_callback_;
1020 bool initial_video_height_recorded_ = false;
1022 enum class OverlayMode {
1023 // All overlays are turned off.
1026 // Use AndroidOverlay for overlays.
1030 OverlayMode overlay_mode_ = OverlayMode::kNoOverlays;
1032 // Optional callback to request the routing token for AndroidOverlay.
1033 media::RequestRoutingTokenCallback request_routing_token_cb_;
1035 // If |overlay_routing_token_is_pending_| is false, then
1036 // |overlay_routing_token_| contains the routing token we should send, if any.
1037 // Otherwise, |overlay_routing_token_| is undefined. We set the flag while
1038 // we have a request for the token that hasn't been answered yet; i.e., it
1039 // means that we don't know what, if any, token we should be using.
1040 bool overlay_routing_token_is_pending_ = false;
1041 media::OverlayInfo::RoutingToken overlay_routing_token_;
1043 media::OverlayInfo overlay_info_;
1045 base::CancelableOnceClosure update_background_status_cb_;
1047 // We cannot use `update_background_status_cb_.IsCancelled()` as that changes
1048 // when the callback is run, even if not explicitly cancelled. This is
1049 // initialized to true to keep in line with the existing behavior of
1050 // base::CancellableOnceClosure.
1051 bool is_background_status_change_cancelled_ = true;
1053 mojo::Remote<media::mojom::MediaMetricsProvider> media_metrics_provider_;
1054 mojo::Remote<media::mojom::PlaybackEventsRecorder> playback_events_recorder_;
1056 absl::optional<ReadyState> stale_state_override_for_testing_;
1058 // True if we attempt to start the media pipeline in a suspended state for
1059 // preload=metadata. Cleared upon pipeline startup.
1060 bool attempting_suspended_start_ = false;
1062 // True if a frame has ever been rendered.
1063 bool has_first_frame_ = false;
1065 // True if we have not yet rendered a first frame, but one is needed. Set back
1066 // to false as soon as |has_first_frame_| is set to true.
1067 bool needs_first_frame_ = false;
1069 // True if StartPipeline() completed a lazy load startup.
1070 bool did_lazy_load_ = false;
1072 // Whether the renderer should automatically suspend media playback in
1074 bool is_background_suspend_enabled_ = false;
1076 // If disabled, video will be auto paused when in background. Affects the
1077 // value of ShouldPausePlaybackWhenHidden().
1078 bool is_background_video_playback_enabled_ = true;
1080 // Whether background video optimization is supported on current platform.
1081 bool is_background_video_track_optimization_supported_ = true;
1083 bool was_suspended_for_frame_closed_ = false;
1085 #if defined(TIZEN_MULTIMEDIA)
1086 bool was_suspended_by_player_ = false;
1089 base::CancelableOnceClosure have_enough_after_lazy_load_cb_;
1091 media::RendererType renderer_type_ = media::RendererType::kDefault;
1092 media::SimpleWatchTimer simple_watch_timer_;
1094 LearningExperimentHelper will_play_helper_;
1096 #if defined(TIZEN_VIDEO_HOLE)
1097 bool is_video_hole_;
1098 gfx::RectF last_computed_rect_;
1099 base::RepeatingTimer layer_bound_update_timer_;
1102 // Stores the optional override Demuxer until it is used in DoLoad().
1103 std::unique_ptr<media::Demuxer> demuxer_override_;
1105 std::unique_ptr<PowerStatusHelper> power_status_helper_;
1107 // Created while playing, deleted otherwise.
1108 std::unique_ptr<SmoothnessHelper> smoothness_helper_;
1109 absl::optional<int> last_reported_fps_;
1111 // Time of the last call to GetCurrentFrameFromCompositor(). Used to prevent
1112 // background optimizations from being applied when capturing is active.
1113 base::TimeTicks last_frame_request_time_;
1115 // Count the number of times a video frame is being readback.
1116 unsigned video_frame_readback_count_ = 0;
1118 base::WeakPtr<WebMediaPlayerImpl> weak_this_;
1119 base::WeakPtrFactory<WebMediaPlayerImpl> weak_factory_{this};
1122 } // namespace blink
1124 #endif // THIRD_PARTY_BLINK_RENDERER_PLATFORM_MEDIA_WEB_MEDIA_PLAYER_IMPL_H_