1 // Copyright 2013 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef THIRD_PARTY_BLINK_RENDERER_PLATFORM_MEDIA_WEB_MEDIA_PLAYER_IMPL_H_
6 #define THIRD_PARTY_BLINK_RENDERER_PLATFORM_MEDIA_WEB_MEDIA_PLAYER_IMPL_H_
14 #include "base/cancelable_callback.h"
15 #include "base/compiler_specific.h"
16 #include "base/memory/raw_ptr.h"
17 #include "base/memory/ref_counted.h"
18 #include "base/memory/scoped_refptr.h"
19 #include "base/memory/weak_ptr.h"
20 #include "base/task/sequenced_task_runner.h"
21 #include "base/threading/thread.h"
22 #include "base/time/default_tick_clock.h"
23 #include "base/time/time.h"
24 #include "base/timer/elapsed_timer.h"
25 #include "base/timer/timer.h"
26 #include "build/build_config.h"
27 #include "media/base/cdm_config.h"
28 #include "media/base/data_source.h"
29 #include "media/base/demuxer.h"
30 #include "media/base/eme_constants.h"
31 #include "media/base/encryption_scheme.h"
32 #include "media/base/media_observer.h"
33 #include "media/base/media_tracks.h"
34 #include "media/base/overlay_info.h"
35 #include "media/base/pipeline_impl.h"
36 #include "media/base/renderer_factory_selector.h"
37 #include "media/base/routing_token_callback.h"
38 #include "media/base/simple_watch_timer.h"
39 #include "media/filters/demuxer_manager.h"
40 #include "media/mojo/mojom/media_metrics_provider.mojom.h"
41 #include "media/mojo/mojom/playback_events_recorder.mojom.h"
42 #include "media/renderers/paint_canvas_video_renderer.h"
43 #include "mojo/public/cpp/bindings/pending_remote.h"
44 #include "mojo/public/cpp/bindings/remote.h"
45 #include "services/media_session/public/cpp/media_position.h"
46 #include "third_party/abseil-cpp/absl/types/optional.h"
47 #include "third_party/blink/public/platform/media/video_frame_compositor.h"
48 #include "third_party/blink/public/platform/media/web_media_player_builder.h"
49 #include "third_party/blink/public/platform/media/webmediaplayer_delegate.h"
50 #include "third_party/blink/public/platform/web_audio_source_provider.h"
51 #include "third_party/blink/public/platform/web_content_decryption_module_result.h"
52 #include "third_party/blink/public/platform/web_media_player.h"
53 #include "third_party/blink/public/platform/web_surface_layer_bridge.h"
54 #include "third_party/blink/public/web/modules/media/webmediaplayer_util.h"
55 #include "third_party/blink/renderer/platform/allow_discouraged_type.h"
56 #include "third_party/blink/renderer/platform/media/learning_experiment_helper.h"
57 #include "third_party/blink/renderer/platform/media/multi_buffer_data_source.h"
58 #include "third_party/blink/renderer/platform/media/smoothness_helper.h"
59 #include "third_party/blink/renderer/platform/platform_export.h"
63 class SingleThreadTaskRunner;
76 class LearningTaskController;
84 class MemoryDumpProviderProxy;
85 class PipelineController;
86 class SwitchableAudioRendererSink;
90 class RasterContextProvider;
94 class BufferedDataSourceHostImpl;
95 class PowerStatusHelper;
96 class ThreadSafeBrowserInterfaceBrokerProxy;
98 class VideoDecodeStatsReporter;
99 class VideoFrameCompositor;
100 class WatchTimeReporter;
101 class WebAudioSourceProviderImpl;
102 class WebContentDecryptionModule;
104 class WebMediaPlayerClient;
105 class WebMediaPlayerEncryptedMediaClient;
107 // The canonical implementation of WebMediaPlayer that's backed by
108 // Pipeline. Handles normal resource loading, Media Source, and
110 class PLATFORM_EXPORT WebMediaPlayerImpl
111 : public WebMediaPlayer,
112 public WebMediaPlayerDelegate::Observer,
113 public media::Pipeline::Client,
114 public media::MediaObserverClient,
115 public media::DemuxerManager::Client,
116 public WebSurfaceLayerBridgeObserver,
117 public SmoothnessHelper::Client {
119 // Constructs a WebMediaPlayer implementation using Chromium's media stack.
120 // |delegate| and |renderer_factory_selector| must not be null.
122 WebLocalFrame* frame,
123 WebMediaPlayerClient* client,
124 WebMediaPlayerEncryptedMediaClient* encrypted_client,
125 WebMediaPlayerDelegate* delegate,
126 std::unique_ptr<media::RendererFactorySelector> renderer_factory_selector,
128 std::unique_ptr<VideoFrameCompositor> compositor,
129 std::unique_ptr<media::MediaLog> media_log,
130 media::MediaPlayerLoggingID player_id,
131 WebMediaPlayerBuilder::DeferLoadCB defer_load_cb,
132 scoped_refptr<media::SwitchableAudioRendererSink> audio_renderer_sink,
133 scoped_refptr<base::SequencedTaskRunner> media_task_runner,
134 scoped_refptr<base::TaskRunner> worker_task_runner,
135 scoped_refptr<base::SingleThreadTaskRunner> compositor_task_runner,
136 scoped_refptr<base::SingleThreadTaskRunner>
137 video_frame_compositor_task_runner,
138 WebMediaPlayerBuilder::AdjustAllocatedMemoryCB adjust_allocated_memory_cb,
139 WebContentDecryptionModule* initial_cdm,
140 media::RequestRoutingTokenCallback request_routing_token_cb,
141 base::WeakPtr<media::MediaObserver> media_observer,
142 bool enable_instant_source_buffer_gc,
143 bool embedded_media_experience_enabled,
144 mojo::PendingRemote<media::mojom::MediaMetricsProvider> metrics_provider,
145 CreateSurfaceLayerBridgeCB create_bridge_callback,
146 scoped_refptr<viz::RasterContextProvider> raster_context_provider,
147 #if defined(TIZEN_VIDEO_HOLE)
150 bool use_surface_layer,
151 bool is_background_suspend_enabled,
152 bool is_background_video_play_enabled,
153 bool is_background_video_track_optimization_supported,
154 std::unique_ptr<media::Demuxer> demuxer_override,
155 scoped_refptr<ThreadSafeBrowserInterfaceBrokerProxy> remote_interfaces);
156 WebMediaPlayerImpl(const WebMediaPlayerImpl&) = delete;
157 WebMediaPlayerImpl& operator=(const WebMediaPlayerImpl&) = delete;
158 ~WebMediaPlayerImpl() override;
160 // WebSurfaceLayerBridgeObserver implementation.
161 void OnWebLayerUpdated() override;
162 void RegisterContentsLayer(cc::Layer* layer) override;
163 void UnregisterContentsLayer(cc::Layer* layer) override;
164 void OnSurfaceIdUpdated(viz::SurfaceId surface_id) override;
166 WebMediaPlayer::LoadTiming Load(LoadType load_type,
167 const WebMediaPlayerSource& source,
169 bool is_cache_disabled) override;
171 // Playback controls.
172 void Play() override;
173 void Pause() override;
174 void Seek(double seconds) override;
175 void SetRate(double rate) override;
176 void SetVolume(double volume) override;
177 void SetLatencyHint(double seconds) override;
178 void SetPreservesPitch(bool preserves_pitch) override;
179 void SetWasPlayedWithUserActivation(
180 bool was_played_with_user_activation) override;
181 void OnRequestPictureInPicture() override;
182 void OnTimeUpdate() override;
183 bool SetSinkId(const WebString& sink_id,
184 WebSetSinkIdCompleteCallback completion_callback) override;
185 void SetPoster(const WebURL& poster) override;
186 void SetPreload(WebMediaPlayer::Preload preload) override;
187 WebTimeRanges Buffered() const override;
188 WebTimeRanges Seekable() const override;
189 #if defined(TIZEN_MULTIMEDIA)
190 void Suspend() override;
191 void Resume() override;
193 bool SuspendedByPlayer() override { return was_suspended_by_player_; }
196 void OnFrozen() override;
198 // paint() the current video frame into |canvas|. This is used to support
199 // various APIs and functionalities, including but not limited to: <canvas>,
200 // ImageBitmap, printing and capturing capabilities.
201 void Paint(cc::PaintCanvas* canvas,
202 const gfx::Rect& rect,
203 cc::PaintFlags& flags) override;
204 scoped_refptr<media::VideoFrame> GetCurrentFrameThenUpdate() override;
205 absl::optional<media::VideoFrame::ID> CurrentFrameId() const override;
206 media::PaintCanvasVideoRenderer* GetPaintCanvasVideoRenderer() override;
208 // True if the loaded media has a playable video/audio track.
209 bool HasVideo() const override;
210 bool HasAudio() const override;
212 void EnabledAudioTracksChanged(
213 const WebVector<WebMediaPlayer::TrackId>& enabledTrackIds) override;
214 void SelectedVideoTrackChanged(
215 WebMediaPlayer::TrackId* selectedTrackId) override;
217 // Dimensions of the video.
218 gfx::Size NaturalSize() const override;
220 gfx::Size VisibleSize() const override;
222 // Getters of playback state.
223 bool Paused() const override;
224 bool Seeking() const override;
225 double Duration() const override;
226 virtual double timelineOffset() const;
227 bool IsEnded() const override;
229 // Shared between the WebMediaPlayer and DemuxerManager::Client interfaces.
230 double CurrentTime() const override;
232 bool PausedWhenHidden() const override;
234 // Internal states of loading and network.
235 // TODO(hclam): Ask the pipeline about the state rather than having reading
236 // them from members which would cause race conditions.
237 WebMediaPlayer::NetworkState GetNetworkState() const override;
238 WebMediaPlayer::ReadyState GetReadyState() const override;
240 WebString GetErrorMessage() const override;
241 bool DidLoadingProgress() override;
242 bool WouldTaintOrigin() const override;
244 double MediaTimeForTimeValue(double timeValue) const override;
246 unsigned DecodedFrameCount() const override;
247 unsigned DroppedFrameCount() const override;
248 uint64_t AudioDecodedByteCount() const override;
249 uint64_t VideoDecodedByteCount() const override;
251 bool PassedTimingAllowOriginCheck() const override;
253 void SetVolumeMultiplier(double multiplier) override;
254 void SetPersistentState(bool persistent) override;
255 void SetPowerExperimentState(bool state) override;
256 void SuspendForFrameClosed() override;
258 bool HasAvailableVideoFrame() const override;
259 bool HasReadableVideoFrame() const override;
261 scoped_refptr<WebAudioSourceProviderImpl> GetAudioSourceProvider() override;
263 void SetContentDecryptionModule(
264 WebContentDecryptionModule* cdm,
265 WebContentDecryptionModuleResult result) override;
267 void EnteredFullscreen() override;
268 void ExitedFullscreen() override;
269 void BecameDominantVisibleContent(bool is_dominant) override;
270 void SetIsEffectivelyFullscreen(
271 WebFullscreenVideoStatus fullscreen_video_status) override;
272 void OnHasNativeControlsChanged(bool) override;
273 void OnDisplayTypeChanged(DisplayType display_type) override;
275 // WebMediaPlayerDelegate::Observer implementation.
276 void OnFrameHidden() override;
277 void OnFrameShown() override;
278 void OnIdleTimeout() override;
280 void RequestRemotePlaybackDisabled(bool disabled) override;
281 void RequestMediaRemoting() override;
283 #if BUILDFLAG(IS_ANDROID)
284 // TODO(https://crbug.com/839651): Rename Flinging[Started/Stopped] to
285 // RemotePlayback[Started/Stopped] once the other RemotePlayback methods have
287 void FlingingStarted() override;
288 void FlingingStopped() override;
290 // Called when the play/pause state of media playing on a remote cast device
291 // changes, and WMPI wasn't the originator of that change (e.g. a phone on the
292 // same network paused the cast device via the casting notification).
293 // This is only used by the FlingingRenderer/FlingingRendererClient, when we
294 // are flinging media (a.k.a. RemotePlayback).
295 // The consistency between the WMPI state and the cast device state is not
296 // guaranteed, and it a best effort, which can always be fixed by the user by
297 // tapping play/pause once. Attempts to enfore stronger consistency guarantees
298 // have lead to unstable states, and a worse user experience.
299 void OnRemotePlayStateChange(media::MediaStatus::State state);
302 // media::MediaObserverClient implementation.
303 void SwitchToRemoteRenderer(
304 const std::string& remote_device_friendly_name) override;
305 void SwitchToLocalRenderer(
306 media::MediaObserverClient::ReasonToSwitchToLocal reason) override;
307 void UpdateRemotePlaybackCompatibility(bool is_compatible) override;
309 // Test helper methods for exercising media suspension. Once called, when
310 // |target_state| is reached or exceeded the stale flag will be set when
311 // computing the play state, which will trigger suspend if the player is
312 // paused; see UpdatePlayState_ComputePlayState() for the exact details.
313 void ForceStaleStateForTesting(ReadyState target_state) override;
314 bool IsSuspendedForTesting() override;
315 bool DidLazyLoad() const override;
316 void OnBecameVisible() override;
317 bool IsOpaque() const override;
318 int GetDelegateId() override;
319 absl::optional<viz::SurfaceId> GetSurfaceId() override;
320 GURL GetSrcAfterRedirects() override;
321 void RequestVideoFrameCallback() override;
322 std::unique_ptr<WebMediaPlayer::VideoFramePresentationMetadata>
323 GetVideoFramePresentationMetadata() override;
324 void UpdateFrameIfStale() override;
326 base::WeakPtr<WebMediaPlayer> AsWeakPtr() override;
327 void RegisterFrameSinkHierarchy() override;
328 void UnregisterFrameSinkHierarchy() override;
330 bool IsBackgroundMediaSuspendEnabled() const {
331 return is_background_suspend_enabled_;
334 // Distinct states that |delegate_| can be in. (Public for testing.)
335 enum class DelegateState {
341 // Playback state variables computed together in UpdatePlayState().
342 // (Public for testing.)
344 DelegateState delegate_state;
346 bool is_memory_reporting_enabled;
350 // Allow background video tracks with ~5 second keyframes (rounding down) to
351 // be disabled to save resources.
352 enum { kMaxKeyframeDistanceToDisableBackgroundVideoMs = 5500 };
355 friend class WebMediaPlayerImplTest;
356 friend class WebMediaPlayerImplBackgroundBehaviorTest;
358 // Helper function that can be used for generating different UMA records:
359 // |key| is the UMA prefix, such as "Media.TimeToPlayReady", for example.
360 // |UmaFunction| is some UMA function, like base::UmaHistogramMediumTimes
361 // which the record actually gets logged with
362 // |T...| are the arguments passed. Usually this is only one piece of data,
363 // such as a base::TimeDelta, in the case of UmaHistogramMediumTimes,
364 // but could also be a series of fields that customize bucket sizes
367 // Finally, the |Flags| template argument is used to determine which suffixes
368 // are logged - An integer enum is provided |SplitHistogramTypes| which is
369 // a bitmask, and can be used to require logging:
370 // PlaybackType: {".SRC", ".MSE", ".HLS" (in the future)} based on demuxer
372 // Encrypted: {".EME"} based on the value of |is_encrypted_|
373 // All: {".All"} all the time.
374 // |Flags| is provided as a template argument instead of a function argument
375 // in order to guard different components in "if constexpr" conditionals,
376 // so we won't even compile in strings such as "Media.TimeToPlayReady.All"
377 // if it's not specified.
378 template <uint32_t Flags, typename... T>
379 void WriteSplitHistogram(void (*UmaFunction)(const std::string&, T...),
380 const std::string& key,
383 void EnableOverlay();
384 void DisableOverlay();
386 // Do we have overlay information? For CVV, this is a surface id. For
387 // AndroidOverlay, this is the routing token.
388 bool HaveOverlayInfo();
390 // Send OverlayInfo to the decoder.
392 // If we've requested but not yet received the surface id or routing token, or
393 // if there's no decoder-provided callback to send the overlay info, then this
394 // call will do nothing.
395 void MaybeSendOverlayInfoToDecoder();
397 void OnPipelineSuspended();
398 void OnBeforePipelineResume();
399 void OnPipelineResumed();
400 void OnPipelineSeeked(bool time_updated);
402 // media::Pipeline::Client overrides.
403 void OnError(media::PipelineStatus status) override;
404 void OnFallback(media::PipelineStatus status) override;
405 void OnEnded() override;
406 void OnMetadata(const media::PipelineMetadata& metadata) override;
407 void OnBufferingStateChange(
408 media::BufferingState state,
409 media::BufferingStateChangeReason reason) override;
410 void OnDurationChange() override;
411 #if BUILDFLAG(IS_TIZEN_TV)
412 void SetParentalRatingResult(bool is_pass) override;
414 void OnWaiting(media::WaitingReason reason) override;
415 void OnAudioConfigChange(const media::AudioDecoderConfig& config) override;
416 void OnVideoConfigChange(const media::VideoDecoderConfig& config) override;
417 void OnVideoNaturalSizeChange(const gfx::Size& size) override;
418 void OnVideoOpacityChange(bool opaque) override;
419 void OnVideoFrameRateChange(absl::optional<int> fps) override;
420 void OnVideoAverageKeyframeDistanceUpdate() override;
421 void OnAudioPipelineInfoChange(const media::AudioPipelineInfo& info) override;
422 void OnVideoPipelineInfoChange(const media::VideoPipelineInfo& info) override;
423 #if defined(TIZEN_MULTIMEDIA)
424 void OnSeekableTimeChange(base::TimeDelta min_time,
425 base::TimeDelta max_time,
426 bool is_live) override;
427 void OnLivePlaybackComplete() override;
429 // Called if a player in the browser process is suspended.
430 void OnRequestSuspend(bool resource_conflicted) override;
432 void FullscreenModeToggled();
435 // media::DemuxerManager::Client overrides.
436 void OnChunkDemuxerOpened(media::ChunkDemuxer* demuxer) override;
437 void OnProgress() override;
438 void OnEncryptedMediaInitData(media::EmeInitDataType init_data_type,
439 const std::vector<uint8_t>& init_data) override;
440 void MakeDemuxerThreadDumper(media::Demuxer* demuxer) override;
441 bool CouldPlayIfEnoughData() override;
442 bool IsMediaPlayerRendererClient() override;
443 void StopForDemuxerReset() override;
444 void RestartForHls() override;
445 bool IsSecurityOriginCryptographic() const override;
446 void UpdateLoadedUrl(const GURL& url) override;
447 void DemuxerRequestsSeek(base::TimeDelta seek_time) override;
449 #if BUILDFLAG(ENABLE_FFMPEG)
450 void AddAudioTrack(const std::string& id,
451 const std::string& label,
452 const std::string& language,
453 bool is_first_track) override;
454 void AddVideoTrack(const std::string& id,
455 const std::string& label,
456 const std::string& language,
457 bool is_first_track) override;
458 #endif // BUILDFLAG(ENABLE_FFMPEG)
460 #if BUILDFLAG(ENABLE_HLS_DEMUXER)
461 void GetUrlData(const GURL& gurl,
462 base::OnceCallback<void(scoped_refptr<UrlData>)> cb);
463 base::SequenceBound<media::HlsDataSourceProvider> GetHlsDataSourceProvider()
465 #endif // BUILDFLAG(ENABLE_HLS_DEMUXER)
467 // Simplified watch time reporting.
468 void OnSimpleWatchTimerTick();
470 // Actually seek. Avoids causing |should_notify_time_changed_| to be set when
471 // |time_updated| is false.
472 void DoSeek(base::TimeDelta time, bool time_updated);
474 // Called after |defer_load_cb_| has decided to allow the load. If
475 // |defer_load_cb_| is null this is called immediately.
476 void DoLoad(LoadType load_type,
479 bool is_cache_disabled);
481 // Called after synchronous initialization of a data source completes.
482 void DataSourceInitialized(bool success);
484 // Called after asynchronous initialization of a multibuffer data source
486 void MultiBufferDataSourceInitialized(bool success);
488 // Called after synchronous or asynchronous MemoryDataSource initialization.
489 void MemoryDataSourceInitialized(bool success, size_t data_size);
491 // Called if the |MultiBufferDataSource| is redirected.
492 void OnDataSourceRedirected();
494 // Called when the data source is downloading or paused.
495 void NotifyDownloading(bool is_downloading);
497 // Called by RenderFrameImpl with the overlay routing token, if we request it.
498 void OnOverlayRoutingToken(const base::UnguessableToken& token);
500 // Called by GpuVideoDecoder on Android to request a surface to render to (if
502 void OnOverlayInfoRequested(
503 bool decoder_requires_restart_for_overlay,
504 media::ProvideOverlayInfoCB provide_overlay_info_cb);
506 // Creates a Renderer via the |renderer_factory_selector_|. If the
507 // |renderer_type| is absl::nullopt, create the base Renderer. Otherwise, set
508 // the base type to be |renderer_type| and create a Renderer of that type.
509 std::unique_ptr<media::Renderer> CreateRenderer(
510 absl::optional<media::RendererType> renderer_type);
512 // Finishes starting the pipeline due to a call to load().
513 void StartPipeline();
515 // Restart the player/pipeline as soon as possible. This will destroy the
516 // current renderer, if any, and create a new one via the RendererFactory; and
517 // then seek to resume playback at the current position.
518 void ScheduleRestart();
520 // Helpers that set the network/ready state and notifies the client if
522 void SetNetworkState(WebMediaPlayer::NetworkState state);
523 void SetReadyState(WebMediaPlayer::ReadyState state);
525 // Returns the current video frame from |compositor_|, and asks the compositor
526 // to update its frame if it is stale.
527 // Can return a nullptr.
528 scoped_refptr<media::VideoFrame> GetCurrentFrameFromCompositor() const;
530 // Sets CdmContext from |cdm| on the pipeline and calls OnCdmAttached()
532 void SetCdmInternal(WebContentDecryptionModule* cdm);
534 // Called when a CDM has been attached to the |pipeline_|.
535 void OnCdmAttached(bool success);
537 // Inspects the current playback state and:
538 // - notifies |delegate_|,
539 // - toggles the memory usage reporting timer, and
540 // - toggles suspend/resume as necessary.
542 // This method should be called any time its dependent values change. These
546 // - delegate_->IsHidden(),
547 // - network_state_, ready_state_,
548 // - is_idle_, must_suspend_,
549 // - paused_, ended_,
550 // - pending_suspend_resume_cycle_,
551 // - enter_pip_callback_,
552 void UpdatePlayState();
554 // Methods internal to UpdatePlayState().
555 PlayState UpdatePlayState_ComputePlayState(bool is_flinging,
556 bool can_auto_suspend,
558 bool is_backgrounded,
559 bool is_in_picture_in_picture);
560 void SetDelegateState(DelegateState new_state, bool is_idle);
561 void SetMemoryReportingState(bool is_memory_reporting_enabled);
562 void SetSuspendState(bool is_suspended);
564 // Called at low frequency to tell external observers how much memory we're
565 // using for video playback. Called by |memory_usage_reporting_timer_|.
566 // Memory usage reporting is done in two steps, because |demuxer_| must be
567 // accessed on the media thread.
568 void ReportMemoryUsage();
569 void FinishMemoryUsageReport(int64_t demuxer_memory_usage);
571 void OnMainThreadMemoryDump(int32_t id,
572 const base::trace_event::MemoryDumpArgs& args,
573 base::trace_event::ProcessMemoryDump* pmd);
574 static void OnMediaThreadMemoryDump(
576 media::Demuxer* demuxer,
577 const base::trace_event::MemoryDumpArgs& args,
578 base::trace_event::ProcessMemoryDump* pmd);
580 // Called during OnHidden() when we want a suspended player to enter the
581 // paused state after some idle timeout.
582 void ScheduleIdlePauseTimer();
584 // Returns |true| before HaveFutureData whenever there has been loading
585 // progress and we have not been resumed for at least kLoadingToIdleTimeout
588 // This is used to delay suspension long enough for preroll to complete, which
589 // is necessay because play() will not be called before HaveFutureData (and
590 // thus we think we are idle forever).
591 bool IsPrerollAttemptNeeded();
593 void CreateWatchTimeReporter();
594 void UpdateSecondaryProperties();
596 void CreateVideoDecodeStatsReporter();
598 // Returns true if the player is hidden.
599 bool IsHidden() const;
601 // Returns true if the player is in streaming mode, meaning that the source
602 // or the demuxer doesn't support timeline or seeking.
603 bool IsStreaming() const;
605 // Return whether |pipeline_metadata_| is compatible with an overlay. This
606 // is intended for android.
607 bool DoesOverlaySupportMetadata() const;
609 // Whether the playback should be paused when hidden. Uses metadata so has
610 // meaning only after the pipeline has started, otherwise returns false.
611 // Doesn't check if the playback can actually be paused depending on the
613 bool ShouldPausePlaybackWhenHidden() const;
615 // Whether the video track should be disabled when hidden. Uses metadata so
616 // has meaning only after the pipeline has started, otherwise returns false.
617 // Doesn't check if the video track can actually be disabled depending on the
619 bool ShouldDisableVideoWhenHidden() const;
621 // If enabling or disabling background video optimization has been delayed,
622 // because of the pipeline not running, seeking or resuming, this method
623 // needs to be called to update the optimization state.
624 void UpdateBackgroundVideoOptimizationState();
626 // Pauses a hidden video only player to save power if possible.
627 // Must be called when either of the following happens:
628 // - right after the video was hidden,
629 // - right ater the pipeline has resumed if the video is hidden.
630 void PauseVideoIfNeeded();
632 // Disables the video track to save power if possible.
633 // Must be called when either of the following happens:
634 // - right after the video was hidden,
635 // - right after the pipeline has started (|seeking_| is used to detect the
636 // when pipeline started) if the video is hidden,
637 // - right ater the pipeline has resumed if the video is hidden.
638 void DisableVideoTrackIfNeeded();
640 // Enables the video track if it was disabled before to save power.
641 // Must be called when either of the following happens:
642 // - right after the video was shown,
643 // - right before the pipeline is requested to resume
644 // (see https://crbug.com/678374),
645 // - right after the pipeline has resumed if the video is not hidden.
646 void EnableVideoTrackIfNeeded();
648 // Overrides the pipeline statistics returned by GetPiplineStatistics() for
650 void SetPipelineStatisticsForTest(const media::PipelineStatistics& stats);
652 // Returns the pipeline statistics or the value overridden by tests.
653 media::PipelineStatistics GetPipelineStatistics() const;
655 // Overrides the pipeline media duration returned by
656 // GetPipelineMediaDuration() for tests.
657 void SetPipelineMediaDurationForTest(base::TimeDelta duration);
659 // Returns the pipeline media duration or the value overridden by tests.
660 base::TimeDelta GetPipelineMediaDuration() const;
662 media::MediaContentType GetMediaContentType() const;
664 // Records |duration| to the appropriate metric based on whether we're
665 // handling a src= or MSE based playback.
666 void RecordUnderflowDuration(base::TimeDelta duration);
668 // Returns true when we estimate that we can play the rest of the media
669 // without buffering.
670 bool CanPlayThrough();
672 // Internal implementation of Pipeline::Client::OnBufferingStateChange(). When
673 // |for_suspended_start| is true, the given state will be set even if the
674 // pipeline is not currently stable.
675 void OnBufferingStateChangeInternal(media::BufferingState state,
676 media::BufferingStateChangeReason reason,
677 bool for_suspended_start = false);
679 // Records |natural_size| to MediaLog and video height to UMA.
680 void RecordVideoNaturalSize(const gfx::Size& natural_size);
682 void SetTickClockForTest(const base::TickClock* tick_clock);
684 // Returns the current time without clamping to Duration() as required by
685 // HTMLMediaElement for handling ended. This method will never return a
686 // negative or kInfiniteDuration value. See http://crbug.com/409280,
687 // http://crbug.com/645998, and http://crbug.com/751823 for reasons why.
688 base::TimeDelta GetCurrentTimeInternal() const;
690 // Called by the compositor the very first time a frame is received.
691 void OnFirstFrame(base::TimeTicks frame_time, bool is_frame_readable);
693 // Records the encryption scheme used by the stream |stream_name|. This is
694 // only recorded when metadata is available.
695 void RecordEncryptionScheme(const std::string& stream_name,
696 media::EncryptionScheme encryption_scheme);
698 // Returns whether the player is currently displayed in Picture-in-Picture.
699 // It will return true even if the player is in AutoPIP mode.
700 // The player MUST have a `client_` when this call happen.
701 bool IsInPictureInPicture() const;
703 // Sets the UKM container name if needed.
704 void MaybeSetContainerNameForMetrics();
706 // Switch to SurfaceLayer, either initially or from VideoLayer.
707 void ActivateSurfaceLayerForVideo();
709 // Called by |compositor_| upon presenting a frame, after
710 // RequestAnimationFrame() is called.
711 void OnNewFramePresentedCallback();
713 // Notifies |demuxer_manager_| of playback and rate changes which may increase
714 // the amount of data the DataSource buffers. Does nothing prior to reaching
715 // kReadyStateHaveEnoughData for the first time.
716 void MaybeUpdateBufferSizesForPlayback();
718 // Create / recreate |smoothness_helper_|, with current features. Will take
719 // no action if we already have a smoothness helper with the same features
720 // that we want now. Will destroy the helper if we shouldn't be measuring
721 // smoothness right now.
722 void UpdateSmoothnessHelper();
724 // Get the LearningTaskController for |task_name|.
725 std::unique_ptr<media::learning::LearningTaskController>
726 GetLearningTaskController(const char* task_name);
728 // Returns whether the player has an audio track and whether it should be
729 // allowed to play it.
730 bool HasUnmutedAudio() const;
732 // Returns true if the video frame from this player are being captured.
733 bool IsVideoBeingCaptured() const;
735 #if defined(TIZEN_VIDEO_HOLE)
736 bool ShouldUseVideoHole() const;
737 void CreateVideoHoleFrame();
738 void OnDrawableContentRectChanged(gfx::Rect rect, bool is_video);
739 bool UpdateBoundaryRectangle();
740 void StartLayerBoundUpdateTimer();
741 void StopLayerBoundUpdateTimer();
742 void OnLayerBoundUpdateTimerFired();
745 // Report UMAs when this object instance is destroyed.
746 void ReportSessionUMAs() const;
748 absl::optional<media::DemuxerType> GetDemuxerType() const;
750 // Useful to bind for a cb to be called when a demuxer is created.
751 media::PipelineStatus OnDemuxerCreated(media::Demuxer* demuxer,
752 media::Pipeline::StartType start_type,
756 // Notifies the `client_` and the `delegate_` about metadata change.
757 void DidMediaMetadataChange();
759 const raw_ptr<WebLocalFrame, ExperimentalRenderer> frame_;
761 WebMediaPlayer::NetworkState network_state_ =
762 WebMediaPlayer::kNetworkStateEmpty;
763 WebMediaPlayer::ReadyState ready_state_ =
764 WebMediaPlayer::kReadyStateHaveNothing;
765 WebMediaPlayer::ReadyState highest_ready_state_ =
766 WebMediaPlayer::kReadyStateHaveNothing;
768 // Preload state for when a DataSource is created after setPreload().
769 media::DataSource::Preload preload_ = media::DataSource::METADATA;
771 // Poster state (for UMA reporting).
772 bool has_poster_ = false;
774 // Task runner for posting tasks on Chrome's main thread. Also used
775 // for DCHECKs so methods calls won't execute in the wrong thread.
776 const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
777 const scoped_refptr<base::SequencedTaskRunner> media_task_runner_;
778 const scoped_refptr<base::TaskRunner> worker_task_runner_;
780 // This is the ID that is used within the internals of the media element
781 // primarily for correlating logs.
782 const media::MediaPlayerLoggingID media_player_id_;
784 std::unique_ptr<media::MediaLog> media_log_;
786 // |pipeline_controller_| owns an instance of Pipeline.
787 std::unique_ptr<media::PipelineController> pipeline_controller_;
789 // The LoadType passed in the |load_type| parameter of the load() call.
790 LoadType load_type_ = kLoadTypeURL;
792 // Cache of metadata for answering hasAudio(), hasVideo(), and naturalSize().
793 media::PipelineMetadata pipeline_metadata_;
795 // Whether the video is known to be opaque or not.
796 bool opaque_ = false;
800 // TODO(scherkus): we have these because Pipeline favours the simplicity of a
801 // single "playback rate" over worrying about paused/stopped etc... It forces
802 // all clients to manage the pause+playback rate externally, but is that
803 // really a bad thing?
805 // TODO(scherkus): since SetPlaybackRate(0) is asynchronous and we don't want
806 // to hang the render thread during pause(), we record the time at the same
807 // time we pause and then return that value in currentTime(). Otherwise our
808 // clock can creep forward a little bit while the asynchronous
809 // SetPlaybackRate(0) is being executed.
810 double playback_rate_ = 0.0;
812 // Counter that limits spam to |media_log_| of |playback_rate_| changes.
813 int num_playback_rate_logs_ = 0;
815 // Set while paused. |paused_time_| is only valid when |paused_| is true.
817 base::TimeDelta paused_time_;
819 // Set if paused automatically when hidden and need to resume when visible.
820 // Reset if paused for any other reason.
821 bool paused_when_hidden_ = false;
823 // Set when starting, seeking, and resuming (all of which require a Pipeline
824 // seek). |seek_time_| is only valid when |seeking_| is true.
825 bool seeking_ = false;
826 base::TimeDelta seek_time_;
828 // Set when doing a restart (a suspend and resume in sequence) of the pipeline
829 // in order to destruct and reinitialize the decoders. This is separate from
830 // |pending_resume_| and |pending_suspend_| because they can be elided in
831 // certain cases, whereas for a restart they must happen.
832 // TODO(sandersd,watk): Create a simpler interface for a pipeline restart.
833 bool pending_suspend_resume_cycle_ = false;
835 // TODO(scherkus): Replace with an explicit ended signal to HTMLMediaElement,
836 // see http://crbug.com/409280
839 // Tracks whether to issue time changed notifications during buffering state
841 bool should_notify_time_changed_ = false;
843 bool overlay_enabled_ = false;
845 // Cors and Caching flags set during `Load` and used while creating demuxers.
846 CorsMode cors_mode_ = kCorsModeUnspecified;
847 bool is_cache_disabled_ = false;
849 // Whether the current decoder requires a restart on overlay transitions.
850 bool decoder_requires_restart_for_overlay_ = false;
852 const raw_ptr<WebMediaPlayerClient, ExperimentalRenderer> client_;
853 const raw_ptr<WebMediaPlayerEncryptedMediaClient, ExperimentalRenderer>
856 // WebMediaPlayer notifies the |delegate_| of playback state changes using
857 // |delegate_id_|; an id provided after registering with the delegate. The
858 // WebMediaPlayer may also receive directives (play, pause) from the delegate
859 // via the WebMediaPlayerDelegate::Observer interface after registration.
861 // NOTE: HTMLMediaElement is a ExecutionContextLifecycleObserver, and
862 // will receive a call to contextDestroyed() when Document::shutdown()
863 // is called. Document::shutdown() is called before the frame detaches (and
864 // before the frame is destroyed). RenderFrameImpl owns |delegate_| and is
865 // guaranteed to outlive |this|; thus it is safe to store |delegate_| as a raw
867 raw_ptr<WebMediaPlayerDelegate, ExperimentalRenderer> delegate_;
868 int delegate_id_ = 0;
870 // The playback state last reported to |delegate_|, to avoid setting duplicate
872 // TODO(sandersd): The delegate should be implementing deduplication.
873 DelegateState delegate_state_ = DelegateState::GONE;
874 bool delegate_has_audio_ = false;
876 WebMediaPlayerBuilder::DeferLoadCB defer_load_cb_;
878 // Members for notifying upstream clients about internal memory usage. The
879 // |adjust_allocated_memory_cb_| must only be called on |main_task_runner_|.
880 base::RepeatingTimer memory_usage_reporting_timer_;
881 WebMediaPlayerBuilder::AdjustAllocatedMemoryCB adjust_allocated_memory_cb_;
882 int64_t last_reported_memory_usage_ = 0;
883 std::unique_ptr<media::MemoryDumpProviderProxy> main_thread_mem_dumper_;
884 std::unique_ptr<media::MemoryDumpProviderProxy> media_thread_mem_dumper_;
886 // Routes audio playback to either AudioRendererSink or WebAudio.
887 scoped_refptr<WebAudioSourceProviderImpl> audio_source_provider_;
889 // Manages the lifetime of the DataSource, and soon the Demuxer.
890 std::unique_ptr<media::DemuxerManager> demuxer_manager_;
892 raw_ptr<const base::TickClock, ExperimentalRenderer> tick_clock_ = nullptr;
894 std::unique_ptr<BufferedDataSourceHostImpl> buffered_data_source_host_;
895 const raw_ptr<UrlIndex, ExperimentalRenderer> url_index_;
896 scoped_refptr<viz::RasterContextProvider> raster_context_provider_;
898 // Video rendering members.
899 // The |compositor_| runs on the compositor thread, or if
900 // kEnableSurfaceLayerForVideo is enabled, the media thread. This task runner
901 // posts tasks for the |compositor_| on the correct thread.
902 scoped_refptr<base::SingleThreadTaskRunner> vfc_task_runner_;
903 std::unique_ptr<VideoFrameCompositor>
904 compositor_; // Deleted on |vfc_task_runner_|.
905 media::PaintCanvasVideoRenderer video_renderer_;
907 // The compositor layer for displaying the video content when using composited
909 scoped_refptr<cc::VideoLayer> video_layer_;
911 std::unique_ptr<WebContentDecryptionModuleResult> set_cdm_result_;
913 // If a CdmContext is attached keep a reference to the CdmContextRef, so that
914 // it is not destroyed until after the pipeline is done with it.
915 std::unique_ptr<media::CdmContextRef> cdm_context_ref_;
917 // Keep track of the CdmContextRef while it is in the process of attaching to
919 std::unique_ptr<media::CdmContextRef> pending_cdm_context_ref_;
921 // True when encryption is detected, either by demuxer or by presence of a
922 // ContentDecyprtionModule (CDM).
923 bool is_encrypted_ = false;
925 // Captured once the cdm is provided to SetCdmInternal(). Used in creation of
926 // |video_decode_stats_reporter_|.
927 absl::optional<media::CdmConfig> cdm_config_;
929 // Tracks if we are currently flinging a video (e.g. in a RemotePlayback
930 // session). Used to prevent videos from being paused when hidden.
931 // TODO(https://crbug.com/839651): remove or rename this flag, when removing
933 bool is_flinging_ = false;
935 // Tracks if we are currently using a remote renderer. See
936 // SwitchToRemoteRenderer().
937 bool is_remote_rendering_ = false;
939 // The last volume received by setVolume() and the last volume multiplier from
940 // SetVolumeMultiplier(). The multiplier is typical 1.0, but may be less
941 // if the WebMediaPlayerDelegate has requested a volume reduction (ducking)
942 // for a transient sound. Playout volume is derived by volume * multiplier.
943 double volume_ = 1.0;
944 double volume_multiplier_ = 1.0;
946 std::unique_ptr<media::RendererFactorySelector> renderer_factory_selector_;
948 // For canceling AndroidOverlay routing token requests.
949 base::CancelableOnceCallback<void(const base::UnguessableToken&)>
952 // If overlay info is requested before we have it, then the request is saved
953 // and satisfied once the overlay info is available. If the decoder does not
954 // require restart to change surfaces, this is callback is kept until cleared
956 media::ProvideOverlayInfoCB provide_overlay_info_cb_;
958 // On Android an overlay surface means using
959 // SurfaceView instead of SurfaceTexture.
961 // Allow overlays for all video on android.
962 bool always_enable_overlays_ = false;
964 // Suppresses calls to OnPipelineError() after destruction / shutdown has been
965 // started; prevents us from spuriously logging errors that are transient or
967 bool suppress_destruction_errors_ = false;
969 // NOTE: |using_media_player_renderer_| is set based on the usage of a
970 // MediaResource::Type::URL in StartPipeline(). This works because
971 // MediaPlayerRendererClientFactory is the only factory that uses
972 // MediaResource::Type::URL for now.
973 bool using_media_player_renderer_ = false;
975 #if BUILDFLAG(IS_ANDROID)
976 // Set during the initial DoLoad() call. Used to determine whether to allow
977 // credentials or not for MediaPlayerRenderer.
978 bool allow_media_player_renderer_credentials_ = false;
981 // Stores the current position state of the media.
982 media_session::MediaPosition media_position_state_;
984 // Called sometime after the media is suspended in a playing state in
985 // OnFrameHidden(), causing the state to change to paused.
986 base::OneShotTimer background_pause_timer_;
988 // Monitors the watch time of the played content.
989 std::unique_ptr<WatchTimeReporter> watch_time_reporter_;
990 media::AudioDecoderType audio_decoder_type_ =
991 media::AudioDecoderType::kUnknown;
992 media::VideoDecoderType video_decoder_type_ =
993 media::VideoDecoderType::kUnknown;
995 // The time at which DoLoad() is executed.
996 base::TimeTicks load_start_time_;
998 // Time elapsed time from |load_start_time_| to OnMetadata(). Used to later
999 // adjust |load_start_time_| if a suspended startup occurred.
1000 base::TimeDelta time_to_metadata_;
1001 bool skip_metrics_due_to_startup_suspend_ = false;
1003 bool have_reported_time_to_play_ready_ = false;
1005 // Records pipeline statistics for describing media capabilities.
1006 std::unique_ptr<VideoDecodeStatsReporter> video_decode_stats_reporter_;
1008 // Elapsed time since we've last reached BUFFERING_HAVE_NOTHING.
1009 std::unique_ptr<base::ElapsedTimer> underflow_timer_;
1011 // Used to track loading progress, used by IsPrerollAttemptNeeded().
1012 // |preroll_attempt_pending_| indicates that the clock has been reset
1013 // (awaiting a resume to start), while |preroll_attempt_start_time_| tracks
1014 // when a preroll attempt began.
1015 bool preroll_attempt_pending_ = false;
1016 base::TimeTicks preroll_attempt_start_time_;
1018 // Monitors the player events.
1019 base::WeakPtr<media::MediaObserver> observer_;
1021 // Owns the weblayer and obtains/maintains SurfaceIds.
1022 std::unique_ptr<WebSurfaceLayerBridge> bridge_;
1024 // The maximum video keyframe distance that allows triggering background
1025 // playback optimizations (non-MSE).
1026 base::TimeDelta max_keyframe_distance_to_disable_background_video_;
1028 // The maximum video keyframe distance that allows triggering background
1029 // playback optimizations (MSE).
1030 base::TimeDelta max_keyframe_distance_to_disable_background_video_mse_;
1032 // Whether disabled the video track as an optimization.
1033 bool video_track_disabled_ = false;
1035 // Whether the pipeline is being resumed at the moment.
1036 bool is_pipeline_resuming_ = false;
1038 // When this is true, pipeline will not be auto suspended.
1039 bool disable_pipeline_auto_suspend_ = false;
1041 // Pipeline statistics overridden by tests.
1042 absl::optional<media::PipelineStatistics> pipeline_statistics_for_test_;
1044 // Pipeline media duration overridden by tests.
1045 absl::optional<base::TimeDelta> pipeline_media_duration_for_test_;
1047 // Whether the video requires a user gesture to resume after it was paused in
1048 // the background. Affects the value of ShouldPausePlaybackWhenHidden().
1049 bool video_locked_when_paused_when_hidden_ = false;
1051 // Whether embedded media experience is currently enabled.
1052 bool embedded_media_experience_enabled_ = false;
1054 // When should we use SurfaceLayer for video?
1055 bool use_surface_layer_ = false;
1057 // Whether surface layer is currently in use to display frames.
1058 bool surface_layer_for_video_enabled_ = false;
1060 CreateSurfaceLayerBridgeCB create_bridge_callback_;
1062 bool initial_video_height_recorded_ = false;
1064 enum class OverlayMode {
1065 // All overlays are turned off.
1068 // Use AndroidOverlay for overlays.
1072 OverlayMode overlay_mode_ = OverlayMode::kNoOverlays;
1074 // Optional callback to request the routing token for AndroidOverlay.
1075 media::RequestRoutingTokenCallback request_routing_token_cb_;
1077 // If |overlay_routing_token_is_pending_| is false, then
1078 // |overlay_routing_token_| contains the routing token we should send, if any.
1079 // Otherwise, |overlay_routing_token_| is undefined. We set the flag while
1080 // we have a request for the token that hasn't been answered yet; i.e., it
1081 // means that we don't know what, if any, token we should be using.
1082 bool overlay_routing_token_is_pending_ = false;
1083 media::OverlayInfo::RoutingToken overlay_routing_token_;
1085 media::OverlayInfo overlay_info_;
1087 base::CancelableOnceClosure update_background_status_cb_;
1089 // We cannot use `update_background_status_cb_.IsCancelled()` as that changes
1090 // when the callback is run, even if not explicitly cancelled. This is
1091 // initialized to true to keep in line with the existing behavior of
1092 // base::CancellableOnceClosure.
1093 bool is_background_status_change_cancelled_ = true;
1095 mojo::Remote<media::mojom::MediaMetricsProvider> media_metrics_provider_;
1096 mojo::Remote<media::mojom::PlaybackEventsRecorder> playback_events_recorder_;
1098 absl::optional<ReadyState> stale_state_override_for_testing_;
1100 // True if we attempt to start the media pipeline in a suspended state for
1101 // preload=metadata. Cleared upon pipeline startup.
1102 bool attempting_suspended_start_ = false;
1104 // True if a frame has ever been rendered.
1105 bool has_first_frame_ = false;
1107 // True if we have not yet rendered a first frame, but one is needed. Set back
1108 // to false as soon as |has_first_frame_| is set to true.
1109 bool needs_first_frame_ = false;
1111 // Whether the rendered frame is readable, e.g. can be converted to image.
1112 bool is_frame_readable_ = false;
1114 // True if StartPipeline() completed a lazy load startup.
1115 bool did_lazy_load_ = false;
1117 // Whether the renderer should automatically suspend media playback in
1119 bool is_background_suspend_enabled_ = false;
1121 // If disabled, video will be auto paused when in background. Affects the
1122 // value of ShouldPausePlaybackWhenHidden().
1123 bool is_background_video_playback_enabled_ = true;
1125 // Whether background video optimization is supported on current platform.
1126 bool is_background_video_track_optimization_supported_ = true;
1128 const bool should_pause_background_muted_audio_;
1130 bool was_suspended_for_frame_closed_ = false;
1132 #if defined(TIZEN_MULTIMEDIA)
1133 bool was_suspended_by_player_ = false;
1135 // used for live streams.
1136 bool is_live_stream_ = false;
1137 base::TimeDelta min_seekable_time_;
1138 base::TimeDelta max_seekable_time_;
1141 // Request pipeline to suspend. It should not block other signals after
1143 bool pending_oneshot_suspend_ = false;
1145 base::CancelableOnceClosure have_enough_after_lazy_load_cb_;
1147 media::RendererType renderer_type_ = media::RendererType::kRendererImpl;
1148 media::SimpleWatchTimer simple_watch_timer_;
1150 LearningExperimentHelper will_play_helper_;
1152 #if defined(TIZEN_VIDEO_HOLE)
1153 bool is_video_hole_;
1154 gfx::RectF last_computed_rect_;
1155 base::RepeatingTimer layer_bound_update_timer_;
1158 std::unique_ptr<PowerStatusHelper> power_status_helper_;
1160 // Created while playing, deleted otherwise.
1161 std::unique_ptr<SmoothnessHelper> smoothness_helper_;
1162 absl::optional<int> last_reported_fps_;
1164 // Time of the last call to GetCurrentFrameFromCompositor(). Used to prevent
1165 // background optimizations from being applied when capturing is active.
1166 base::TimeTicks last_frame_request_time_;
1168 // Count the number of times a video frame is being readback.
1169 unsigned video_frame_readback_count_ = 0;
1171 base::WeakPtr<WebMediaPlayerImpl> weak_this_;
1172 base::WeakPtrFactory<WebMediaPlayerImpl> weak_factory_{this};
1175 } // namespace blink
1177 #endif // THIRD_PARTY_BLINK_RENDERER_PLATFORM_MEDIA_WEB_MEDIA_PLAYER_IMPL_H_