review.tizen.org Git - platform/framework/web/chromium-efl.git/blob

   1 // Copyright (c) 2020 Samsung Electronics. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/tizen/elementary_media_stream_source/control_thread/source_impl.h"
   6
   7 #include <algorithm>
   8 #include <limits>
   9 #include <string>
  10 #include <utility>
  11
  12 #include "absl/types/optional.h"
  13 #include "content/renderer/media/tizen/elementary_media_stream_source/any_thread/common_types.h"
  14 #include "content/renderer/media/tizen/elementary_media_stream_source/any_thread/source_dispatcher.h"
  15 #include "content/renderer/media/tizen/elementary_media_stream_source/any_thread/source_dispatcher_client.h"
  16 #include "content/renderer/media/tizen/elementary_media_stream_source/control_thread/demuxer.h"
  17 #include "content/renderer/media/tizen/elementary_media_stream_source/control_thread/source_client.h"
  18 #include "content/renderer/media/tizen/elementary_media_stream_source/worker_thread/source_dispatcher_client.h"
  19 #include "media/base/audio_decoder_config.h"
  20 #include "media/base/timestamp_constants.h"
  21 #include "media/base/video_decoder_config.h"
  22 #include "services/elementary_media_stream_source/public/cpp/logger.h"
  23
  24 namespace content {
  25
  26 namespace elementary_media_stream_source {
  27
  28 namespace control_thread {
  29
  30 SourceImpl::SourceImpl(any_thread::SourceParams params)
  31     : control_client_(nullptr),
  32       dispatcher_(std::move(params.dispatcher)),
  33       is_open_delayed_(false),
  34       is_resuming_(false),
  35       is_seeking_(false),
  36       pipeline_mode_(params.pipeline_mode),
  37       playback_position_(0),
  38       player_state_(PlayerState::kClosed),
  39       seek_video_keyframe_timestamp_(media::kNoTimestamp),
  40       state_(std::move(params.state)),
  41       input_session_id_(kInitialSessionId),
  42       output_session_id_(kInitialSessionId) {
  43   EMSS_DEBUG() << "Constructing object";
  44 }
  45
  46 SourceImpl::~SourceImpl() {
  47   EMSS_DEBUG() << "Destructing object";
  48 }
  49
  50 //////////////////// DemuxerClient ////////////////////
  51
  52 void SourceImpl::OnClosedCaptions(const std::vector<uint8_t>& closed_captions) {
  53   EMSS_DEBUG();
  54
  55   if (control_client_)
  56     control_client_->OnClosedCaptions(closed_captions);
  57 }
  58
  59 void SourceImpl::OnPlayerError(BackendError error, const std::string& message) {
  60   EMSS_DEBUG() << error << ", message = " << message;
  61
  62   const auto new_session_id =
  63       IsNextInputSessionIdRequired()
  64           ? absl::make_optional(GenerateNextInputSessionId())
  65           : absl::nullopt;
  66
  67   for (const auto& client_weak_ptr : clients_) {
  68     auto client = client_weak_ptr.lock();
  69
  70     if (!client)
  71       continue;
  72
  73     client->OnPlayerError(new_session_id);
  74   }
  75
  76   OnBufferedRangesChanged();
  77
  78   SetPlayerState(PlayerState::kClosed);
  79
  80   if (control_client_)
  81     control_client_->OnPlayerError(error, message);
  82 }
  83
  84 void SourceImpl::OnResumeComplete() {
  85   EMSS_DEBUG();
  86
  87   is_resuming_ = false;
  88 }
  89
  90 void SourceImpl::RegisterDemuxer(std::shared_ptr<Demuxer> demuxer) {
  91   EMSS_DEBUG();
  92
  93   demuxer_ = demuxer;
  94 }
  95
  96 void SourceImpl::Seek(const base::TimeDelta& seek_time) {
  97   EMSS_DEBUG() << "seek time: " << seek_time;
  98
  99   EMSS_LOG_ASSERT(pipeline_mode_.latency_mode == LatencyMode::kNormal)
 100       << "Cannot seek in low latency mode";
 101
 102   seek_video_keyframe_timestamp_ = media::kNoTimestamp;
 103
 104   SetTracksState(TrackState::kOpen, ChangeReason::kOpen);
 105
 106   if (player_state_ != PlayerState::kOpenedPending)
 107     return;
 108
 109   OnBufferedRangesChanged();
 110   SetPlayerState(PlayerState::kOpened);
 111 }
 112
 113 void SourceImpl::SetIsSeeking(bool is_seeking) {
 114   EMSS_DEBUG() << "is_seeking: " << (is_seeking ? "true" : "false");
 115
 116   is_seeking_ = is_seeking;
 117 }
 118
 119 void SourceImpl::StartWaitingForSeek(const base::TimeDelta& seek_time) {
 120   EMSS_DEBUG() << "seek time: " << seek_time;
 121
 122   EMSS_LOG_ASSERT(pipeline_mode_.latency_mode == LatencyMode::kNormal)
 123       << "Cannot StartWaitingForSeek in low latency mode";
 124
 125   SetPlayerState(PlayerState::kOpenedPending);
 126
 127   const auto new_session_id =
 128       IsNextInputSessionIdRequired()
 129           ? absl::make_optional(GenerateNextInputSessionId())
 130           : absl::nullopt;
 131
 132   for (const auto& client_weak_ptr : clients_) {
 133     auto client = client_weak_ptr.lock();
 134
 135     if (!client)
 136       continue;
 137
 138     client->RequestSeek(seek_time, new_session_id);
 139   }
 140 }
 141
 142 void SourceImpl::Stop() {
 143   EMSS_DEBUG();
 144
 145   is_open_delayed_ = false;
 146
 147   SetPlayerState(PlayerState::kDetached);
 148   SetTracksState(TrackState::kClosed, ChangeReason::kCloseSourceDetached);
 149
 150   OnFlushRequested();
 151
 152   for (const auto& client_weak_ptr : clients_) {
 153     auto client = client_weak_ptr.lock();
 154
 155     if (!client)
 156       continue;
 157
 158     client->Stop();
 159   }
 160
 161   if (auto demuxer = demuxer_.lock())
 162     demuxer->OnPipelineClosed();
 163
 164   if (control_client_)
 165     control_client_->Stop();
 166 }
 167
 168 //////////////////// Source ////////////////////
 169
 170 void SourceImpl::CloseTracksIfAllReachedEOS() {
 171   EMSS_DEBUG();
 172
 173   if (std::any_of(clients_.begin(), clients_.end(),
 174                   [](const std::weak_ptr<SourceClient>& weak_ptr) {
 175                     if (auto client = weak_ptr.lock())
 176                       return !client->IsFullyClosable();
 177                     return false;
 178                   })) {
 179     EMSS_VERBOSE();
 180     return;
 181   }
 182
 183   SetTracksState(TrackState::kClosed, ChangeReason::kCloseTrackEnded);
 184 }
 185
 186 void SourceImpl::OnBufferedRangesChanged() const {
 187   EMSS_DEBUG();
 188
 189   if (pipeline_mode_.latency_mode != LatencyMode::kNormal) {
 190     // In LL ranges never change and are always (0, +inf) set during setup.
 191     EMSS_DEBUG();
 192     return;
 193   }
 194
 195   if (auto dispatcher = dispatcher_.lock()) {
 196     dispatcher->DispatchTask(
 197         &worker_thread::SourceDispatcherClient::BufferedRangesChanged);
 198   }
 199 }
 200
 201 void SourceImpl::OnSeekTrackCompleted(
 202     TrackType type,
 203     base::TimeDelta first_keyframe_timestamp) {
 204   EMSS_DEBUG() << "<" << type
 205                << "> completed seek to: " << first_keyframe_timestamp;
 206
 207   switch (type) {
 208     case TrackType::kAudio:
 209       break;
 210     case TrackType::kVideo:
 211       seek_video_keyframe_timestamp_ = first_keyframe_timestamp;
 212       break;
 213     default:
 214       EMSS_LOG_ASSERT(false) << "Unsupported DemuxerStream type = " << type;
 215       return;
 216   }
 217
 218   if (std::any_of(clients_.begin(), clients_.end(),
 219                   [](const std::weak_ptr<SourceClient>& weak_ptr) {
 220                     if (auto client = weak_ptr.lock())
 221                       return client->IsSeeking();
 222                     return false;
 223                   }))
 224     return;
 225
 226   OnBufferedRangesChanged();
 227
 228   // Use video keyframe timestamp for seeking if available.
 229   // If not, use audio keyframe.
 230   auto keyframe_timestamp =
 231       seek_video_keyframe_timestamp_ != media::kNoTimestamp
 232           ? seek_video_keyframe_timestamp_
 233           : first_keyframe_timestamp;
 234
 235   OnPlaybackPositionChanged(keyframe_timestamp.InSecondsF(),
 236                             output_session_id_);
 237
 238   if (auto demuxer = demuxer_.lock())
 239     demuxer->OnDemuxerSeekDone(keyframe_timestamp, output_session_id_);
 240 }
 241
 242 void SourceImpl::OnTrackEnded() {
 243   EMSS_DEBUG();
 244
 245   if (player_state_ != PlayerState::kOpened) {
 246     EMSS_DEBUG();
 247     return;
 248   }
 249
 250   if (std::all_of(clients_.begin(), clients_.end(),
 251                   [](const std::weak_ptr<SourceClient>& weak_ptr) {
 252                     if (auto client = weak_ptr.lock())
 253                       return client->IsEnded() && !client->IsOpen();
 254                     return true;
 255                   })) {
 256     SetPlayerState(PlayerState::kEnded);
 257   }
 258 }
 259
 260 //////////////////// SourceDispatcherClient ////////////////////
 261
 262 void SourceImpl::RegisterClient(std::shared_ptr<SourceClient> client) {
 263   EMSS_DEBUG() << "<" << client->Type() << ">";
 264
 265   clients_[+client->Type()] = client;
 266   client->RegisterSource(std::static_pointer_cast<Source>(shared_from_this()));
 267 }
 268
 269 ////////// blink::WebElementaryMediaStreamSourceControl //////////
 270
 271 media::Ranges<base::TimeDelta> SourceImpl::GetBufferedRanges() const {
 272   return state_->GetLockedState()->buffered_ranges;
 273 }
 274
 275 void SourceImpl::OnFlushRequested() {
 276   EMSS_DEBUG();
 277
 278   for (const auto& client_weak_ptr : clients_) {
 279     auto client = client_weak_ptr.lock();
 280
 281     if (!client)
 282       continue;
 283
 284     client->OnFlushRequested();
 285   }
 286
 287   OnBufferedRangesChanged();
 288 }
 289
 290 void SourceImpl::OnPause() {
 291   EMSS_DEBUG();
 292
 293   if (pipeline_mode_.latency_mode == LatencyMode::kNormal) {
 294     EMSS_DEBUG();
 295     return;
 296   }
 297
 298   is_open_delayed_ = false;
 299
 300   if (player_state_ != PlayerState::kOpened) {
 301     EMSS_DEBUG();
 302     return;
 303   }
 304
 305   SetPlayerState(PlayerState::kOpenedPending);
 306   SetTracksState(TrackState::kClosed, ChangeReason::kCloseSourceClosed);
 307 }
 308
 309 void SourceImpl::OnPlay() {
 310   EMSS_DEBUG();
 311
 312   if (pipeline_mode_.latency_mode == LatencyMode::kNormal) {
 313     EMSS_DEBUG();
 314     return;
 315   }
 316
 317   if (player_state_ == PlayerState::kDetached ||
 318       player_state_ == PlayerState::kClosed) {
 319     EMSS_DEBUG() << "Parent media element is already playing, delaying "
 320                  << PlayerState::kOpened << " state until EMSS is set up";
 321     is_open_delayed_ = true;
 322     return;
 323   }
 324
 325   if (player_state_ != PlayerState::kOpenedPending) {
 326     EMSS_DEBUG();
 327     return;
 328   }
 329
 330   SetTracksState(TrackState::kOpen, ChangeReason::kOpen);
 331   SetPlayerState(PlayerState::kOpened);
 332 }
 333
 334 void SourceImpl::OnResume() {
 335   EMSS_DEBUG();
 336
 337   is_resuming_ = true;
 338
 339   if (auto demuxer = demuxer_.lock())
 340     demuxer->OnPipelineResuming();
 341 }
 342
 343 void SourceImpl::OnSuspend() {
 344   EMSS_DEBUG();
 345
 346   if (auto demuxer = demuxer_.lock())
 347     demuxer->OnPipelineSuspended();
 348
 349   if (player_state_ != PlayerState::kOpened) {
 350     EMSS_DEBUG();
 351     return;
 352   }
 353
 354   // Note on handling multitasking: explicit reaction to Suspend is required in
 355   // order to properly close source & tracks with kCloseSourceSuspended reason.
 356   // On the other hand, Resume doesn't need to be handled, because:
 357   // - in normal latency, player would be opened by a seek after App is resumed,
 358   // - in low latency, player would be opened by the unpausing HTMLMediaElement.
 359
 360   SetPlayerState(PlayerState::kOpenedPending);
 361   SetTracksState(TrackState::kClosed, ChangeReason::kCloseSourceSuspended);
 362 }
 363
 364 void SourceImpl::UpdatePlaybackPosition(double playback_position,
 365                                         uint32_t session_id) {
 366   EMSS_VERBOSE() << "Reported playback position: " << playback_position;
 367
 368   if (is_resuming_) {
 369     EMSS_VERBOSE() << "Resuming. Ignoring playback_position update";
 370     return;
 371   }
 372
 373   if (is_seeking_) {
 374     EMSS_VERBOSE() << "Processing seek. Ignoring playback_position update";
 375     return;
 376   }
 377
 378   if (session_id != output_session_id_) {
 379     EMSS_VERBOSE() << "Dropping time update = " << playback_position
 380                    << "s because session id changed, update id = " << session_id
 381                    << ", current = " << output_session_id_;
 382     return;
 383   }
 384
 385   if (playback_position_ != playback_position)
 386     OnPlaybackPositionChanged(playback_position, session_id);
 387 }
 388
 389 void SourceImpl::RegisterClient(
 390     WebElementaryMediaStreamSourceControlClient* control_client) {
 391   EMSS_DEBUG();
 392
 393   control_client_ = control_client;
 394 }
 395
 396 void SourceImpl::SetDuration(double duration) {
 397   EMSS_DEBUG();
 398
 399   if (duration < 0) {
 400     EMSS_LOG(ERROR) << "Cannot set negative duration.";
 401     return;
 402   }
 403
 404   if (pipeline_mode_.latency_mode != LatencyMode::kNormal) {
 405     EMSS_LOG_ASSERT(duration == std::numeric_limits<double>::infinity())
 406         << "Cannot set duration in low latency mode to other value "
 407            "than infinity.";
 408   }
 409
 410   if (auto demuxer = demuxer_.lock()) {
 411     demuxer->SetDuration(
 412         base::Milliseconds(base::Seconds(duration).InMillisecondsF()));
 413   }
 414
 415   state_->GetLockedState()->duration_ = duration;
 416 }
 417
 418 CallbackResult SourceImpl::SetSourceClosed() {
 419   EMSS_DEBUG();
 420
 421   SetPlayerState(PlayerState::kClosed);
 422   auto result =
 423       SetTracksState(TrackState::kClosed, ChangeReason::kCloseSourceClosed);
 424
 425   OnFlushRequested();
 426
 427   if (auto demuxer = demuxer_.lock())
 428     demuxer->OnPipelineClosed();
 429
 430   return result;
 431 }
 432
 433 CallbackResult SourceImpl::SetSourceOpened() {
 434   EMSS_DEBUG();
 435
 436   NotifyDemuxerInitialized();
 437
 438   SetPlayerState(PlayerState::kOpenedPending);
 439
 440   auto result = CallbackResult::kSuccess;
 441
 442   const auto is_open_delayed = is_open_delayed_;
 443   if (is_open_delayed) {
 444     EMSS_DEBUG() << "Immediately entering delayed " << PlayerState::kOpened
 445                  << " state";
 446     is_open_delayed_ = false;
 447   }
 448
 449   // In LowLatency track will open later when pipeline starts playing, so
 450   // continue with OnOpenedDone immediately, unless `is_open_delayed` is set
 451   // which means pipeline *is* playing already (i.e. HTMLMediaElement wants to
 452   // play ASAP).
 453   if (pipeline_mode_.latency_mode == LatencyMode::kNormal || is_open_delayed) {
 454     // * In NormalLatency packets can be appended as soon as track is
 455     //   configured, so open tracks before continuing with OnOpenedDone.
 456     // * It's possible pipeline is already in playing state, so just open EMSS
 457     //   in LowLatency if `is_open_delayed`.
 458     result = SetTracksState(TrackState::kOpen, ChangeReason::kOpen);
 459   }
 460
 461   if (result != CallbackResult::kSuccess) {
 462     SetPlayerState(PlayerState::kClosed);
 463   } else if (pipeline_mode_.latency_mode == LatencyMode::kNormal ||
 464              is_open_delayed) {
 465     // In low latency source will open upon Play (if `is_open_delayed` play
 466     // was called already).
 467     SetPlayerState(PlayerState::kOpened);
 468   }
 469
 470   return result;
 471 }
 472
 473 //////////////////// private ////////////////////
 474
 475 uint32_t SourceImpl::GenerateNextInputSessionId() {
 476   ++input_session_id_;
 477   EMSS_DEBUG() << "New input session id = " << input_session_id_;
 478   return input_session_id_;
 479 }
 480
 481 bool SourceImpl::IsNextInputSessionIdRequired() const {
 482   EMSS_VERBOSE();
 483
 484   // If tracks are not fully closed, we do require a new session id. Otherwise
 485   // current session id is new enough (it was set when tracks were closing and
 486   // it should not be changed again).
 487   return std::all_of(clients_.begin(), clients_.end(),
 488                      [](const std::weak_ptr<SourceClient>& weak_ptr) {
 489                        if (auto client = weak_ptr.lock())
 490                          return !client->IsFullyClosed();
 491                        return true;
 492                      });
 493 }
 494
 495 void SourceImpl::NotifyDemuxerInitialized() {
 496   EMSS_DEBUG();
 497
 498   auto audio_config = ([&]() -> MaybeAudioConfig {
 499     if (auto client = clients_[+TrackType::kAudio].lock()) {
 500       return client->GetAudioConfig();
 501     }
 502     return std::nullopt;
 503   })();
 504
 505   auto video_config = ([&]() -> MaybeVideoConfig {
 506     if (auto client = clients_[+TrackType::kVideo].lock()) {
 507       return client->GetVideoConfig();
 508     }
 509     return std::nullopt;
 510   })();
 511
 512   if (auto dispatcher = dispatcher_.lock()) {
 513     dispatcher->RunTask(
 514         &any_thread::SourceDispatcherClient::OnInitialConfigReady,
 515         std::move(audio_config), std::move(video_config));
 516   }
 517
 518   if (auto demuxer = demuxer_.lock()) {
 519     if (pipeline_mode_.latency_mode != LatencyMode::kNormal) {
 520       SetDuration(std::numeric_limits<double>::infinity());
 521       // In normal latency ranges inform backend player about currently buffered
 522       // timestamp range that was buffered by the backend and is playable. This
 523       // concept doesn't apply in live streaming (i.e. low latency mode): source
 524       // is considered to have infinite duration and infinite amount of
 525       // _unbuffered_ packets available at any time. Using actual track ranges
 526       // (where most recent pts is close to playback position) confuses
 527       // buffering mechanism, causing HTMLMediaElement ready state to jump back
 528       // and forth between have metadata and have enough data states, which may
 529       // result in unexpected behavior (e.g. not resolving Play() promise, media
 530       // element ready state being incorrectly reported, etc).
 531       media::Ranges<base::TimeDelta> live_source_ranges;
 532       live_source_ranges.Add(base::TimeDelta::FromSeconds(0),
 533                              base::TimeDelta::Max());
 534       state_->GetLockedState()->buffered_ranges = live_source_ranges;
 535       demuxer->OnBufferedTimeRangesChanged(std::move(live_source_ranges));
 536     }
 537
 538     demuxer->NotifyDemuxerInitialized();
 539   }
 540 }
 541
 542 void SourceImpl::OnPlaybackPositionChanged(double new_playback_position,
 543                                            uint32_t session_id) {
 544   EMSS_VERBOSE() << "New playback position: " << new_playback_position
 545                  << ", session_id: " << session_id;
 546   playback_position_ = new_playback_position;
 547
 548   if (control_client_)
 549     control_client_->OnPlaybackPositionChanged(playback_position_, session_id);
 550 }
 551
 552 void SourceImpl::SetPlayerState(PlayerState state) {
 553   EMSS_DEBUG() << state;
 554
 555   if (!control_client_) {
 556     EMSS_LOG(WARNING) << "Control client not registered";
 557     return;
 558   }
 559
 560   if (player_state_ == state) {
 561     EMSS_LOG(INFO) << "Requested to re-enter state: " << state << ". Ignoring.";
 562     return;
 563   }
 564
 565   auto previous_state = player_state_;
 566   player_state_ = state;
 567
 568   switch (state) {
 569     case PlayerState::kClosed:
 570       // Can be entered from any state.
 571       control_client_->SetPlayerClosed();
 572       break;
 573     case PlayerState::kDetached:
 574       // Can be entered from any state.
 575       control_client_->SetPlayerDetached();
 576       break;
 577     case PlayerState::kOpened:
 578       EMSS_LOG_ASSERT(previous_state == PlayerState::kOpenedPending)
 579           << "Requested illegal state transition from " << previous_state
 580           << " to " << state;
 581       control_client_->SetPlayerOpened();
 582       break;
 583     case PlayerState::kOpenedPending:
 584       // Can be entered from any state.
 585       control_client_->SetPlayerOpenedPending();
 586       break;
 587     case PlayerState::kEnded:
 588       EMSS_LOG_ASSERT(previous_state == PlayerState::kOpened)
 589           << "Requested illegal state transition from " << previous_state
 590           << " to " << state;
 591       control_client_->SetPlayerEnded();
 592       break;
 593   }
 594 }
 595
 596 CallbackResult SourceImpl::SetTracksState(TrackState track_state,
 597                                           ChangeReason reason) {
 598   EMSS_DEBUG() << track_state;
 599
 600   if (track_state == TrackState::kOpen) {
 601     // Opening tracks always precede opening the source itself. We're opening
 602     // a new session for output, so sync session id now.
 603     SyncOutputSessionId();
 604   }
 605
 606   CallbackResult result = CallbackResult::kSuccess;
 607   const absl::optional<uint32_t> new_session_id =
 608       track_state == TrackState::kClosed && IsNextInputSessionIdRequired()
 609           ? absl::make_optional(GenerateNextInputSessionId())
 610           : absl::nullopt;
 611
 612   for (const auto& client_weak_ptr : clients_) {
 613     auto client = client_weak_ptr.lock();
 614
 615     if (!client)
 616       continue;
 617
 618     result = client->ChangeTrackState(track_state, reason, new_session_id);
 619
 620     if (result != CallbackResult::kSuccess)
 621       break;
 622   }
 623
 624   return result;
 625 }
 626
 627 void SourceImpl::SyncOutputSessionId() {
 628   if (output_session_id_ == input_session_id_) {
 629     EMSS_DEBUG();
 630     return;
 631   }
 632
 633   output_session_id_ = input_session_id_;
 634   EMSS_DEBUG() << "New output session id = " << output_session_id_;
 635   if (control_client_)
 636     control_client_->SetOutputSessionId(output_session_id_);
 637
 638   if (auto demuxer = demuxer_.lock())
 639     demuxer->OnSessionIdChange(output_session_id_);
 640 }
 641
 642 }  // namespace control_thread
 643
 644 }  // namespace elementary_media_stream_source
 645
 646 }  // namespace content