src/content/renderer/media/remote_media_stream_impl.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/remote_media_stream_impl.h"
   6
   7 #include <string>
   8
   9 #include "base/bind.h"
  10 #include "base/location.h"
  11 #include "base/logging.h"
  12 #include "base/strings/utf_string_conversions.h"
  13 #include "base/thread_task_runner_handle.h"
  14 #include "content/renderer/media/media_stream.h"
  15 #include "content/renderer/media/media_stream_track.h"
  16 #include "content/renderer/media/media_stream_video_track.h"
  17 #include "content/renderer/media/webrtc/media_stream_remote_video_source.h"
  18 #include "content/renderer/media/webrtc/peer_connection_dependency_factory.h"
  19 #include "third_party/WebKit/public/platform/WebString.h"
  20
  21 namespace content {
  22 namespace {
  23
  24 template <typename WebRtcTrackVector, typename AdapterType>
  25 void CreateAdaptersForTracks(
  26     const WebRtcTrackVector& tracks,
  27     std::vector<scoped_refptr<AdapterType>>* observers,
  28     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread) {
  29   for (auto& track : tracks)
  30     observers->push_back(new AdapterType(main_thread, track));
  31 }
  32
  33 template<typename VectorType>
  34 bool IsTrackInVector(const VectorType& v, const std::string& id) {
  35   for (const auto& t : v) {
  36     if (t->id() == id)
  37       return true;
  38   }
  39   return false;
  40 }
  41 }  // namespace
  42
  43 // TODO(tommi): Move this class to a separate set of files.
  44 class RemoteMediaStreamAudioTrack : public MediaStreamTrack {
  45  public:
  46   RemoteMediaStreamAudioTrack(
  47       const scoped_refptr<webrtc::AudioTrackInterface>& track,
  48       const scoped_refptr<base::SingleThreadTaskRunner>& signaling_thread)
  49       : MediaStreamTrack(false), track_(track),
  50         signaling_thread_(signaling_thread) {
  51   }
  52
  53   ~RemoteMediaStreamAudioTrack() override {}
  54
  55  private:
  56   void SetEnabled(bool enabled) override {
  57     track_->set_enabled(enabled);
  58   }
  59
  60   void Stop() override {
  61     // Stop means that a track should be stopped permanently. But
  62     // since there is no proper way of doing that on a remote track, we can
  63     // at least disable the track. Blink will not call down to the content layer
  64     // after a track has been stopped.
  65     SetEnabled(false);
  66   }
  67
  68  private:
  69   const scoped_refptr<webrtc::AudioTrackInterface> track_;
  70   const scoped_refptr<base::SingleThreadTaskRunner> signaling_thread_;
  71 };
  72
  73 // Base class used for mapping between webrtc and blink MediaStream tracks.
  74 // An instance of a RemoteMediaStreamTrackAdapter is stored in
  75 // RemoteMediaStreamImpl per remote audio and video track.
  76 template<typename WebRtcMediaStreamTrackType>
  77 class RemoteMediaStreamTrackAdapter
  78     : public base::RefCountedThreadSafe<
  79           RemoteMediaStreamTrackAdapter<WebRtcMediaStreamTrackType>> {
  80  public:
  81   RemoteMediaStreamTrackAdapter(
  82       const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
  83       WebRtcMediaStreamTrackType* webrtc_track)
  84       : main_thread_(main_thread), webrtc_track_(webrtc_track),
  85         id_(webrtc_track->id()) {
  86   }
  87
  88   const scoped_refptr<WebRtcMediaStreamTrackType>& observed_track() {
  89     return webrtc_track_;
  90   }
  91
  92   blink::WebMediaStreamTrack* webkit_track() {
  93     DCHECK(main_thread_->BelongsToCurrentThread());
  94     DCHECK(!webkit_track_.isNull());
  95     return &webkit_track_;
  96   }
  97
  98   const std::string& id() const { return id_; }
  99
 100   bool initialized() const {
 101     DCHECK(main_thread_->BelongsToCurrentThread());
 102     return !webkit_track_.isNull();
 103   }
 104
 105   void Initialize() {
 106     DCHECK(main_thread_->BelongsToCurrentThread());
 107     DCHECK(!initialized());
 108     webkit_initialize_.Run();
 109     webkit_initialize_.Reset();
 110     DCHECK(initialized());
 111   }
 112
 113  protected:
 114   friend class base::RefCountedThreadSafe<
 115       RemoteMediaStreamTrackAdapter<WebRtcMediaStreamTrackType>>;
 116
 117   virtual ~RemoteMediaStreamTrackAdapter() {
 118     DCHECK(main_thread_->BelongsToCurrentThread());
 119   }
 120
 121   void InitializeWebkitTrack(blink::WebMediaStreamSource::Type type) {
 122     DCHECK(main_thread_->BelongsToCurrentThread());
 123     DCHECK(webkit_track_.isNull());
 124
 125     blink::WebString webkit_track_id(base::UTF8ToUTF16(id_));
 126     blink::WebMediaStreamSource webkit_source;
 127     webkit_source.initialize(webkit_track_id, type, webkit_track_id);
 128     webkit_track_.initialize(webkit_track_id, webkit_source);
 129     DCHECK(!webkit_track_.isNull());
 130   }
 131
 132   const scoped_refptr<base::SingleThreadTaskRunner> main_thread_;
 133   // This callback will be run when Initialize() is called and then freed.
 134   // The callback is used by derived classes to bind objects that need to be
 135   // instantiated and initialized on the signaling thread but then moved to
 136   // and used on the main thread when initializing the webkit object(s).
 137   base::Callback<void()> webkit_initialize_;
 138
 139  private:
 140   const scoped_refptr<WebRtcMediaStreamTrackType> webrtc_track_;
 141   blink::WebMediaStreamTrack webkit_track_;
 142   // const copy of the webrtc track id that allows us to check it from both the
 143   // main and signaling threads without incurring a synchronous thread hop.
 144   const std::string id_;
 145
 146   DISALLOW_COPY_AND_ASSIGN(RemoteMediaStreamTrackAdapter);
 147 };
 148
 149 class RemoteVideoTrackAdapter
 150     : public RemoteMediaStreamTrackAdapter<webrtc::VideoTrackInterface> {
 151  public:
 152   // Called on the signaling thread
 153   RemoteVideoTrackAdapter(
 154       const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 155       webrtc::VideoTrackInterface* webrtc_track)
 156       : RemoteMediaStreamTrackAdapter(main_thread, webrtc_track) {
 157     source_observer_ = new MediaStreamRemoteVideoSource::Observer(main_thread,
 158         observed_track().get());
 159     // Here, we use base::Unretained() to avoid a circular reference.
 160     webkit_initialize_ = base::Bind(
 161         &RemoteVideoTrackAdapter::InitializeWebkitVideoTrack,
 162         base::Unretained(this), observed_track()->enabled());
 163   }
 164
 165  protected:
 166   ~RemoteVideoTrackAdapter() override {
 167     // If the source_observer_ is valid at this point, then that means we
 168     // haven't been initialized and we need to unregister the observer before
 169     // it gets automatically deleted.  We can't unregister from the destructor
 170     // of the observer because at that point, the refcount will be 0 and we
 171     // could receive an event during that time which would attempt to increment
 172     // the refcount while we're running the dtor on this thread...
 173     if (source_observer_.get()) {
 174       DCHECK(!initialized());
 175       source_observer_->Unregister();
 176     }
 177
 178     // TODO(tommi): There's got to be a smarter way of solving this.
 179     // We should probably separate the implementation of the observer itself
 180     // and the ownerof the weakptr to the main thread's implementation of
 181     // OnChanged.
 182   }
 183
 184  private:
 185   void InitializeWebkitVideoTrack(bool enabled) {
 186     DCHECK(main_thread_->BelongsToCurrentThread());
 187     scoped_ptr<MediaStreamRemoteVideoSource> video_source(
 188         new MediaStreamRemoteVideoSource(source_observer_));
 189     // Now that we're being initialized, we do not want to unregister the
 190     // observer in the dtor.  This will be done when the source goes out of
 191     // scope.  A source can be shared by more than one tracks (e.g via cloning)
 192     // so we must not unregister an observer that belongs to a live source.
 193     source_observer_ = nullptr;
 194     InitializeWebkitTrack(blink::WebMediaStreamSource::TypeVideo);
 195     webkit_track()->source().setExtraData(video_source.get());
 196     // Initial constraints must be provided to a MediaStreamVideoTrack. But
 197     // no constraints are available initially on a remote video track.
 198     blink::WebMediaConstraints constraints;
 199     constraints.initialize();
 200     MediaStreamVideoTrack* media_stream_track =
 201         new MediaStreamVideoTrack(video_source.release(), constraints,
 202             MediaStreamVideoSource::ConstraintsCallback(), enabled);
 203     webkit_track()->setExtraData(media_stream_track);
 204   }
 205
 206   // We hold on to the source observer in case we don't get initialized and
 207   // need to unregister it in the dtor.
 208   // TODO(tommi): There has to be a more elegant solution.
 209   scoped_refptr<MediaStreamRemoteVideoSource::Observer> source_observer_;
 210 };
 211
 212 // RemoteAudioTrackAdapter is responsible for listening on state
 213 // change notifications on a remote webrtc audio MediaStreamTracks and notify
 214 // WebKit.
 215 class RemoteAudioTrackAdapter
 216     : public RemoteMediaStreamTrackAdapter<webrtc::AudioTrackInterface>,
 217       public webrtc::ObserverInterface {
 218  public:
 219   RemoteAudioTrackAdapter(
 220       const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 221       webrtc::AudioTrackInterface* webrtc_track);
 222
 223   void Unregister();
 224
 225  protected:
 226   ~RemoteAudioTrackAdapter() override;
 227
 228  private:
 229   void InitializeWebkitAudioTrack(
 230       scoped_ptr<RemoteMediaStreamAudioTrack> media_stream_track);
 231
 232   // webrtc::ObserverInterface implementation.
 233   void OnChanged() override;
 234
 235   void OnChangedOnMainThread(
 236       webrtc::MediaStreamTrackInterface::TrackState state);
 237
 238 #if DCHECK_IS_ON
 239   bool unregistered_;
 240 #endif
 241
 242   webrtc::MediaStreamTrackInterface::TrackState state_;
 243
 244   DISALLOW_COPY_AND_ASSIGN(RemoteAudioTrackAdapter);
 245 };
 246
 247 // Called on the signaling thread.
 248 RemoteAudioTrackAdapter::RemoteAudioTrackAdapter(
 249     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 250     webrtc::AudioTrackInterface* webrtc_track)
 251     : RemoteMediaStreamTrackAdapter(main_thread, webrtc_track),
 252 #if DCHECK_IS_ON
 253       unregistered_(false),
 254 #endif
 255       state_(observed_track()->state()) {
 256   observed_track()->RegisterObserver(this);
 257   scoped_ptr<RemoteMediaStreamAudioTrack> media_stream_track(
 258       new RemoteMediaStreamAudioTrack(observed_track().get(),
 259           base::ThreadTaskRunnerHandle::Get()));
 260   // Here, we use base::Unretained() to avoid a circular reference.
 261   webkit_initialize_ = base::Bind(
 262           &RemoteAudioTrackAdapter::InitializeWebkitAudioTrack,
 263           base::Unretained(this), base::Passed(&media_stream_track));
 264 }
 265
 266 RemoteAudioTrackAdapter::~RemoteAudioTrackAdapter() {
 267 #if DCHECK_IS_ON
 268   DCHECK(unregistered_);
 269 #endif
 270 }
 271
 272 void RemoteAudioTrackAdapter::Unregister() {
 273 #if DCHECK_IS_ON
 274   DCHECK(!unregistered_);
 275   unregistered_ = true;
 276 #endif
 277   observed_track()->UnregisterObserver(this);
 278 }
 279
 280 void RemoteAudioTrackAdapter::InitializeWebkitAudioTrack(
 281     scoped_ptr<RemoteMediaStreamAudioTrack> media_stream_track) {
 282   InitializeWebkitTrack(blink::WebMediaStreamSource::TypeAudio);
 283   webkit_track()->setExtraData(media_stream_track.release());
 284 }
 285
 286 void RemoteAudioTrackAdapter::OnChanged() {
 287   main_thread_->PostTask(FROM_HERE,
 288       base::Bind(&RemoteAudioTrackAdapter::OnChangedOnMainThread,
 289           this, observed_track()->state()));
 290 }
 291
 292 void RemoteAudioTrackAdapter::OnChangedOnMainThread(
 293     webrtc::MediaStreamTrackInterface::TrackState state) {
 294   DCHECK(main_thread_->BelongsToCurrentThread());
 295
 296   if (state == state_ || !initialized())
 297     return;
 298
 299   state_ = state;
 300
 301   switch (state) {
 302     case webrtc::MediaStreamTrackInterface::kInitializing:
 303       // Ignore the kInitializing state since there is no match in
 304       // WebMediaStreamSource::ReadyState.
 305       break;
 306     case webrtc::MediaStreamTrackInterface::kLive:
 307       webkit_track()->source().setReadyState(
 308           blink::WebMediaStreamSource::ReadyStateLive);
 309       break;
 310     case webrtc::MediaStreamTrackInterface::kEnded:
 311       webkit_track()->source().setReadyState(
 312           blink::WebMediaStreamSource::ReadyStateEnded);
 313       break;
 314     default:
 315       NOTREACHED();
 316       break;
 317   }
 318 }
 319
 320 RemoteMediaStreamImpl::Observer::Observer(
 321     const base::WeakPtr<RemoteMediaStreamImpl>& media_stream,
 322     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 323     webrtc::MediaStreamInterface* webrtc_stream)
 324     : media_stream_(media_stream),
 325       main_thread_(main_thread),
 326       webrtc_stream_(webrtc_stream) {
 327   webrtc_stream_->RegisterObserver(this);
 328 }
 329
 330 RemoteMediaStreamImpl::Observer::~Observer() {
 331   DCHECK(!webrtc_stream_.get()) << "Unregister hasn't been called";
 332 }
 333
 334 void RemoteMediaStreamImpl::Observer::InitializeOnMainThread(
 335     const std::string& label) {
 336   DCHECK(main_thread_->BelongsToCurrentThread());
 337   if (media_stream_)
 338     media_stream_->InitializeOnMainThread(label);
 339 }
 340
 341 void RemoteMediaStreamImpl::Observer::Unregister() {
 342   DCHECK(main_thread_->BelongsToCurrentThread());
 343   webrtc_stream_->UnregisterObserver(this);
 344   // Since we're guaranteed to not get further notifications, it's safe to
 345   // release the webrtc_stream_ here.
 346   webrtc_stream_ = nullptr;
 347 }
 348
 349 void RemoteMediaStreamImpl::Observer::OnChanged() {
 350   scoped_ptr<RemoteAudioTrackAdapters> audio(new RemoteAudioTrackAdapters());
 351   scoped_ptr<RemoteVideoTrackAdapters> video(new RemoteVideoTrackAdapters());
 352
 353   CreateAdaptersForTracks(
 354       webrtc_stream_->GetAudioTracks(), audio.get(), main_thread_);
 355   CreateAdaptersForTracks(
 356       webrtc_stream_->GetVideoTracks(), video.get(), main_thread_);
 357
 358   main_thread_->PostTask(FROM_HERE,
 359       base::Bind(&RemoteMediaStreamImpl::Observer::OnChangedOnMainThread,
 360       this, base::Passed(&audio), base::Passed(&video)));
 361 }
 362
 363 void RemoteMediaStreamImpl::Observer::OnChangedOnMainThread(
 364     scoped_ptr<RemoteAudioTrackAdapters> audio_tracks,
 365     scoped_ptr<RemoteVideoTrackAdapters> video_tracks) {
 366   DCHECK(main_thread_->BelongsToCurrentThread());
 367   if (media_stream_)
 368     media_stream_->OnChanged(audio_tracks.Pass(), video_tracks.Pass());
 369 }
 370
 371 // Called on the signaling thread.
 372 RemoteMediaStreamImpl::RemoteMediaStreamImpl(
 373     const scoped_refptr<base::SingleThreadTaskRunner>& main_thread,
 374     webrtc::MediaStreamInterface* webrtc_stream)
 375     : signaling_thread_(base::ThreadTaskRunnerHandle::Get()),
 376       weak_factory_(this),
 377       observer_(new RemoteMediaStreamImpl::Observer(weak_factory_.GetWeakPtr(),
 378                 main_thread, webrtc_stream)) {
 379   CreateAdaptersForTracks(webrtc_stream->GetAudioTracks(),
 380       &audio_track_observers_, main_thread);
 381   CreateAdaptersForTracks(webrtc_stream->GetVideoTracks(),
 382       &video_track_observers_, main_thread);
 383
 384   main_thread->PostTask(FROM_HERE,
 385       base::Bind(&RemoteMediaStreamImpl::Observer::InitializeOnMainThread,
 386           observer_, webrtc_stream->label()));
 387 }
 388
 389 RemoteMediaStreamImpl::~RemoteMediaStreamImpl() {
 390   DCHECK(observer_->main_thread()->BelongsToCurrentThread());
 391   for (auto& track : audio_track_observers_)
 392     track->Unregister();
 393   observer_->Unregister();
 394 }
 395
 396 void RemoteMediaStreamImpl::InitializeOnMainThread(const std::string& label) {
 397   DCHECK(observer_->main_thread()->BelongsToCurrentThread());
 398   blink::WebVector<blink::WebMediaStreamTrack> webkit_audio_tracks(
 399         audio_track_observers_.size());
 400   for (size_t i = 0; i < audio_track_observers_.size(); ++i) {
 401     audio_track_observers_[i]->Initialize();
 402     webkit_audio_tracks[i] = *audio_track_observers_[i]->webkit_track();
 403   }
 404
 405   blink::WebVector<blink::WebMediaStreamTrack> webkit_video_tracks(
 406         video_track_observers_.size());
 407   for (size_t i = 0; i < video_track_observers_.size(); ++i) {
 408     video_track_observers_[i]->Initialize();
 409     webkit_video_tracks[i] = *video_track_observers_[i]->webkit_track();
 410   }
 411
 412   webkit_stream_.initialize(base::UTF8ToUTF16(label),
 413                             webkit_audio_tracks, webkit_video_tracks);
 414   webkit_stream_.setExtraData(new MediaStream(observer_->stream().get()));
 415 }
 416
 417 void RemoteMediaStreamImpl::OnChanged(
 418     scoped_ptr<RemoteAudioTrackAdapters> audio_tracks,
 419     scoped_ptr<RemoteVideoTrackAdapters> video_tracks) {
 420   // Find removed tracks.
 421   auto audio_it = audio_track_observers_.begin();
 422   while (audio_it != audio_track_observers_.end()) {
 423     if (!IsTrackInVector(*audio_tracks.get(), (*audio_it)->id())) {
 424       (*audio_it)->Unregister();
 425        webkit_stream_.removeTrack(*(*audio_it)->webkit_track());
 426        audio_it = audio_track_observers_.erase(audio_it);
 427     } else {
 428       ++audio_it;
 429     }
 430   }
 431
 432   auto video_it = video_track_observers_.begin();
 433   while (video_it != video_track_observers_.end()) {
 434     if (!IsTrackInVector(*video_tracks.get(), (*video_it)->id())) {
 435       webkit_stream_.removeTrack(*(*video_it)->webkit_track());
 436       video_it = video_track_observers_.erase(video_it);
 437     } else {
 438       ++video_it;
 439     }
 440   }
 441
 442   // Find added tracks.
 443   for (auto& track : *audio_tracks.get()) {
 444     if (!IsTrackInVector(audio_track_observers_, track->id())) {
 445       track->Initialize();
 446       audio_track_observers_.push_back(track);
 447       webkit_stream_.addTrack(*track->webkit_track());
 448       // Set the track to null to avoid unregistering it below now that it's
 449       // been associated with a media stream.
 450       track = nullptr;
 451     }
 452   }
 453
 454   // Find added video tracks.
 455   for (const auto& track : *video_tracks.get()) {
 456     if (!IsTrackInVector(video_track_observers_, track->id())) {
 457       track->Initialize();
 458       video_track_observers_.push_back(track);
 459       webkit_stream_.addTrack(*track->webkit_track());
 460     }
 461   }
 462
 463   // Unregister all the audio track observers that were not used.
 464   // We need to do this before destruction since the observers can't unregister
 465   // from within the dtor due to a race.
 466   for (auto& track : *audio_tracks.get()) {
 467     if (track.get())
 468       track->Unregister();
 469   }
 470 }
 471
 472 }  // namespace content