1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_video_source.h"
11 #include "base/debug/trace_event.h"
12 #include "base/logging.h"
13 #include "base/strings/string_number_conversions.h"
14 #include "content/child/child_process.h"
15 #include "content/renderer/media/media_stream_constraints_util.h"
16 #include "content/renderer/media/media_stream_video_track.h"
17 #include "content/renderer/media/video_track_adapter.h"
21 // Constraint keys. Specified by draft-alvestrand-constraints-resolution-00b
22 const char MediaStreamVideoSource::kMinAspectRatio[] = "minAspectRatio";
23 const char MediaStreamVideoSource::kMaxAspectRatio[] = "maxAspectRatio";
24 const char MediaStreamVideoSource::kMaxWidth[] = "maxWidth";
25 const char MediaStreamVideoSource::kMinWidth[] = "minWidth";
26 const char MediaStreamVideoSource::kMaxHeight[] = "maxHeight";
27 const char MediaStreamVideoSource::kMinHeight[] = "minHeight";
28 const char MediaStreamVideoSource::kMaxFrameRate[] = "maxFrameRate";
29 const char MediaStreamVideoSource::kMinFrameRate[] = "minFrameRate";
31 const char* kSupportedConstraints[] = {
32 MediaStreamVideoSource::kMaxAspectRatio,
33 MediaStreamVideoSource::kMinAspectRatio,
34 MediaStreamVideoSource::kMaxWidth,
35 MediaStreamVideoSource::kMinWidth,
36 MediaStreamVideoSource::kMaxHeight,
37 MediaStreamVideoSource::kMinHeight,
38 MediaStreamVideoSource::kMaxFrameRate,
39 MediaStreamVideoSource::kMinFrameRate,
42 const int MediaStreamVideoSource::kDefaultWidth = 640;
43 const int MediaStreamVideoSource::kDefaultHeight = 480;
44 const int MediaStreamVideoSource::kDefaultFrameRate = 30;
45 const int MediaStreamVideoSource::kUnknownFrameRate = 0;
49 // Google-specific key prefix. Constraints with this prefix are ignored if they
51 const char kGooglePrefix[] = "goog";
53 // Returns true if |constraint| has mandatory constraints.
54 bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
55 blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
56 constraints.getMandatoryConstraints(mandatory_constraints);
57 return !mandatory_constraints.isEmpty();
60 // Retrieve the desired max width and height from |constraints|. If not set,
61 // the |desired_width| and |desired_height| are set to
62 // std::numeric_limits<int>::max();
63 // If either max width or height is set as a mandatory constraint, the optional
64 // constraints are not checked.
65 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
66 int* desired_width, int* desired_height) {
67 *desired_width = std::numeric_limits<int>::max();
68 *desired_height = std::numeric_limits<int>::max();
70 bool mandatory = GetMandatoryConstraintValueAsInteger(
72 MediaStreamVideoSource::kMaxWidth,
74 mandatory |= GetMandatoryConstraintValueAsInteger(
76 MediaStreamVideoSource::kMaxHeight,
81 GetOptionalConstraintValueAsInteger(constraints,
82 MediaStreamVideoSource::kMaxWidth,
84 GetOptionalConstraintValueAsInteger(constraints,
85 MediaStreamVideoSource::kMaxHeight,
89 // Retrieve the desired max and min aspect ratio from |constraints|. If not set,
90 // the |min_aspect_ratio| is set to 0 and |max_aspect_ratio| is set to
91 // std::numeric_limits<double>::max();
92 // If either min or max aspect ratio is set as a mandatory constraint, the
93 // optional constraints are not checked.
94 void GetDesiredMinAndMaxAspectRatio(
95 const blink::WebMediaConstraints& constraints,
96 double* min_aspect_ratio,
97 double* max_aspect_ratio) {
98 *min_aspect_ratio = 0;
99 *max_aspect_ratio = std::numeric_limits<double>::max();
101 bool mandatory = GetMandatoryConstraintValueAsDouble(
103 MediaStreamVideoSource::kMinAspectRatio,
105 mandatory |= GetMandatoryConstraintValueAsDouble(
107 MediaStreamVideoSource::kMaxAspectRatio,
112 GetOptionalConstraintValueAsDouble(
114 MediaStreamVideoSource::kMinAspectRatio,
116 GetOptionalConstraintValueAsDouble(
118 MediaStreamVideoSource::kMaxAspectRatio,
122 // Returns true if |constraint| is fulfilled. |format| can be changed by a
123 // constraint, e.g. the frame rate can be changed by setting maxFrameRate.
124 bool UpdateFormatForConstraint(
125 const blink::WebMediaConstraint& constraint,
127 media::VideoCaptureFormat* format) {
128 DCHECK(format != NULL);
130 if (!format->IsValid())
133 std::string constraint_name = constraint.m_name.utf8();
134 std::string constraint_value = constraint.m_value.utf8();
136 if (constraint_name.find(kGooglePrefix) == 0) {
137 // These are actually options, not constraints, so they can be satisfied
138 // regardless of the format.
142 if (constraint_name == MediaStreamSource::kSourceId) {
143 // This is a constraint that doesn't affect the format.
147 // Ignore Chrome specific Tab capture constraints.
148 if (constraint_name == kMediaStreamSource ||
149 constraint_name == kMediaStreamSourceId)
152 if (constraint_name == MediaStreamVideoSource::kMinAspectRatio ||
153 constraint_name == MediaStreamVideoSource::kMaxAspectRatio) {
154 // These constraints are handled by cropping if the camera outputs the wrong
157 return base::StringToDouble(constraint_value, &value);
161 if (!base::StringToDouble(constraint_value, &value)) {
162 DLOG(WARNING) << "Can't parse MediaStream constraint. Name:"
163 << constraint_name << " Value:" << constraint_value;
167 if (constraint_name == MediaStreamVideoSource::kMinWidth) {
168 return (value <= format->frame_size.width());
169 } else if (constraint_name == MediaStreamVideoSource::kMaxWidth) {
171 } else if (constraint_name == MediaStreamVideoSource::kMinHeight) {
172 return (value <= format->frame_size.height());
173 } else if (constraint_name == MediaStreamVideoSource::kMaxHeight) {
175 } else if (constraint_name == MediaStreamVideoSource::kMinFrameRate) {
176 return (value > 0.0) && (value <= format->frame_rate);
177 } else if (constraint_name == MediaStreamVideoSource::kMaxFrameRate) {
179 // The frame rate is set by constraint.
180 // Don't allow 0 as frame rate if it is a mandatory constraint.
181 // Set the frame rate to 1 if it is not mandatory.
189 (format->frame_rate > value) ? value : format->frame_rate;
192 LOG(WARNING) << "Found unknown MediaStream constraint. Name:"
193 << constraint_name << " Value:" << constraint_value;
198 // Removes media::VideoCaptureFormats from |formats| that don't meet
200 void FilterFormatsByConstraint(
201 const blink::WebMediaConstraint& constraint,
203 media::VideoCaptureFormats* formats) {
204 DVLOG(3) << "FilterFormatsByConstraint("
205 << "{ constraint.m_name = " << constraint.m_name.utf8()
206 << " constraint.m_value = " << constraint.m_value.utf8()
207 << " mandatory = " << mandatory << "})";
208 media::VideoCaptureFormats::iterator format_it = formats->begin();
209 while (format_it != formats->end()) {
210 // Modify the format_it to fulfill the constraint if possible.
211 // Delete it otherwise.
212 if (!UpdateFormatForConstraint(constraint, mandatory, &(*format_it))) {
213 format_it = formats->erase(format_it);
220 // Returns the media::VideoCaptureFormats that matches |constraints|.
221 media::VideoCaptureFormats FilterFormats(
222 const blink::WebMediaConstraints& constraints,
223 const media::VideoCaptureFormats& supported_formats,
224 blink::WebString* unsatisfied_constraint) {
225 if (constraints.isNull()) {
226 return supported_formats;
229 double max_aspect_ratio;
230 double min_aspect_ratio;
231 GetDesiredMinAndMaxAspectRatio(constraints,
235 if (min_aspect_ratio > max_aspect_ratio || max_aspect_ratio < 0.05f) {
236 DLOG(WARNING) << "Wrong requested aspect ratio.";
237 return media::VideoCaptureFormats();
241 GetMandatoryConstraintValueAsInteger(constraints,
242 MediaStreamVideoSource::kMinWidth,
245 GetMandatoryConstraintValueAsInteger(constraints,
246 MediaStreamVideoSource::kMinHeight,
250 GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
252 if (min_width > max_width || min_height > max_height)
253 return media::VideoCaptureFormats();
255 double min_frame_rate = 0.0f;
256 double max_frame_rate = 0.0f;
257 if (GetConstraintValueAsDouble(constraints,
258 MediaStreamVideoSource::kMaxFrameRate,
260 GetConstraintValueAsDouble(constraints,
261 MediaStreamVideoSource::kMinFrameRate,
263 if (min_frame_rate > max_frame_rate) {
264 DLOG(WARNING) << "Wrong requested frame rate.";
265 return media::VideoCaptureFormats();
269 blink::WebVector<blink::WebMediaConstraint> mandatory;
270 blink::WebVector<blink::WebMediaConstraint> optional;
271 constraints.getMandatoryConstraints(mandatory);
272 constraints.getOptionalConstraints(optional);
273 media::VideoCaptureFormats candidates = supported_formats;
274 for (size_t i = 0; i < mandatory.size(); ++i) {
275 FilterFormatsByConstraint(mandatory[i], true, &candidates);
276 if (candidates.empty()) {
277 *unsatisfied_constraint = mandatory[i].m_name;
282 if (candidates.empty())
285 // Ok - all mandatory checked and we still have candidates.
286 // Let's try filtering using the optional constraints. The optional
287 // constraints must be filtered in the order they occur in |optional|.
288 // But if a constraint produce zero candidates, the constraint is ignored and
289 // the next constraint is tested.
290 // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#idl-def-Constraints
291 for (size_t i = 0; i < optional.size(); ++i) {
292 media::VideoCaptureFormats current_candidates = candidates;
293 FilterFormatsByConstraint(optional[i], false, ¤t_candidates);
294 if (!current_candidates.empty()) {
295 candidates = current_candidates;
299 // We have done as good as we can to filter the supported resolutions.
303 const media::VideoCaptureFormat& GetBestFormatBasedOnArea(
304 const media::VideoCaptureFormats& formats,
306 media::VideoCaptureFormats::const_iterator it = formats.begin();
307 media::VideoCaptureFormats::const_iterator best_it = formats.begin();
308 int best_diff = std::numeric_limits<int>::max();
309 for (; it != formats.end(); ++it) {
310 int diff = abs(area - it->frame_size.width() * it->frame_size.height());
311 if (diff < best_diff) {
319 // Find the format that best matches the default video size.
320 // This algorithm is chosen since a resolution must be picked even if no
321 // constraints are provided. We don't just select the maximum supported
322 // resolution since higher resolutions cost more in terms of complexity and
323 // many cameras have lower frame rate and have more noise in the image at
324 // their maximum supported resolution.
325 void GetBestCaptureFormat(
326 const media::VideoCaptureFormats& formats,
327 const blink::WebMediaConstraints& constraints,
328 media::VideoCaptureFormat* capture_format) {
329 DCHECK(!formats.empty());
333 GetDesiredMaxWidthAndHeight(constraints, &max_width, &max_height);
335 *capture_format = GetBestFormatBasedOnArea(
337 std::min(max_width, MediaStreamVideoSource::kDefaultWidth) *
338 std::min(max_height, MediaStreamVideoSource::kDefaultHeight));
341 } // anonymous namespace
344 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
345 const blink::WebMediaStreamSource& source) {
346 return static_cast<MediaStreamVideoSource*>(source.extraData());
350 bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
351 for (size_t i = 0; i < arraysize(kSupportedConstraints); ++i) {
352 if (kSupportedConstraints[i] == name)
358 MediaStreamVideoSource::MediaStreamVideoSource()
360 track_adapter_(new VideoTrackAdapter(
361 ChildProcess::current()->io_message_loop_proxy())),
362 weak_factory_(this) {
365 MediaStreamVideoSource::~MediaStreamVideoSource() {
366 DCHECK(CalledOnValidThread());
369 void MediaStreamVideoSource::AddTrack(
370 MediaStreamVideoTrack* track,
371 const VideoCaptureDeliverFrameCB& frame_callback,
372 const blink::WebMediaConstraints& constraints,
373 const ConstraintsCallback& callback) {
374 DCHECK(CalledOnValidThread());
375 DCHECK(!constraints.isNull());
376 DCHECK(std::find(tracks_.begin(), tracks_.end(),
377 track) == tracks_.end());
378 tracks_.push_back(track);
380 requested_constraints_.push_back(
381 RequestedConstraints(track, frame_callback, constraints, callback));
385 // Tab capture and Screen capture needs the maximum requested height
386 // and width to decide on the resolution.
387 int max_requested_width = 0;
388 GetMandatoryConstraintValueAsInteger(constraints, kMaxWidth,
389 &max_requested_width);
391 int max_requested_height = 0;
392 GetMandatoryConstraintValueAsInteger(constraints, kMaxHeight,
393 &max_requested_height);
395 double max_requested_frame_rate;
396 if (!GetConstraintValueAsDouble(constraints, kMaxFrameRate,
397 &max_requested_frame_rate)) {
398 max_requested_frame_rate = kDefaultFrameRate;
401 state_ = RETRIEVING_CAPABILITIES;
402 GetCurrentSupportedFormats(
404 max_requested_height,
405 max_requested_frame_rate,
406 base::Bind(&MediaStreamVideoSource::OnSupportedFormats,
407 weak_factory_.GetWeakPtr()));
412 case RETRIEVING_CAPABILITIES: {
413 // The |callback| will be triggered once the source has started or
414 // the capabilities have been retrieved.
419 // Currently, reconfiguring the source is not supported.
425 void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
426 DCHECK(CalledOnValidThread());
427 std::vector<MediaStreamVideoTrack*>::iterator it =
428 std::find(tracks_.begin(), tracks_.end(), video_track);
429 DCHECK(it != tracks_.end());
432 // Check if |video_track| is waiting for applying new constraints and remove
433 // the request in that case.
434 for (std::vector<RequestedConstraints>::iterator it =
435 requested_constraints_.begin();
436 it != requested_constraints_.end(); ++it) {
437 if (it->track == video_track) {
438 requested_constraints_.erase(it);
442 // Call |frame_adapter_->RemoveTrack| here even if adding the track has
443 // failed and |frame_adapter_->AddCallback| has not been called.
444 track_adapter_->RemoveTrack(video_track);
450 const scoped_refptr<base::MessageLoopProxy>&
451 MediaStreamVideoSource::io_message_loop() const {
452 DCHECK(CalledOnValidThread());
453 return track_adapter_->io_message_loop();
456 void MediaStreamVideoSource::DoStopSource() {
457 DCHECK(CalledOnValidThread());
458 DVLOG(3) << "DoStopSource()";
461 track_adapter_->StopFrameMonitoring();
464 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
467 void MediaStreamVideoSource::OnSupportedFormats(
468 const media::VideoCaptureFormats& formats) {
469 DCHECK(CalledOnValidThread());
470 DCHECK_EQ(RETRIEVING_CAPABILITIES, state_);
472 supported_formats_ = formats;
473 if (!FindBestFormatWithConstraints(supported_formats_,
475 SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
476 // This object can be deleted after calling FinalizeAddTrack. See comment
477 // in the header file.
483 DVLOG(3) << "Starting the capturer with " << current_format_.ToString();
487 base::Bind(&VideoTrackAdapter::DeliverFrameOnIO, track_adapter_));
490 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
491 const media::VideoCaptureFormats& formats,
492 media::VideoCaptureFormat* best_format) {
493 DCHECK(CalledOnValidThread());
494 // Find the first constraints that we can fulfill.
495 for (std::vector<RequestedConstraints>::iterator request_it =
496 requested_constraints_.begin();
497 request_it != requested_constraints_.end(); ++request_it) {
498 const blink::WebMediaConstraints& requested_constraints =
499 request_it->constraints;
501 // If the source doesn't support capability enumeration it is still ok if
502 // no mandatory constraints have been specified. That just means that
503 // we will start with whatever format is native to the source.
504 if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
505 *best_format = media::VideoCaptureFormat();
508 blink::WebString unsatisfied_constraint;
509 media::VideoCaptureFormats filtered_formats =
510 FilterFormats(requested_constraints, formats, &unsatisfied_constraint);
511 if (filtered_formats.size() > 0) {
512 // A request with constraints that can be fulfilled.
513 GetBestCaptureFormat(filtered_formats,
514 requested_constraints,
522 void MediaStreamVideoSource::OnStartDone(MediaStreamRequestResult result) {
523 DCHECK(CalledOnValidThread());
524 DVLOG(3) << "OnStartDone({result =" << result << "})";
525 if (result == MEDIA_DEVICE_OK) {
526 DCHECK_EQ(STARTING, state_);
528 SetReadyState(blink::WebMediaStreamSource::ReadyStateLive);
530 track_adapter_->StartFrameMonitoring(
531 current_format_.frame_rate,
532 base::Bind(&MediaStreamVideoSource::SetMutedState,
533 weak_factory_.GetWeakPtr()));
539 // This object can be deleted after calling FinalizeAddTrack. See comment in
544 void MediaStreamVideoSource::FinalizeAddTrack() {
545 DCHECK(CalledOnValidThread());
546 media::VideoCaptureFormats formats;
547 formats.push_back(current_format_);
549 std::vector<RequestedConstraints> callbacks;
550 callbacks.swap(requested_constraints_);
551 for (std::vector<RequestedConstraints>::iterator it = callbacks.begin();
552 it != callbacks.end(); ++it) {
553 MediaStreamRequestResult result = MEDIA_DEVICE_OK;
554 blink::WebString unsatisfied_constraint;
556 if (HasMandatoryConstraints(it->constraints) &&
557 FilterFormats(it->constraints, formats,
558 &unsatisfied_constraint).empty())
559 result = MEDIA_DEVICE_CONSTRAINT_NOT_SATISFIED;
561 if (state_ != STARTED && result == MEDIA_DEVICE_OK)
562 result = MEDIA_DEVICE_TRACK_START_FAILURE;
564 if (result == MEDIA_DEVICE_OK) {
567 GetDesiredMaxWidthAndHeight(it->constraints, &max_width, &max_height);
568 double max_aspect_ratio;
569 double min_aspect_ratio;
570 GetDesiredMinAndMaxAspectRatio(it->constraints,
573 double max_frame_rate = 0.0f;
574 GetConstraintValueAsDouble(it->constraints,
575 kMaxFrameRate, &max_frame_rate);
577 track_adapter_->AddTrack(it->track, it->frame_callback,
578 max_width, max_height,
579 min_aspect_ratio, max_aspect_ratio,
583 DVLOG(3) << "FinalizeAddTrack() result " << result;
585 if (!it->callback.is_null()) {
586 it->callback.Run(this, result, unsatisfied_constraint);
591 void MediaStreamVideoSource::SetReadyState(
592 blink::WebMediaStreamSource::ReadyState state) {
593 DVLOG(3) << "MediaStreamVideoSource::SetReadyState state " << state;
594 DCHECK(CalledOnValidThread());
595 if (!owner().isNull())
596 owner().setReadyState(state);
597 for (std::vector<MediaStreamVideoTrack*>::iterator it = tracks_.begin();
598 it != tracks_.end(); ++it) {
599 (*it)->OnReadyStateChanged(state);
603 void MediaStreamVideoSource::SetMutedState(bool muted_state) {
604 DVLOG(3) << "MediaStreamVideoSource::SetMutedState state=" << muted_state;
605 DCHECK(CalledOnValidThread());
606 if (!owner().isNull()) {
607 owner().setReadyState(muted_state
608 ? blink::WebMediaStreamSource::ReadyStateMuted
609 : blink::WebMediaStreamSource::ReadyStateLive);
613 MediaStreamVideoSource::RequestedConstraints::RequestedConstraints(
614 MediaStreamVideoTrack* track,
615 const VideoCaptureDeliverFrameCB& frame_callback,
616 const blink::WebMediaConstraints& constraints,
617 const ConstraintsCallback& callback)
619 frame_callback(frame_callback),
620 constraints(constraints),
624 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {
627 } // namespace content