1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/renderer/media/cast_rtp_stream.h"
8 #include "base/debug/trace_event.h"
9 #include "base/logging.h"
10 #include "base/memory/weak_ptr.h"
11 #include "base/strings/stringprintf.h"
12 #include "base/sys_info.h"
13 #include "chrome/renderer/media/cast_session.h"
14 #include "chrome/renderer/media/cast_udp_transport.h"
15 #include "content/public/renderer/media_stream_audio_sink.h"
16 #include "content/public/renderer/media_stream_video_sink.h"
17 #include "content/public/renderer/render_thread.h"
18 #include "content/public/renderer/video_encode_accelerator.h"
19 #include "media/audio/audio_parameters.h"
20 #include "media/base/audio_bus.h"
21 #include "media/base/audio_fifo.h"
22 #include "media/base/bind_to_current_loop.h"
23 #include "media/base/multi_channel_resampler.h"
24 #include "media/base/video_frame.h"
25 #include "media/cast/cast_config.h"
26 #include "media/cast/cast_defines.h"
27 #include "media/cast/cast_sender.h"
28 #include "media/cast/net/cast_transport_config.h"
29 #include "third_party/WebKit/public/platform/WebMediaStreamSource.h"
30 #include "ui/gfx/geometry/size.h"
32 using media::cast::AudioSenderConfig;
33 using media::cast::VideoSenderConfig;
37 const char kCodecNameOpus[] = "OPUS";
38 const char kCodecNameVp8[] = "VP8";
39 const char kCodecNameH264[] = "H264";
41 // To convert from kilobits per second to bits to per second.
42 const int kBitrateMultiplier = 1000;
44 // This constant defines the number of sets of audio data to buffer
45 // in the FIFO. If input audio and output data have different resampling
46 // rates then buffer is necessary to avoid audio glitches.
47 // See CastAudioSink::ResampleData() and CastAudioSink::OnSetFormat()
49 const int kBufferAudioData = 2;
51 CastRtpPayloadParams DefaultOpusPayload() {
52 CastRtpPayloadParams payload;
53 payload.payload_type = 127;
54 payload.max_latency_ms = media::cast::kDefaultRtpMaxDelayMs;
56 payload.feedback_ssrc = 2;
57 payload.clock_rate = media::cast::kDefaultAudioSamplingRate;
58 // The value is 0 which means VBR.
59 payload.min_bitrate = payload.max_bitrate =
60 media::cast::kDefaultAudioEncoderBitrate;
62 payload.max_frame_rate = 100; // 10 ms audio frames
63 payload.codec_name = kCodecNameOpus;
67 CastRtpPayloadParams DefaultVp8Payload() {
68 CastRtpPayloadParams payload;
69 payload.payload_type = 96;
70 payload.max_latency_ms = media::cast::kDefaultRtpMaxDelayMs;
72 payload.feedback_ssrc = 12;
73 payload.clock_rate = media::cast::kVideoFrequency;
74 payload.max_bitrate = 2000;
75 payload.min_bitrate = 50;
77 payload.max_frame_rate = media::cast::kDefaultMaxFrameRate;
80 payload.codec_name = kCodecNameVp8;
84 CastRtpPayloadParams DefaultH264Payload() {
85 CastRtpPayloadParams payload;
86 // TODO(hshi): set different ssrc/rtpPayloadType values for H264 and VP8
87 // once b/13696137 is fixed.
88 payload.payload_type = 96;
89 payload.max_latency_ms = media::cast::kDefaultRtpMaxDelayMs;
91 payload.feedback_ssrc = 12;
92 payload.clock_rate = media::cast::kVideoFrequency;
93 payload.max_bitrate = 2000;
94 payload.min_bitrate = 50;
96 payload.max_frame_rate = media::cast::kDefaultMaxFrameRate;
99 payload.codec_name = kCodecNameH264;
103 bool IsHardwareVP8EncodingSupported() {
104 // Query for hardware VP8 encoder support.
105 std::vector<media::VideoEncodeAccelerator::SupportedProfile> vea_profiles =
106 content::GetSupportedVideoEncodeAcceleratorProfiles();
107 for (size_t i = 0; i < vea_profiles.size(); ++i) {
108 if (vea_profiles[i].profile >= media::VP8PROFILE_MIN &&
109 vea_profiles[i].profile <= media::VP8PROFILE_MAX) {
116 bool IsHardwareH264EncodingSupported() {
117 // Query for hardware H.264 encoder support.
118 std::vector<media::VideoEncodeAccelerator::SupportedProfile> vea_profiles =
119 content::GetSupportedVideoEncodeAcceleratorProfiles();
120 for (size_t i = 0; i < vea_profiles.size(); ++i) {
121 if (vea_profiles[i].profile >= media::H264PROFILE_MIN &&
122 vea_profiles[i].profile <= media::H264PROFILE_MAX) {
129 int NumberOfEncodeThreads() {
130 // We want to give CPU cycles for capturing and not to saturate the system
131 // just for encoding. So on a lower end system with only 1 or 2 cores we
132 // use only one thread for encoding.
133 if (base::SysInfo::NumberOfProcessors() <= 2)
136 // On higher end we want to use 2 threads for encoding to reduce latency.
137 // In theory a physical CPU core has maximum 2 hyperthreads. Having 3 or
138 // more logical processors means the system has at least 2 physical cores.
142 std::vector<CastRtpParams> SupportedAudioParams() {
143 // TODO(hclam): Fill in more codecs here.
144 std::vector<CastRtpParams> supported_params;
145 supported_params.push_back(CastRtpParams(DefaultOpusPayload()));
146 return supported_params;
149 std::vector<CastRtpParams> SupportedVideoParams() {
150 std::vector<CastRtpParams> supported_params;
151 if (IsHardwareH264EncodingSupported())
152 supported_params.push_back(CastRtpParams(DefaultH264Payload()));
153 supported_params.push_back(CastRtpParams(DefaultVp8Payload()));
154 return supported_params;
157 bool ToAudioSenderConfig(const CastRtpParams& params,
158 AudioSenderConfig* config) {
159 config->ssrc = params.payload.ssrc;
160 config->incoming_feedback_ssrc = params.payload.feedback_ssrc;
161 if (config->ssrc == config->incoming_feedback_ssrc)
163 config->min_playout_delay =
164 base::TimeDelta::FromMilliseconds(
165 params.payload.min_latency_ms ?
166 params.payload.min_latency_ms :
167 params.payload.max_latency_ms);
168 config->max_playout_delay =
169 base::TimeDelta::FromMilliseconds(params.payload.max_latency_ms);
170 if (config->min_playout_delay <= base::TimeDelta())
172 if (config->min_playout_delay > config->max_playout_delay)
174 config->rtp_payload_type = params.payload.payload_type;
175 config->use_external_encoder = false;
176 config->frequency = params.payload.clock_rate;
177 if (config->frequency < 8000)
179 config->channels = params.payload.channels;
180 if (config->channels < 1)
182 config->bitrate = params.payload.max_bitrate * kBitrateMultiplier;
183 if (params.payload.codec_name == kCodecNameOpus)
184 config->codec = media::cast::CODEC_AUDIO_OPUS;
187 config->aes_key = params.payload.aes_key;
188 config->aes_iv_mask = params.payload.aes_iv_mask;
192 bool ToVideoSenderConfig(const CastRtpParams& params,
193 VideoSenderConfig* config) {
194 config->ssrc = params.payload.ssrc;
195 config->incoming_feedback_ssrc = params.payload.feedback_ssrc;
196 if (config->ssrc == config->incoming_feedback_ssrc)
198 config->min_playout_delay =
199 base::TimeDelta::FromMilliseconds(
200 params.payload.min_latency_ms ?
201 params.payload.min_latency_ms :
202 params.payload.max_latency_ms);
203 config->max_playout_delay =
204 base::TimeDelta::FromMilliseconds(params.payload.max_latency_ms);
205 if (config->min_playout_delay <= base::TimeDelta())
207 if (config->min_playout_delay > config->max_playout_delay)
209 config->rtp_payload_type = params.payload.payload_type;
210 config->width = params.payload.width;
211 config->height = params.payload.height;
212 if (config->width < 2 || config->height < 2)
214 config->min_bitrate = config->start_bitrate =
215 params.payload.min_bitrate * kBitrateMultiplier;
216 config->max_bitrate = params.payload.max_bitrate * kBitrateMultiplier;
217 if (config->min_bitrate > config->max_bitrate)
219 config->start_bitrate = config->min_bitrate;
220 config->max_frame_rate = static_cast<int>(
221 std::max(1.0, params.payload.max_frame_rate) + 0.5);
222 if (config->max_frame_rate > 120)
224 if (params.payload.codec_name == kCodecNameVp8) {
225 config->use_external_encoder = IsHardwareVP8EncodingSupported();
226 config->codec = media::cast::CODEC_VIDEO_VP8;
227 } else if (params.payload.codec_name == kCodecNameH264) {
228 config->use_external_encoder = IsHardwareH264EncodingSupported();
229 config->codec = media::cast::CODEC_VIDEO_H264;
233 if (!config->use_external_encoder) {
234 config->number_of_encode_threads = NumberOfEncodeThreads();
236 config->aes_key = params.payload.aes_key;
237 config->aes_iv_mask = params.payload.aes_iv_mask;
243 // This class receives MediaStreamTrack events and video frames from a
246 // Threading: Video frames are received on the IO thread and then
247 // forwarded to media::cast::VideoFrameInput through a static method.
248 // Member variables of this class are only accessed on the render thread.
249 class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>,
250 public content::MediaStreamVideoSink {
252 // |track| provides data for this sink.
253 // |expected_natural_size| is the expected dimension of the video frame.
254 // |error_callback| is called if video formats don't match.
255 CastVideoSink(const blink::WebMediaStreamTrack& track,
256 const gfx::Size& expected_natural_size,
257 const CastRtpStream::ErrorCallback& error_callback)
260 expected_natural_size_(expected_natural_size),
261 error_callback_(error_callback) {}
263 virtual ~CastVideoSink() {
265 RemoveFromVideoTrack(this, track_);
268 // This static method is used to forward video frames to |frame_input|.
269 static void OnVideoFrame(
270 // These parameters are already bound when callback is created.
271 const gfx::Size& expected_natural_size,
272 const CastRtpStream::ErrorCallback& error_callback,
273 const scoped_refptr<media::cast::VideoFrameInput> frame_input,
274 // These parameters are passed for each frame.
275 const scoped_refptr<media::VideoFrame>& frame,
276 const media::VideoCaptureFormat& format,
277 const base::TimeTicks& estimated_capture_time) {
278 if (frame->natural_size() != expected_natural_size) {
280 base::StringPrintf("Video frame resolution does not match config."
281 " Expected %dx%d. Got %dx%d.",
282 expected_natural_size.width(),
283 expected_natural_size.height(),
284 frame->natural_size().width(),
285 frame->natural_size().height()));
289 base::TimeTicks timestamp;
290 if (estimated_capture_time.is_null())
291 timestamp = base::TimeTicks::Now();
293 timestamp = estimated_capture_time;
295 // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
296 TRACE_EVENT_INSTANT2(
297 "cast_perf_test", "MediaStreamVideoSink::OnVideoFrame",
298 TRACE_EVENT_SCOPE_THREAD,
299 "timestamp", timestamp.ToInternalValue(),
300 "time_delta", frame->timestamp().ToInternalValue());
301 frame_input->InsertRawVideoFrame(frame, timestamp);
304 // Attach this sink to a video track represented by |track_|.
305 // Data received from the track will be submitted to |frame_input|.
307 const scoped_refptr<media::cast::VideoFrameInput>& frame_input) {
308 DCHECK(!sink_added_);
313 &CastVideoSink::OnVideoFrame,
314 expected_natural_size_,
321 blink::WebMediaStreamTrack track_;
323 gfx::Size expected_natural_size_;
324 CastRtpStream::ErrorCallback error_callback_;
326 DISALLOW_COPY_AND_ASSIGN(CastVideoSink);
329 // Receives audio data from a MediaStreamTrack. Data is submitted to
330 // media::cast::FrameInput.
332 // Threading: Audio frames are received on the real-time audio thread.
333 // Note that RemoveFromAudioTrack() is synchronous and we have
334 // gurantee that there will be no more audio data after calling it.
335 class CastAudioSink : public base::SupportsWeakPtr<CastAudioSink>,
336 public content::MediaStreamAudioSink {
338 // |track| provides data for this sink.
339 // |error_callback| is called if audio formats don't match.
340 CastAudioSink(const blink::WebMediaStreamTrack& track,
341 const CastRtpStream::ErrorCallback& error_callback,
343 int output_sample_rate)
346 error_callback_(error_callback),
348 output_channels_(output_channels),
349 output_sample_rate_(output_sample_rate),
352 virtual ~CastAudioSink() {
354 RemoveFromAudioTrack(this, track_);
357 // Called on real-time audio thread.
358 // content::MediaStreamAudioSink implementation.
359 virtual void OnData(const int16* audio_data,
361 int number_of_channels,
362 int number_of_frames) OVERRIDE {
363 scoped_ptr<media::AudioBus> input_bus;
365 input_bus = ResampleData(
366 audio_data, sample_rate, number_of_channels, number_of_frames);
370 input_bus = media::AudioBus::Create(
371 number_of_channels, number_of_frames);
372 input_bus->FromInterleaved(
373 audio_data, number_of_frames, number_of_channels);
376 // TODO(hclam): Pass in the accurate capture time to have good
377 // audio / video sync.
378 frame_input_->InsertAudio(input_bus.Pass(), base::TimeTicks::Now());
381 // Return a resampled audio data from input. This is called when the
382 // input sample rate doesn't match the output.
383 // The flow of data is as follows:
385 // AudioFifo |fifo_| ->
386 // MultiChannelResampler |resampler|.
388 // The resampler pulls data out of the FIFO and resample the data in
389 // frequency domain. It might call |fifo_| for more than once. But no more
390 // than |kBufferAudioData| times. We preroll audio data into the FIFO to
391 // make sure there's enough data for resampling.
392 scoped_ptr<media::AudioBus> ResampleData(
393 const int16* audio_data,
395 int number_of_channels,
396 int number_of_frames) {
397 DCHECK_EQ(number_of_channels, output_channels_);
398 fifo_input_bus_->FromInterleaved(
399 audio_data, number_of_frames, number_of_channels);
400 fifo_->Push(fifo_input_bus_.get());
402 if (input_preroll_ < kBufferAudioData - 1) {
404 return scoped_ptr<media::AudioBus>();
407 scoped_ptr<media::AudioBus> output_bus(
408 media::AudioBus::Create(
410 output_sample_rate_ * fifo_input_bus_->frames() / sample_rate));
412 // Resampler will then call ProvideData() below to fetch data from
414 resampler_->Resample(output_bus->frames(), output_bus.get());
415 return output_bus.Pass();
418 // Called on real-time audio thread.
419 virtual void OnSetFormat(const media::AudioParameters& params) OVERRIDE {
420 if (params.sample_rate() == output_sample_rate_)
422 fifo_.reset(new media::AudioFifo(
424 kBufferAudioData * params.frames_per_buffer()));
425 fifo_input_bus_ = media::AudioBus::Create(
426 params.channels(), params.frames_per_buffer());
427 resampler_.reset(new media::MultiChannelResampler(
429 static_cast<double>(params.sample_rate()) / output_sample_rate_,
430 params.frames_per_buffer(),
431 base::Bind(&CastAudioSink::ProvideData, base::Unretained(this))));
434 // Add this sink to the track. Data received from the track will be
435 // submitted to |frame_input|.
437 const scoped_refptr<media::cast::AudioFrameInput>& frame_input) {
438 DCHECK(!sink_added_);
441 // This member is written here and then accessed on the IO thread
442 // We will not get data until AddToAudioTrack is called so it is
443 // safe to access this member now.
444 frame_input_ = frame_input;
445 AddToAudioTrack(this, track_);
448 void ProvideData(int frame_delay, media::AudioBus* output_bus) {
449 fifo_->Consume(output_bus, 0, output_bus->frames());
453 blink::WebMediaStreamTrack track_;
455 CastRtpStream::ErrorCallback error_callback_;
456 base::WeakPtrFactory<CastAudioSink> weak_factory_;
458 const int output_channels_;
459 const int output_sample_rate_;
461 // These member are accessed on the real-time audio time only.
462 scoped_refptr<media::cast::AudioFrameInput> frame_input_;
463 scoped_ptr<media::MultiChannelResampler> resampler_;
464 scoped_ptr<media::AudioFifo> fifo_;
465 scoped_ptr<media::AudioBus> fifo_input_bus_;
468 DISALLOW_COPY_AND_ASSIGN(CastAudioSink);
471 CastRtpParams::CastRtpParams(const CastRtpPayloadParams& payload_params)
472 : payload(payload_params) {}
474 CastCodecSpecificParams::CastCodecSpecificParams() {}
476 CastCodecSpecificParams::~CastCodecSpecificParams() {}
478 CastRtpPayloadParams::CastRtpPayloadParams()
492 CastRtpPayloadParams::~CastRtpPayloadParams() {}
494 CastRtpParams::CastRtpParams() {}
496 CastRtpParams::~CastRtpParams() {}
498 CastRtpStream::CastRtpStream(const blink::WebMediaStreamTrack& track,
499 const scoped_refptr<CastSession>& session)
500 : track_(track), cast_session_(session), weak_factory_(this) {}
502 CastRtpStream::~CastRtpStream() {}
504 std::vector<CastRtpParams> CastRtpStream::GetSupportedParams() {
506 return SupportedAudioParams();
508 return SupportedVideoParams();
511 CastRtpParams CastRtpStream::GetParams() { return params_; }
513 void CastRtpStream::Start(const CastRtpParams& params,
514 const base::Closure& start_callback,
515 const base::Closure& stop_callback,
516 const ErrorCallback& error_callback) {
517 VLOG(1) << "CastRtpStream::Start = " << (IsAudio() ? "audio" : "video");
518 stop_callback_ = stop_callback;
519 error_callback_ = error_callback;
522 AudioSenderConfig config;
523 if (!ToAudioSenderConfig(params, &config)) {
524 DidEncounterError("Invalid parameters for audio.");
528 // In case of error we have to go through DidEncounterError() to stop
529 // the streaming after reporting the error.
530 audio_sink_.reset(new CastAudioSink(
532 media::BindToCurrentLoop(base::Bind(&CastRtpStream::DidEncounterError,
533 weak_factory_.GetWeakPtr())),
534 params.payload.channels,
535 params.payload.clock_rate));
536 cast_session_->StartAudio(
538 base::Bind(&CastAudioSink::AddToTrack, audio_sink_->AsWeakPtr()),
539 base::Bind(&CastRtpStream::DidEncounterError,
540 weak_factory_.GetWeakPtr()));
541 start_callback.Run();
543 VideoSenderConfig config;
544 if (!ToVideoSenderConfig(params, &config)) {
545 DidEncounterError("Invalid parameters for video.");
548 // See the code for audio above for explanation of callbacks.
549 video_sink_.reset(new CastVideoSink(
551 gfx::Size(config.width, config.height),
552 media::BindToCurrentLoop(base::Bind(&CastRtpStream::DidEncounterError,
553 weak_factory_.GetWeakPtr()))));
554 cast_session_->StartVideo(
556 base::Bind(&CastVideoSink::AddToTrack, video_sink_->AsWeakPtr()),
557 base::Bind(&CastRtpStream::DidEncounterError,
558 weak_factory_.GetWeakPtr()));
559 start_callback.Run();
563 void CastRtpStream::Stop() {
564 VLOG(1) << "CastRtpStream::Stop = " << (IsAudio() ? "audio" : "video");
567 if (!stop_callback_.is_null())
568 stop_callback_.Run();
571 void CastRtpStream::ToggleLogging(bool enable) {
572 VLOG(1) << "CastRtpStream::ToggleLogging(" << enable << ") = "
573 << (IsAudio() ? "audio" : "video");
574 cast_session_->ToggleLogging(IsAudio(), enable);
577 void CastRtpStream::GetRawEvents(
578 const base::Callback<void(scoped_ptr<base::BinaryValue>)>& callback,
579 const std::string& extra_data) {
580 VLOG(1) << "CastRtpStream::GetRawEvents = "
581 << (IsAudio() ? "audio" : "video");
582 cast_session_->GetEventLogsAndReset(IsAudio(), extra_data, callback);
585 void CastRtpStream::GetStats(
586 const base::Callback<void(scoped_ptr<base::DictionaryValue>)>& callback) {
587 VLOG(1) << "CastRtpStream::GetStats = "
588 << (IsAudio() ? "audio" : "video");
589 cast_session_->GetStatsAndReset(IsAudio(), callback);
592 bool CastRtpStream::IsAudio() const {
593 return track_.source().type() == blink::WebMediaStreamSource::TypeAudio;
596 void CastRtpStream::DidEncounterError(const std::string& message) {
597 VLOG(1) << "CastRtpStream::DidEncounterError(" << message << ") = "
598 << (IsAudio() ? "audio" : "video");
599 // Save the WeakPtr first because the error callback might delete this object.
600 base::WeakPtr<CastRtpStream> ptr = weak_factory_.GetWeakPtr();
601 error_callback_.Run(message);
602 content::RenderThread::Get()->GetMessageLoop()->PostTask(
604 base::Bind(&CastRtpStream::Stop, ptr));