1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // This program benchmarks the theoretical throughput of the cast library.
6 // It runs using a fake clock, simulated network and fake codecs. This allows
7 // tests to run much faster than real time.
8 // To run the program, run:
9 // $ ./out/Release/cast_benchmarks | tee benchmarkoutput.asc
10 // This may take a while, when it is done, you can view the data with
11 // meshlab by running:
12 // $ meshlab benchmarkoutput.asc
13 // After starting meshlab, turn on Render->Show Axis. The red axis will
14 // represent bandwidth (in megabits) the blue axis will be packet drop
15 // (in percent) and the green axis will be latency (in milliseconds).
17 // This program can also be used for profiling. On linux it has
18 // built-in support for this. Simply set the environment variable
19 // PROFILE_FILE before running it, like so:
20 // $ export PROFILE_FILE=cast_benchmark.profile
21 // Then after running the program, you can view the profile with:
22 // $ pprof ./out/Release/cast_benchmarks $PROFILE_FILE --gv
30 #include "base/at_exit.h"
31 #include "base/bind.h"
32 #include "base/bind_helpers.h"
33 #include "base/command_line.h"
34 #include "base/debug/profiler.h"
35 #include "base/stl_util.h"
36 #include "base/strings/string_number_conversions.h"
37 #include "base/strings/stringprintf.h"
38 #include "base/test/simple_test_tick_clock.h"
39 #include "base/threading/thread.h"
40 #include "base/time/tick_clock.h"
41 #include "media/base/audio_bus.h"
42 #include "media/base/video_frame.h"
43 #include "media/cast/cast_config.h"
44 #include "media/cast/cast_environment.h"
45 #include "media/cast/cast_receiver.h"
46 #include "media/cast/cast_sender.h"
47 #include "media/cast/logging/simple_event_subscriber.h"
48 #include "media/cast/net/cast_transport_config.h"
49 #include "media/cast/net/cast_transport_defines.h"
50 #include "media/cast/net/cast_transport_sender.h"
51 #include "media/cast/net/cast_transport_sender_impl.h"
52 #include "media/cast/test/fake_single_thread_task_runner.h"
53 #include "media/cast/test/loopback_transport.h"
54 #include "media/cast/test/skewed_single_thread_task_runner.h"
55 #include "media/cast/test/skewed_tick_clock.h"
56 #include "media/cast/test/utility/audio_utility.h"
57 #include "media/cast/test/utility/default_config.h"
58 #include "media/cast/test/utility/test_util.h"
59 #include "media/cast/test/utility/udp_proxy.h"
60 #include "media/cast/test/utility/video_utility.h"
61 #include "testing/gtest/include/gtest/gtest.h"
68 static const int64 kStartMillisecond = INT64_C(1245);
69 static const int kAudioChannels = 2;
70 static const int kVideoHdWidth = 1280;
71 static const int kVideoHdHeight = 720;
72 static const int kTargetPlayoutDelayMs = 300;
74 // The tests are commonly implemented with |kFrameTimerMs| RunTask function;
75 // a normal video is 30 fps hence the 33 ms between frames.
76 static const int kFrameTimerMs = 33;
78 void UpdateCastTransportStatus(CastTransportStatus status) {
79 bool result = (status == TRANSPORT_AUDIO_INITIALIZED ||
80 status == TRANSPORT_VIDEO_INITIALIZED);
84 void AudioInitializationStatus(CastInitializationStatus status) {
85 EXPECT_EQ(STATUS_AUDIO_INITIALIZED, status);
88 void VideoInitializationStatus(CastInitializationStatus status) {
89 EXPECT_EQ(STATUS_VIDEO_INITIALIZED, status);
92 void IgnoreRawEvents(const std::vector<PacketEvent>& packet_events,
93 const std::vector<FrameEvent>& frame_events) {
98 // Wraps a CastTransportSender and records some statistics about
99 // the data that goes through it.
100 class CastTransportSenderWrapper : public CastTransportSender {
102 // Takes ownership of |transport|.
103 void Init(CastTransportSender* transport,
104 uint64* encoded_video_bytes,
105 uint64* encoded_audio_bytes) {
106 transport_.reset(transport);
107 encoded_video_bytes_ = encoded_video_bytes;
108 encoded_audio_bytes_ = encoded_audio_bytes;
111 virtual void InitializeAudio(
112 const CastTransportRtpConfig& config,
113 const RtcpCastMessageCallback& cast_message_cb,
114 const RtcpRttCallback& rtt_cb) OVERRIDE {
115 transport_->InitializeAudio(config, cast_message_cb, rtt_cb);
118 virtual void InitializeVideo(
119 const CastTransportRtpConfig& config,
120 const RtcpCastMessageCallback& cast_message_cb,
121 const RtcpRttCallback& rtt_cb) OVERRIDE {
122 transport_->InitializeVideo(config, cast_message_cb, rtt_cb);
125 virtual void InsertCodedAudioFrame(
126 const EncodedFrame& audio_frame) OVERRIDE {
127 *encoded_audio_bytes_ += audio_frame.data.size();
128 transport_->InsertCodedAudioFrame(audio_frame);
131 virtual void InsertCodedVideoFrame(
132 const EncodedFrame& video_frame) OVERRIDE {
133 *encoded_video_bytes_ += video_frame.data.size();
134 transport_->InsertCodedVideoFrame(video_frame);
137 virtual void SendSenderReport(
139 base::TimeTicks current_time,
140 uint32 current_time_as_rtp_timestamp) OVERRIDE {
141 transport_->SendSenderReport(ssrc,
143 current_time_as_rtp_timestamp);
146 // Retransmission request.
147 virtual void ResendPackets(
149 const MissingFramesAndPacketsMap& missing_packets,
150 bool cancel_rtx_if_not_in_list,
151 base::TimeDelta dedupe_window) OVERRIDE {
152 transport_->ResendPackets(
153 is_audio, missing_packets, cancel_rtx_if_not_in_list, dedupe_window);
156 virtual PacketReceiverCallback PacketReceiverForTesting() OVERRIDE {
157 return transport_->PacketReceiverForTesting();
161 scoped_ptr<CastTransportSender> transport_;
162 uint64* encoded_video_bytes_;
163 uint64* encoded_audio_bytes_;
166 struct MeasuringPoint {
167 MeasuringPoint(double bitrate_, double latency_, double percent_packet_drop_)
170 percent_packet_drop(percent_packet_drop_) {}
171 bool operator<=(const MeasuringPoint& other) const {
172 return bitrate >= other.bitrate && latency <= other.latency &&
173 percent_packet_drop <= other.percent_packet_drop;
175 bool operator>=(const MeasuringPoint& other) const {
176 return bitrate <= other.bitrate && latency >= other.latency &&
177 percent_packet_drop >= other.percent_packet_drop;
180 std::string AsString() const {
181 return base::StringPrintf(
182 "%f Mbit/s %f ms %f %% ", bitrate, latency, percent_packet_drop);
187 double percent_packet_drop;
190 class RunOneBenchmark {
194 task_runner_(new test::FakeSingleThreadTaskRunner(&testing_clock_)),
195 testing_clock_sender_(new test::SkewedTickClock(&testing_clock_)),
197 new test::SkewedSingleThreadTaskRunner(task_runner_)),
198 testing_clock_receiver_(new test::SkewedTickClock(&testing_clock_)),
199 task_runner_receiver_(
200 new test::SkewedSingleThreadTaskRunner(task_runner_)),
201 cast_environment_sender_(new CastEnvironment(
202 scoped_ptr<base::TickClock>(testing_clock_sender_).Pass(),
205 task_runner_sender_)),
206 cast_environment_receiver_(new CastEnvironment(
207 scoped_ptr<base::TickClock>(testing_clock_receiver_).Pass(),
208 task_runner_receiver_,
209 task_runner_receiver_,
210 task_runner_receiver_)),
211 receiver_to_sender_(cast_environment_receiver_),
212 sender_to_receiver_(cast_environment_sender_),
213 video_bytes_encoded_(0),
214 audio_bytes_encoded_(0),
216 testing_clock_.Advance(
217 base::TimeDelta::FromMilliseconds(kStartMillisecond));
220 void Configure(Codec video_codec,
222 int audio_sampling_frequency,
223 int max_number_of_video_buffers_used) {
224 audio_sender_config_.ssrc = 1;
225 audio_sender_config_.incoming_feedback_ssrc = 2;
226 audio_sender_config_.target_playout_delay =
227 base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs);
228 audio_sender_config_.rtp_payload_type = 96;
229 audio_sender_config_.use_external_encoder = false;
230 audio_sender_config_.frequency = audio_sampling_frequency;
231 audio_sender_config_.channels = kAudioChannels;
232 audio_sender_config_.bitrate = kDefaultAudioEncoderBitrate;
233 audio_sender_config_.codec = audio_codec;
235 audio_receiver_config_.feedback_ssrc =
236 audio_sender_config_.incoming_feedback_ssrc;
237 audio_receiver_config_.incoming_ssrc = audio_sender_config_.ssrc;
238 audio_receiver_config_.rtp_payload_type =
239 audio_sender_config_.rtp_payload_type;
240 audio_receiver_config_.frequency = audio_sender_config_.frequency;
241 audio_receiver_config_.channels = kAudioChannels;
242 audio_receiver_config_.max_frame_rate = 100;
243 audio_receiver_config_.codec = audio_sender_config_.codec;
244 audio_receiver_config_.rtp_max_delay_ms = kTargetPlayoutDelayMs;
246 video_sender_config_.ssrc = 3;
247 video_sender_config_.incoming_feedback_ssrc = 4;
248 video_sender_config_.target_playout_delay =
249 base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs);
250 video_sender_config_.rtp_payload_type = 97;
251 video_sender_config_.use_external_encoder = false;
252 video_sender_config_.width = kVideoHdWidth;
253 video_sender_config_.height = kVideoHdHeight;
255 video_sender_config_.max_bitrate = 10000000; // 10Mbit max
256 video_sender_config_.min_bitrate = 1000000; // 1Mbit min
257 video_sender_config_.start_bitrate = 1000000; // 1Mbit start
259 video_sender_config_.max_bitrate = 4000000; // 4Mbit all the time
260 video_sender_config_.min_bitrate = 4000000;
261 video_sender_config_.start_bitrate = 4000000;
263 video_sender_config_.max_qp = 56;
264 video_sender_config_.min_qp = 4;
265 video_sender_config_.max_frame_rate = 30;
266 video_sender_config_.max_number_of_video_buffers_used =
267 max_number_of_video_buffers_used;
268 video_sender_config_.codec = video_codec;
270 video_receiver_config_.feedback_ssrc =
271 video_sender_config_.incoming_feedback_ssrc;
272 video_receiver_config_.incoming_ssrc = video_sender_config_.ssrc;
273 video_receiver_config_.rtp_payload_type =
274 video_sender_config_.rtp_payload_type;
275 video_receiver_config_.codec = video_sender_config_.codec;
276 video_receiver_config_.frequency = kVideoFrequency;
277 video_receiver_config_.channels = 1;
278 video_receiver_config_.max_frame_rate = 100;
279 video_receiver_config_.rtp_max_delay_ms = kTargetPlayoutDelayMs;
282 void SetSenderClockSkew(double skew, base::TimeDelta offset) {
283 testing_clock_sender_->SetSkew(skew, offset);
284 task_runner_sender_->SetSkew(1.0 / skew);
287 void SetReceiverClockSkew(double skew, base::TimeDelta offset) {
288 testing_clock_receiver_->SetSkew(skew, offset);
289 task_runner_receiver_->SetSkew(1.0 / skew);
292 void Create(const MeasuringPoint& p) {
293 cast_receiver_ = CastReceiver::Create(cast_environment_receiver_,
294 audio_receiver_config_,
295 video_receiver_config_,
296 &receiver_to_sender_);
297 net::IPEndPoint dummy_endpoint;
298 transport_sender_.Init(
299 new CastTransportSenderImpl(
301 testing_clock_sender_,
303 base::Bind(&UpdateCastTransportStatus),
304 base::Bind(&IgnoreRawEvents),
305 base::TimeDelta::FromSeconds(1),
307 &sender_to_receiver_),
308 &video_bytes_encoded_,
309 &audio_bytes_encoded_);
312 CastSender::Create(cast_environment_sender_, &transport_sender_);
314 // Initializing audio and video senders.
315 cast_sender_->InitializeAudio(audio_sender_config_,
316 base::Bind(&AudioInitializationStatus));
317 cast_sender_->InitializeVideo(video_sender_config_,
318 base::Bind(&VideoInitializationStatus),
319 CreateDefaultVideoEncodeAcceleratorCallback(),
320 CreateDefaultVideoEncodeMemoryCallback());
322 receiver_to_sender_.Initialize(
323 CreateSimplePipe(p).Pass(),
324 transport_sender_.PacketReceiverForTesting(),
325 task_runner_, &testing_clock_);
326 sender_to_receiver_.Initialize(
327 CreateSimplePipe(p).Pass(), cast_receiver_->packet_receiver(),
328 task_runner_, &testing_clock_);
331 virtual ~RunOneBenchmark() {
332 cast_sender_.reset();
333 cast_receiver_.reset();
334 task_runner_->RunTasks();
337 void SendFakeVideoFrame() {
339 cast_sender_->video_frame_input()->InsertRawVideoFrame(
340 media::VideoFrame::CreateBlackFrame(gfx::Size(2, 2)),
341 testing_clock_sender_->NowTicks());
344 void RunTasks(int ms) {
345 task_runner_->Sleep(base::TimeDelta::FromMilliseconds(ms));
348 void BasicPlayerGotVideoFrame(
349 const scoped_refptr<media::VideoFrame>& video_frame,
350 const base::TimeTicks& render_time,
352 video_ticks_.push_back(
353 std::make_pair(testing_clock_receiver_->NowTicks(), render_time));
354 cast_receiver_->RequestDecodedVideoFrame(base::Bind(
355 &RunOneBenchmark::BasicPlayerGotVideoFrame, base::Unretained(this)));
358 void BasicPlayerGotAudioFrame(scoped_ptr<AudioBus> audio_bus,
359 const base::TimeTicks& playout_time,
360 bool is_continuous) {
361 audio_ticks_.push_back(
362 std::make_pair(testing_clock_receiver_->NowTicks(), playout_time));
363 cast_receiver_->RequestDecodedAudioFrame(base::Bind(
364 &RunOneBenchmark::BasicPlayerGotAudioFrame, base::Unretained(this)));
367 void StartBasicPlayer() {
368 cast_receiver_->RequestDecodedVideoFrame(base::Bind(
369 &RunOneBenchmark::BasicPlayerGotVideoFrame, base::Unretained(this)));
370 cast_receiver_->RequestDecodedAudioFrame(base::Bind(
371 &RunOneBenchmark::BasicPlayerGotAudioFrame, base::Unretained(this)));
374 scoped_ptr<test::PacketPipe> CreateSimplePipe(const MeasuringPoint& p) {
375 scoped_ptr<test::PacketPipe> pipe = test::NewBuffer(65536, p.bitrate);
377 test::NewRandomDrop(p.percent_packet_drop / 100.0).Pass());
378 pipe->AppendToPipe(test::NewConstantDelay(p.latency / 1000.0));
382 void Run(const MeasuringPoint& p) {
383 available_bitrate_ = p.bitrate;
385 CODEC_VIDEO_FAKE, CODEC_AUDIO_PCM16, 32000, 1);
389 for (int frame = 0; frame < 1000; frame++) {
390 SendFakeVideoFrame();
391 RunTasks(kFrameTimerMs);
393 RunTasks(100 * kFrameTimerMs); // Empty the pipeline.
394 VLOG(1) << "=============INPUTS============";
395 VLOG(1) << "Bitrate: " << p.bitrate << " mbit/s";
396 VLOG(1) << "Latency: " << p.latency << " ms";
397 VLOG(1) << "Packet drop drop: " << p.percent_packet_drop << "%";
398 VLOG(1) << "=============OUTPUTS============";
399 VLOG(1) << "Frames lost: " << frames_lost();
400 VLOG(1) << "Late frames: " << late_frames();
401 VLOG(1) << "Playout margin: " << frame_playout_buffer().AsString();
402 VLOG(1) << "Video bandwidth used: " << video_bandwidth() << " mbit/s ("
403 << (video_bandwidth() * 100 / desired_video_bitrate()) << "%)";
404 VLOG(1) << "Good run: " << SimpleGood();
408 int frames_lost() const { return frames_sent_ - video_ticks_.size(); }
410 int late_frames() const {
412 // Ignore the first two seconds of video or so.
413 for (size_t i = 60; i < video_ticks_.size(); i++) {
414 if (video_ticks_[i].first > video_ticks_[i].second) {
421 test::MeanAndError frame_playout_buffer() const {
422 std::vector<double> values;
423 for (size_t i = 0; i < video_ticks_.size(); i++) {
425 (video_ticks_[i].second - video_ticks_[i].first).InMillisecondsF());
427 return test::MeanAndError(values);
431 double video_bandwidth() const {
432 double seconds = (kFrameTimerMs * frames_sent_ / 1000.0);
433 double megabits = video_bytes_encoded_ * 8 / 1000000.0;
434 return megabits / seconds;
438 double audio_bandwidth() const {
439 double seconds = (kFrameTimerMs * frames_sent_ / 1000.0);
440 double megabits = audio_bytes_encoded_ * 8 / 1000000.0;
441 return megabits / seconds;
444 double desired_video_bitrate() {
445 return std::min<double>(available_bitrate_,
446 video_sender_config_.max_bitrate / 1000000.0);
450 return frames_lost() <= 1 && late_frames() <= 1 &&
451 video_bandwidth() > desired_video_bitrate() * 0.8 &&
452 video_bandwidth() < desired_video_bitrate() * 1.2;
456 FrameReceiverConfig audio_receiver_config_;
457 FrameReceiverConfig video_receiver_config_;
458 AudioSenderConfig audio_sender_config_;
459 VideoSenderConfig video_sender_config_;
461 base::TimeTicks start_time_;
463 // These run in "test time"
464 base::SimpleTestTickClock testing_clock_;
465 scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_;
467 // These run on the sender timeline.
468 test::SkewedTickClock* testing_clock_sender_;
469 scoped_refptr<test::SkewedSingleThreadTaskRunner> task_runner_sender_;
471 // These run on the receiver timeline.
472 test::SkewedTickClock* testing_clock_receiver_;
473 scoped_refptr<test::SkewedSingleThreadTaskRunner> task_runner_receiver_;
475 scoped_refptr<CastEnvironment> cast_environment_sender_;
476 scoped_refptr<CastEnvironment> cast_environment_receiver_;
478 LoopBackTransport receiver_to_sender_;
479 LoopBackTransport sender_to_receiver_;
480 CastTransportSenderWrapper transport_sender_;
481 uint64 video_bytes_encoded_;
482 uint64 audio_bytes_encoded_;
484 scoped_ptr<CastReceiver> cast_receiver_;
485 scoped_ptr<CastSender> cast_sender_;
488 double available_bitrate_;
489 std::vector<std::pair<base::TimeTicks, base::TimeTicks> > audio_ticks_;
490 std::vector<std::pair<base::TimeTicks, base::TimeTicks> > video_ticks_;
493 enum CacheResult { FOUND_TRUE, FOUND_FALSE, NOT_FOUND };
496 class BenchmarkCache {
498 CacheResult Lookup(const T& x) {
499 base::AutoLock key(lock_);
500 for (size_t i = 0; i < results_.size(); i++) {
501 if (results_[i].second) {
502 if (x <= results_[i].first) {
503 VLOG(2) << "TRUE because: " << x.AsString()
504 << " <= " << results_[i].first.AsString();
508 if (x >= results_[i].first) {
509 VLOG(2) << "FALSE because: " << x.AsString()
510 << " >= " << results_[i].first.AsString();
518 void Add(const T& x, bool result) {
519 base::AutoLock key(lock_);
520 VLOG(2) << "Cache Insert: " << x.AsString() << " = " << result;
521 results_.push_back(std::make_pair(x, result));
526 std::vector<std::pair<T, bool> > results_;
529 struct SearchVariable {
530 SearchVariable() : base(0.0), grade(0.0) {}
531 SearchVariable(double b, double g) : base(b), grade(g) {}
532 SearchVariable blend(const SearchVariable& other, double factor) {
534 CHECK_LE(factor, 1.0);
535 return SearchVariable(base * (1 - factor) + other.base * factor,
536 grade * (1 - factor) + other.grade * factor);
538 double value(double x) const { return base + grade * x; }
543 struct SearchVector {
544 SearchVector blend(const SearchVector& other, double factor) {
546 ret.bitrate = bitrate.blend(other.bitrate, factor);
547 ret.latency = latency.blend(other.latency, factor);
548 ret.packet_drop = packet_drop.blend(other.packet_drop, factor);
552 SearchVector average(const SearchVector& other) {
553 return blend(other, 0.5);
556 MeasuringPoint GetMeasuringPoint(double v) const {
557 return MeasuringPoint(
558 bitrate.value(-v), latency.value(v), packet_drop.value(v));
560 std::string AsString(double v) { return GetMeasuringPoint(v).AsString(); }
562 SearchVariable bitrate;
563 SearchVariable latency;
564 SearchVariable packet_drop;
567 class CastBenchmark {
569 bool RunOnePoint(const SearchVector& v, double multiplier) {
570 MeasuringPoint p = v.GetMeasuringPoint(multiplier);
571 VLOG(1) << "RUN: v = " << multiplier << " p = " << p.AsString();
572 if (p.bitrate <= 0) {
575 switch (cache_.Lookup(p)) {
585 for (int tries = 0; tries < 3 && result; tries++) {
586 RunOneBenchmark benchmark;
588 result &= benchmark.SimpleGood();
590 cache_.Add(p, result);
594 void BinarySearch(SearchVector v, double accuracy) {
597 while (RunOnePoint(v, max)) {
602 while (max - min > accuracy) {
603 double avg = (min + max) / 2;
604 if (RunOnePoint(v, avg)) {
611 // Print a data point to stdout.
612 base::AutoLock key(lock_);
613 MeasuringPoint p = v.GetMeasuringPoint(min);
614 fprintf(stdout, "%f %f %f\n", p.bitrate, p.latency, p.percent_packet_drop);
618 void SpanningSearch(int max,
626 std::vector<linked_ptr<base::Thread> >* threads) {
627 static int thread_num = 0;
631 SearchVector ab = a.blend(b, static_cast<double>(x) / max);
632 SearchVector ac = a.blend(c, static_cast<double>(x) / max);
633 SearchVector v = ab.blend(ac, x == y ? 1.0 : static_cast<double>(y) / x);
635 (*threads)[thread_num % threads->size()]->message_loop()->PostTask(
637 base::Bind(&CastBenchmark::BinarySearch,
638 base::Unretained(this),
643 SpanningSearch(max, x, y, skip, a, b, c, accuracy, threads);
644 SpanningSearch(max, x + skip, y + skip, skip, a, b, c, accuracy, threads);
645 SpanningSearch(max, x + skip, y, skip, a, b, c, accuracy, threads);
646 SpanningSearch(max, x, y + skip, skip, a, b, c, accuracy, threads);
653 std::vector<linked_ptr<base::Thread> > threads;
654 for (int i = 0; i < 16; i++) {
655 threads.push_back(make_linked_ptr(new base::Thread(
656 base::StringPrintf("cast_bench_thread_%d", i))));
660 if (CommandLine::ForCurrentProcess()->HasSwitch("single-run")) {
662 a.bitrate.base = 100.0;
663 a.bitrate.grade = 1.0;
664 a.latency.grade = 1.0;
665 a.packet_drop.grade = 1.0;
666 threads[0]->message_loop()->PostTask(
668 base::Bind(base::IgnoreResult(&CastBenchmark::RunOnePoint),
669 base::Unretained(this),
673 SearchVector a, b, c;
674 a.bitrate.base = b.bitrate.base = c.bitrate.base = 100.0;
675 a.bitrate.grade = 1.0;
676 b.latency.grade = 1.0;
677 c.packet_drop.grade = 1.0;
690 for (size_t i = 0; i < threads.size(); i++) {
696 BenchmarkCache<MeasuringPoint> cache_;
703 int main(int argc, char** argv) {
704 base::AtExitManager at_exit;
705 CommandLine::Init(argc, argv);
706 media::cast::CastBenchmark benchmark;
707 if (getenv("PROFILE_FILE")) {
708 std::string profile_file(getenv("PROFILE_FILE"));
709 base::debug::StartProfiling(profile_file);
711 base::debug::StopProfiling();