src/third_party/webrtc/video/rampup_tests.cc

   1 /*
   2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "testing/gtest/include/gtest/gtest.h"
  12 #include "webrtc/modules/rtp_rtcp/interface/receive_statistics.h"
  13 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
  14 #include "webrtc/modules/rtp_rtcp/interface/rtp_payload_registry.h"
  15 #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h"
  16 #include "webrtc/modules/rtp_rtcp/source/rtcp_utility.h"
  17 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
  18 #include "webrtc/test/testsupport/perf_test.h"
  19 #include "webrtc/video/rampup_tests.h"
  20
  21 namespace webrtc {
  22 namespace {
  23
  24 static const int kMaxPacketSize = 1500;
  25
  26 std::vector<uint32_t> GenerateSsrcs(size_t num_streams,
  27                                     uint32_t ssrc_offset) {
  28   std::vector<uint32_t> ssrcs;
  29   for (size_t i = 0; i != num_streams; ++i)
  30     ssrcs.push_back(static_cast<uint32_t>(ssrc_offset + i));
  31   return ssrcs;
  32 }
  33 }  // namespace
  34
  35 StreamObserver::StreamObserver(const SsrcMap& rtx_media_ssrcs,
  36                                newapi::Transport* feedback_transport,
  37                                Clock* clock,
  38                                RemoteBitrateEstimatorFactory* rbe_factory,
  39                                RateControlType control_type)
  40     : clock_(clock),
  41       test_done_(EventWrapper::Create()),
  42       rtp_parser_(RtpHeaderParser::Create()),
  43       feedback_transport_(feedback_transport),
  44       receive_stats_(ReceiveStatistics::Create(clock)),
  45       payload_registry_(
  46           new RTPPayloadRegistry(RTPPayloadStrategy::CreateStrategy(false))),
  47       crit_(CriticalSectionWrapper::CreateCriticalSection()),
  48       expected_bitrate_bps_(0),
  49       start_bitrate_bps_(0),
  50       rtx_media_ssrcs_(rtx_media_ssrcs),
  51       total_sent_(0),
  52       padding_sent_(0),
  53       rtx_media_sent_(0),
  54       total_packets_sent_(0),
  55       padding_packets_sent_(0),
  56       rtx_media_packets_sent_(0),
  57       test_start_ms_(clock_->TimeInMilliseconds()),
  58       ramp_up_finished_ms_(0) {
  59   // Ideally we would only have to instantiate an RtcpSender, an
  60   // RtpHeaderParser and a RemoteBitrateEstimator here, but due to the current
  61   // state of the RTP module we need a full module and receive statistics to
  62   // be able to produce an RTCP with REMB.
  63   RtpRtcp::Configuration config;
  64   config.receive_statistics = receive_stats_.get();
  65   feedback_transport_.Enable();
  66   config.outgoing_transport = &feedback_transport_;
  67   rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(config));
  68   rtp_rtcp_->SetREMBStatus(true);
  69   rtp_rtcp_->SetRTCPStatus(kRtcpNonCompound);
  70   rtp_parser_->RegisterRtpHeaderExtension(kRtpExtensionAbsoluteSendTime,
  71                                           kAbsSendTimeExtensionId);
  72   rtp_parser_->RegisterRtpHeaderExtension(kRtpExtensionTransmissionTimeOffset,
  73                                           kTransmissionTimeOffsetExtensionId);
  74   const uint32_t kRemoteBitrateEstimatorMinBitrateBps = 30000;
  75   remote_bitrate_estimator_.reset(
  76       rbe_factory->Create(this, clock, control_type,
  77                           kRemoteBitrateEstimatorMinBitrateBps));
  78 }
  79
  80 void StreamObserver::set_expected_bitrate_bps(
  81     unsigned int expected_bitrate_bps) {
  82   CriticalSectionScoped lock(crit_.get());
  83   expected_bitrate_bps_ = expected_bitrate_bps;
  84 }
  85
  86 void StreamObserver::set_start_bitrate_bps(unsigned int start_bitrate_bps) {
  87   CriticalSectionScoped lock(crit_.get());
  88   start_bitrate_bps_ = start_bitrate_bps;
  89 }
  90
  91 void StreamObserver::OnReceiveBitrateChanged(
  92     const std::vector<unsigned int>& ssrcs, unsigned int bitrate) {
  93   CriticalSectionScoped lock(crit_.get());
  94   assert(expected_bitrate_bps_ > 0);
  95   if (start_bitrate_bps_ != 0) {
  96     // For tests with an explicitly set start bitrate, verify the first
  97     // bitrate estimate is close to the start bitrate and lower than the
  98     // test target bitrate. This is to verify a call respects the configured
  99     // start bitrate, but due to the BWE implementation we can't guarantee the
 100     // first estimate really is as high as the start bitrate.
 101     EXPECT_GT(bitrate, 0.9 * start_bitrate_bps_);
 102     start_bitrate_bps_ = 0;
 103   }
 104   if (bitrate >= expected_bitrate_bps_) {
 105     ramp_up_finished_ms_ = clock_->TimeInMilliseconds();
 106     // Just trigger if there was any rtx padding packet.
 107     if (rtx_media_ssrcs_.empty() || rtx_media_sent_ > 0) {
 108       TriggerTestDone();
 109     }
 110   }
 111   rtp_rtcp_->SetREMBData(
 112       bitrate, static_cast<uint8_t>(ssrcs.size()), &ssrcs[0]);
 113   rtp_rtcp_->Process();
 114 }
 115
 116 bool StreamObserver::SendRtp(const uint8_t* packet, size_t length) {
 117   CriticalSectionScoped lock(crit_.get());
 118   RTPHeader header;
 119   EXPECT_TRUE(rtp_parser_->Parse(packet, static_cast<int>(length), &header));
 120   receive_stats_->IncomingPacket(header, length, false);
 121   payload_registry_->SetIncomingPayloadType(header);
 122   remote_bitrate_estimator_->IncomingPacket(
 123       clock_->TimeInMilliseconds(), static_cast<int>(length - 12), header);
 124   if (remote_bitrate_estimator_->TimeUntilNextProcess() <= 0) {
 125     remote_bitrate_estimator_->Process();
 126   }
 127   total_sent_ += length;
 128   padding_sent_ += header.paddingLength;
 129   ++total_packets_sent_;
 130   if (header.paddingLength > 0)
 131     ++padding_packets_sent_;
 132   if (rtx_media_ssrcs_.find(header.ssrc) != rtx_media_ssrcs_.end()) {
 133     rtx_media_sent_ += length - header.headerLength - header.paddingLength;
 134     if (header.paddingLength == 0)
 135       ++rtx_media_packets_sent_;
 136     uint8_t restored_packet[kMaxPacketSize];
 137     uint8_t* restored_packet_ptr = restored_packet;
 138     int restored_length = static_cast<int>(length);
 139     payload_registry_->RestoreOriginalPacket(&restored_packet_ptr,
 140                                              packet,
 141                                              &restored_length,
 142                                              rtx_media_ssrcs_[header.ssrc],
 143                                              header);
 144     length = restored_length;
 145     EXPECT_TRUE(rtp_parser_->Parse(
 146         restored_packet, static_cast<int>(length), &header));
 147   } else {
 148     rtp_rtcp_->SetRemoteSSRC(header.ssrc);
 149   }
 150   return true;
 151 }
 152
 153 bool StreamObserver::SendRtcp(const uint8_t* packet, size_t length) {
 154   return true;
 155 }
 156
 157 EventTypeWrapper StreamObserver::Wait() { return test_done_->Wait(120 * 1000); }
 158
 159 void StreamObserver::ReportResult(const std::string& measurement,
 160                   size_t value,
 161                   const std::string& units) {
 162   webrtc::test::PrintResult(
 163       measurement, "",
 164       ::testing::UnitTest::GetInstance()->current_test_info()->name(),
 165       value, units, false);
 166 }
 167
 168 void StreamObserver::TriggerTestDone() EXCLUSIVE_LOCKS_REQUIRED(crit_) {
 169   ReportResult("ramp-up-total-sent", total_sent_, "bytes");
 170   ReportResult("ramp-up-padding-sent", padding_sent_, "bytes");
 171   ReportResult("ramp-up-rtx-media-sent", rtx_media_sent_, "bytes");
 172   ReportResult("ramp-up-total-packets-sent", total_packets_sent_, "packets");
 173   ReportResult("ramp-up-padding-packets-sent",
 174                padding_packets_sent_,
 175                "packets");
 176   ReportResult("ramp-up-rtx-packets-sent",
 177                rtx_media_packets_sent_,
 178                "packets");
 179   ReportResult("ramp-up-time",
 180                ramp_up_finished_ms_ - test_start_ms_,
 181                "milliseconds");
 182   test_done_->Set();
 183 }
 184
 185 LowRateStreamObserver::LowRateStreamObserver(
 186     newapi::Transport* feedback_transport,
 187     Clock* clock,
 188     size_t number_of_streams,
 189     bool rtx_used)
 190     : clock_(clock),
 191       number_of_streams_(number_of_streams),
 192       rtx_used_(rtx_used),
 193       test_done_(EventWrapper::Create()),
 194       rtp_parser_(RtpHeaderParser::Create()),
 195       feedback_transport_(feedback_transport),
 196       receive_stats_(ReceiveStatistics::Create(clock)),
 197       crit_(CriticalSectionWrapper::CreateCriticalSection()),
 198       send_stream_(NULL),
 199       test_state_(kFirstRampup),
 200       state_start_ms_(clock_->TimeInMilliseconds()),
 201       interval_start_ms_(state_start_ms_),
 202       last_remb_bps_(0),
 203       sent_bytes_(0),
 204       total_overuse_bytes_(0),
 205       suspended_in_stats_(false) {
 206   RtpRtcp::Configuration config;
 207   config.receive_statistics = receive_stats_.get();
 208   feedback_transport_.Enable();
 209   config.outgoing_transport = &feedback_transport_;
 210   rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(config));
 211   rtp_rtcp_->SetREMBStatus(true);
 212   rtp_rtcp_->SetRTCPStatus(kRtcpNonCompound);
 213   rtp_parser_->RegisterRtpHeaderExtension(kRtpExtensionTransmissionTimeOffset,
 214                                           kTransmissionTimeOffsetExtensionId);
 215   AbsoluteSendTimeRemoteBitrateEstimatorFactory rbe_factory;
 216   const uint32_t kRemoteBitrateEstimatorMinBitrateBps = 10000;
 217   remote_bitrate_estimator_.reset(
 218       rbe_factory.Create(this, clock, kMimdControl,
 219                          kRemoteBitrateEstimatorMinBitrateBps));
 220   forward_transport_config_.link_capacity_kbps =
 221       kHighBandwidthLimitBps / 1000;
 222   forward_transport_config_.queue_length_packets = 100;  // Something large.
 223   test::DirectTransport::SetConfig(forward_transport_config_);
 224   test::DirectTransport::SetReceiver(this);
 225 }
 226
 227 void LowRateStreamObserver::SetSendStream(const VideoSendStream* send_stream) {
 228   CriticalSectionScoped lock(crit_.get());
 229   send_stream_ = send_stream;
 230 }
 231
 232 void LowRateStreamObserver::OnReceiveBitrateChanged(
 233     const std::vector<unsigned int>& ssrcs,
 234     unsigned int bitrate) {
 235   CriticalSectionScoped lock(crit_.get());
 236   rtp_rtcp_->SetREMBData(
 237       bitrate, static_cast<uint8_t>(ssrcs.size()), &ssrcs[0]);
 238   rtp_rtcp_->Process();
 239   last_remb_bps_ = bitrate;
 240 }
 241
 242 bool LowRateStreamObserver::SendRtp(const uint8_t* data, size_t length) {
 243   CriticalSectionScoped lock(crit_.get());
 244   sent_bytes_ += length;
 245   int64_t now_ms = clock_->TimeInMilliseconds();
 246   if (now_ms > interval_start_ms_ + 1000) {  // Let at least 1 second pass.
 247     // Verify that the send rate was about right.
 248     unsigned int average_rate_bps = static_cast<unsigned int>(sent_bytes_) *
 249                                     8 * 1000 / (now_ms - interval_start_ms_);
 250     // TODO(holmer): Why is this failing?
 251     // EXPECT_LT(average_rate_bps, last_remb_bps_ * 1.1);
 252     if (average_rate_bps > last_remb_bps_ * 1.1) {
 253       total_overuse_bytes_ +=
 254           sent_bytes_ -
 255           last_remb_bps_ / 8 * (now_ms - interval_start_ms_) / 1000;
 256     }
 257     EvolveTestState(average_rate_bps);
 258     interval_start_ms_ = now_ms;
 259     sent_bytes_ = 0;
 260   }
 261   return test::DirectTransport::SendRtp(data, length);
 262 }
 263
 264 PacketReceiver::DeliveryStatus LowRateStreamObserver::DeliverPacket(
 265     const uint8_t* packet, size_t length) {
 266   CriticalSectionScoped lock(crit_.get());
 267   RTPHeader header;
 268   EXPECT_TRUE(rtp_parser_->Parse(packet, static_cast<int>(length), &header));
 269   receive_stats_->IncomingPacket(header, length, false);
 270   remote_bitrate_estimator_->IncomingPacket(
 271       clock_->TimeInMilliseconds(), static_cast<int>(length - 12), header);
 272   if (remote_bitrate_estimator_->TimeUntilNextProcess() <= 0) {
 273     remote_bitrate_estimator_->Process();
 274   }
 275   suspended_in_stats_ = send_stream_->GetStats().suspended;
 276   return DELIVERY_OK;
 277 }
 278
 279 bool LowRateStreamObserver::SendRtcp(const uint8_t* packet, size_t length) {
 280   return true;
 281 }
 282
 283 std::string LowRateStreamObserver::GetModifierString() {
 284   std::string str("_");
 285   char temp_str[5];
 286   sprintf(temp_str, "%i",
 287       static_cast<int>(number_of_streams_));
 288   str += std::string(temp_str);
 289   str += "stream";
 290   str += (number_of_streams_ > 1 ? "s" : "");
 291   str += "_";
 292   str += (rtx_used_ ? "" : "no");
 293   str += "rtx";
 294   return str;
 295 }
 296
 297 void LowRateStreamObserver::EvolveTestState(unsigned int bitrate_bps) {
 298   int64_t now = clock_->TimeInMilliseconds();
 299   CriticalSectionScoped lock(crit_.get());
 300   assert(send_stream_ != NULL);
 301   switch (test_state_) {
 302     case kFirstRampup: {
 303       EXPECT_FALSE(suspended_in_stats_);
 304       if (bitrate_bps > kExpectedHighBitrateBps) {
 305         // The first ramp-up has reached the target bitrate. Change the
 306         // channel limit, and move to the next test state.
 307         forward_transport_config_.link_capacity_kbps =
 308             kLowBandwidthLimitBps / 1000;
 309         test::DirectTransport::SetConfig(forward_transport_config_);
 310         test_state_ = kLowRate;
 311         webrtc::test::PrintResult("ramp_up_down_up",
 312                                   GetModifierString(),
 313                                   "first_rampup",
 314                                   now - state_start_ms_,
 315                                   "ms",
 316                                   false);
 317         state_start_ms_ = now;
 318         interval_start_ms_ = now;
 319         sent_bytes_ = 0;
 320       }
 321       break;
 322     }
 323     case kLowRate: {
 324       if (bitrate_bps < kExpectedLowBitrateBps && suspended_in_stats_) {
 325         // The ramp-down was successful. Change the channel limit back to a
 326         // high value, and move to the next test state.
 327         forward_transport_config_.link_capacity_kbps =
 328             kHighBandwidthLimitBps / 1000;
 329         test::DirectTransport::SetConfig(forward_transport_config_);
 330         test_state_ = kSecondRampup;
 331         webrtc::test::PrintResult("ramp_up_down_up",
 332                                   GetModifierString(),
 333                                   "rampdown",
 334                                   now - state_start_ms_,
 335                                   "ms",
 336                                   false);
 337         state_start_ms_ = now;
 338         interval_start_ms_ = now;
 339         sent_bytes_ = 0;
 340       }
 341       break;
 342     }
 343     case kSecondRampup: {
 344       if (bitrate_bps > kExpectedHighBitrateBps && !suspended_in_stats_) {
 345         webrtc::test::PrintResult("ramp_up_down_up",
 346                                   GetModifierString(),
 347                                   "second_rampup",
 348                                   now - state_start_ms_,
 349                                   "ms",
 350                                   false);
 351         webrtc::test::PrintResult("ramp_up_down_up",
 352                                   GetModifierString(),
 353                                   "total_overuse",
 354                                   total_overuse_bytes_,
 355                                   "bytes",
 356                                   false);
 357         test_done_->Set();
 358       }
 359       break;
 360     }
 361   }
 362 }
 363
 364 EventTypeWrapper LowRateStreamObserver::Wait() {
 365   return test_done_->Wait(test::CallTest::kLongTimeoutMs);
 366 }
 367
 368 void RampUpTest::RunRampUpTest(bool rtx,
 369                                size_t num_streams,
 370                                unsigned int start_bitrate_bps,
 371                                const std::string& extension_type) {
 372   std::vector<uint32_t> ssrcs(GenerateSsrcs(num_streams, 100));
 373   std::vector<uint32_t> rtx_ssrcs(GenerateSsrcs(num_streams, 200));
 374   StreamObserver::SsrcMap rtx_ssrc_map;
 375   if (rtx) {
 376     for (size_t i = 0; i < ssrcs.size(); ++i)
 377       rtx_ssrc_map[rtx_ssrcs[i]] = ssrcs[i];
 378   }
 379
 380   CreateSendConfig(num_streams);
 381
 382   scoped_ptr<RemoteBitrateEstimatorFactory> rbe_factory;
 383   RateControlType control_type;
 384   if (extension_type == RtpExtension::kAbsSendTime) {
 385     control_type = kAimdControl;
 386     rbe_factory.reset(new AbsoluteSendTimeRemoteBitrateEstimatorFactory);
 387     send_config_.rtp.extensions.push_back(RtpExtension(
 388         extension_type.c_str(), kAbsSendTimeExtensionId));
 389   } else {
 390     control_type = kMimdControl;
 391     rbe_factory.reset(new RemoteBitrateEstimatorFactory);
 392     send_config_.rtp.extensions.push_back(RtpExtension(
 393         extension_type.c_str(), kTransmissionTimeOffsetExtensionId));
 394   }
 395
 396   test::DirectTransport receiver_transport;
 397   StreamObserver stream_observer(rtx_ssrc_map,
 398                                  &receiver_transport,
 399                                  Clock::GetRealTimeClock(),
 400                                  rbe_factory.get(),
 401                                  control_type);
 402
 403   Call::Config call_config(&stream_observer);
 404   if (start_bitrate_bps != 0) {
 405     call_config.start_bitrate_bps = start_bitrate_bps;
 406     stream_observer.set_start_bitrate_bps(start_bitrate_bps);
 407   }
 408
 409   CreateSenderCall(call_config);
 410
 411   receiver_transport.SetReceiver(sender_call_->Receiver());
 412
 413   if (num_streams == 1) {
 414     encoder_config_.streams[0].target_bitrate_bps = 2000000;
 415     encoder_config_.streams[0].max_bitrate_bps = 2000000;
 416   }
 417
 418   send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
 419   send_config_.rtp.ssrcs = ssrcs;
 420   if (rtx) {
 421     send_config_.rtp.rtx.payload_type = kSendRtxPayloadType;
 422     send_config_.rtp.rtx.ssrcs = rtx_ssrcs;
 423     send_config_.rtp.rtx.pad_with_redundant_payloads = true;
 424   }
 425
 426   if (num_streams == 1) {
 427     // For single stream rampup until 1mbps
 428     stream_observer.set_expected_bitrate_bps(kSingleStreamTargetBps);
 429   } else {
 430     // For multi stream rampup until all streams are being sent. That means
 431     // enough birate to send all the target streams plus the min bitrate of
 432     // the last one.
 433     int expected_bitrate_bps = encoder_config_.streams.back().min_bitrate_bps;
 434     for (size_t i = 0; i < encoder_config_.streams.size() - 1; ++i) {
 435       expected_bitrate_bps += encoder_config_.streams[i].target_bitrate_bps;
 436     }
 437     stream_observer.set_expected_bitrate_bps(expected_bitrate_bps);
 438   }
 439
 440   CreateStreams();
 441   CreateFrameGeneratorCapturer();
 442
 443   Start();
 444
 445   EXPECT_EQ(kEventSignaled, stream_observer.Wait());
 446
 447   Stop();
 448   DestroyStreams();
 449 }
 450
 451 void RampUpTest::RunRampUpDownUpTest(size_t number_of_streams, bool rtx) {
 452   test::DirectTransport receiver_transport;
 453   LowRateStreamObserver stream_observer(
 454       &receiver_transport, Clock::GetRealTimeClock(), number_of_streams, rtx);
 455
 456   Call::Config call_config(&stream_observer);
 457   CreateSenderCall(call_config);
 458   receiver_transport.SetReceiver(sender_call_->Receiver());
 459
 460   CreateSendConfig(number_of_streams);
 461
 462   send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
 463   send_config_.rtp.extensions.push_back(RtpExtension(
 464       RtpExtension::kTOffset, kTransmissionTimeOffsetExtensionId));
 465   send_config_.suspend_below_min_bitrate = true;
 466   if (rtx) {
 467     send_config_.rtp.rtx.payload_type = kSendRtxPayloadType;
 468     send_config_.rtp.rtx.ssrcs = GenerateSsrcs(number_of_streams, 200);
 469     send_config_.rtp.rtx.pad_with_redundant_payloads = true;
 470   }
 471
 472   CreateStreams();
 473   stream_observer.SetSendStream(send_stream_);
 474
 475   CreateFrameGeneratorCapturer();
 476
 477   Start();
 478
 479   EXPECT_EQ(kEventSignaled, stream_observer.Wait());
 480
 481   Stop();
 482   DestroyStreams();
 483 }
 484
 485 TEST_F(RampUpTest, SingleStream) {
 486   RunRampUpTest(false, 1, 0, RtpExtension::kTOffset);
 487 }
 488
 489 TEST_F(RampUpTest, Simulcast) {
 490   RunRampUpTest(false, 3, 0, RtpExtension::kTOffset);
 491 }
 492
 493 TEST_F(RampUpTest, SimulcastWithRtx) {
 494   RunRampUpTest(true, 3, 0, RtpExtension::kTOffset);
 495 }
 496
 497 TEST_F(RampUpTest, SingleStreamWithHighStartBitrate) {
 498   RunRampUpTest(false, 1, 0.9 * kSingleStreamTargetBps, RtpExtension::kTOffset);
 499 }
 500
 501 TEST_F(RampUpTest, UpDownUpOneStream) { RunRampUpDownUpTest(1, false); }
 502
 503 TEST_F(RampUpTest, UpDownUpThreeStreams) { RunRampUpDownUpTest(3, false); }
 504
 505 TEST_F(RampUpTest, UpDownUpOneStreamRtx) { RunRampUpDownUpTest(1, true); }
 506
 507 TEST_F(RampUpTest, UpDownUpThreeStreamsRtx) { RunRampUpDownUpTest(3, true); }
 508
 509 }  // namespace webrtc