test/cpp/server/load_reporter/load_reporter_test.cc

   1 /*
   2  *
   3  * Copyright 2018 gRPC authors.
   4  *
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  *
  17  */
  18
  19 #include <grpc/impl/codegen/port_platform.h>
  20
  21 #include <set>
  22 #include <vector>
  23
  24 #include <gmock/gmock.h>
  25 #include <grpc/grpc.h>
  26 #include <gtest/gtest.h>
  27
  28 #include "src/core/ext/filters/load_reporting/registered_opencensus_objects.h"
  29 #include "src/core/lib/iomgr/exec_ctx.h"
  30 #include "src/cpp/server/load_reporter/constants.h"
  31 #include "src/cpp/server/load_reporter/load_reporter.h"
  32 #include "test/core/util/port.h"
  33 #include "test/core/util/test_config.h"
  34
  35 #include "opencensus/stats/testing/test_utils.h"
  36
  37 namespace grpc {
  38 namespace testing {
  39 namespace {
  40
  41 using ::grpc::lb::v1::LoadBalancingFeedback;
  42 using ::grpc::load_reporter::CensusViewProvider;
  43 using ::grpc::load_reporter::CpuStatsProvider;
  44 using ::grpc::load_reporter::LoadReporter;
  45 using ::opencensus::stats::View;
  46 using ::opencensus::stats::ViewData;
  47 using ::opencensus::stats::ViewDataImpl;
  48 using ::opencensus::stats::ViewDescriptor;
  49 using ::testing::DoubleNear;
  50 using ::testing::Return;
  51
  52 constexpr uint64_t kFeedbackSampleWindowSeconds = 5;
  53 constexpr uint64_t kFetchAndSampleIntervalSeconds = 1;
  54 constexpr uint64_t kNumFeedbackSamplesInWindow =
  55     kFeedbackSampleWindowSeconds / kFetchAndSampleIntervalSeconds;
  56
  57 class MockCensusViewProvider : public CensusViewProvider {
  58  public:
  59   MOCK_METHOD0(FetchViewData, CensusViewProvider::ViewDataMap());
  60
  61   const ::opencensus::stats::ViewDescriptor& FindViewDescriptor(
  62       const std::string& view_name) {
  63     auto it = view_descriptor_map().find(view_name);
  64     GPR_ASSERT(it != view_descriptor_map().end());
  65     return it->second;
  66   }
  67 };
  68
  69 class MockCpuStatsProvider : public CpuStatsProvider {
  70  public:
  71   MOCK_METHOD0(GetCpuStats, CpuStatsProvider::CpuStatsSample());
  72 };
  73
  74 class LoadReporterTest : public ::testing::Test {
  75  public:
  76   LoadReporterTest() {}
  77
  78   MockCensusViewProvider* mock_census_view_provider() {
  79     return static_cast<MockCensusViewProvider*>(
  80         load_reporter_->census_view_provider());
  81   }
  82
  83   void PrepareCpuExpectation(size_t call_num) {
  84     auto mock_cpu_stats_provider = static_cast<MockCpuStatsProvider*>(
  85         load_reporter_->cpu_stats_provider());
  86     ::testing::InSequence s;
  87     for (size_t i = 0; i < call_num; ++i) {
  88       EXPECT_CALL(*mock_cpu_stats_provider, GetCpuStats())
  89           .WillOnce(Return(kCpuStatsSamples[i]))
  90           .RetiresOnSaturation();
  91     }
  92   }
  93
  94   CpuStatsProvider::CpuStatsSample initial_cpu_stats_{2, 20};
  95   const std::vector<CpuStatsProvider::CpuStatsSample> kCpuStatsSamples = {
  96       {13, 53},    {64, 96},     {245, 345},  {314, 785},
  97       {874, 1230}, {1236, 2145}, {1864, 2974}};
  98
  99   std::unique_ptr<LoadReporter> load_reporter_;
 100
 101   const std::string kHostname1 = "kHostname1";
 102   const std::string kHostname2 = "kHostname2";
 103   const std::string kHostname3 = "kHostname3";
 104   // Pad to the length of a valid LB ID.
 105   const std::string kLbId1 = "kLbId111";
 106   const std::string kLbId2 = "kLbId222";
 107   const std::string kLbId3 = "kLbId333";
 108   const std::string kLbId4 = "kLbId444";
 109   const std::string kLoadKey1 = "kLoadKey1";
 110   const std::string kLoadKey2 = "kLoadKey2";
 111   const std::string kLoadKey3 = "kLoadKey3";
 112   const std::string kLbTag1 = "kLbTag1";
 113   const std::string kLbTag2 = "kLbTag2";
 114   const std::string kLbToken1 = "kLbId111kLbTag1";
 115   const std::string kLbToken2 = "kLbId222kLbTag2";
 116   const std::string kUser1 = "kUser1";
 117   const std::string kUser2 = "kUser2";
 118   const std::string kUser3 = "kUser3";
 119   const std::string kClientIp0 = "00";
 120   const std::string kClientIp1 = "0800000001";
 121   const std::string kClientIp2 = "3200000000000000000000000000000002";
 122   const std::string kMetric1 = "kMetric1";
 123   const std::string kMetric2 = "kMetric2";
 124
 125  private:
 126   void SetUp() override {
 127     // Access the measures to make them valid.
 128     ::grpc::load_reporter::MeasureStartCount();
 129     ::grpc::load_reporter::MeasureEndCount();
 130     ::grpc::load_reporter::MeasureEndBytesSent();
 131     ::grpc::load_reporter::MeasureEndBytesReceived();
 132     ::grpc::load_reporter::MeasureEndLatencyMs();
 133     ::grpc::load_reporter::MeasureOtherCallMetric();
 134     // Set up the load reporter.
 135     auto mock_cpu = new MockCpuStatsProvider();
 136     auto mock_census = new MockCensusViewProvider();
 137     // Prepare the initial CPU stats data. Note that the expectation should be
 138     // set up before the load reporter is initialized, because CPU stats is
 139     // sampled at that point.
 140     EXPECT_CALL(*mock_cpu, GetCpuStats())
 141         .WillOnce(Return(initial_cpu_stats_))
 142         .RetiresOnSaturation();
 143     load_reporter_ = std::unique_ptr<LoadReporter>(
 144         new LoadReporter(kFeedbackSampleWindowSeconds,
 145                          std::unique_ptr<CensusViewProvider>(mock_census),
 146                          std::unique_ptr<CpuStatsProvider>(mock_cpu)));
 147   }
 148 };
 149
 150 class LbFeedbackTest : public LoadReporterTest {
 151  public:
 152   // Note that [start, start + count) of the fake samples (maybe plus the
 153   // initial record) are in the window now.
 154   void VerifyLbFeedback(const LoadBalancingFeedback& lb_feedback, size_t start,
 155                         size_t count) {
 156     const CpuStatsProvider::CpuStatsSample* base =
 157         start == 0 ? &initial_cpu_stats_ : &kCpuStatsSamples[start - 1];
 158     double expected_cpu_util =
 159         static_cast<double>(kCpuStatsSamples[start + count - 1].first -
 160                             base->first) /
 161         static_cast<double>(kCpuStatsSamples[start + count - 1].second -
 162                             base->second);
 163     ASSERT_THAT(static_cast<double>(lb_feedback.server_utilization()),
 164                 DoubleNear(expected_cpu_util, 0.00001));
 165     double qps_sum = 0, eps_sum = 0;
 166     for (size_t i = 0; i < count; ++i) {
 167       qps_sum += kQpsEpsSamples[start + i].first;
 168       eps_sum += kQpsEpsSamples[start + i].second;
 169     }
 170     double expected_qps = qps_sum / count;
 171     double expected_eps = eps_sum / count;
 172     // TODO(juanlishen): The error is big because we use sleep(). It should be
 173     // much smaller when we use fake clock.
 174     ASSERT_THAT(static_cast<double>(lb_feedback.calls_per_second()),
 175                 DoubleNear(expected_qps, expected_qps * 0.3));
 176     ASSERT_THAT(static_cast<double>(lb_feedback.errors_per_second()),
 177                 DoubleNear(expected_eps, expected_eps * 0.3));
 178     gpr_log(GPR_INFO,
 179             "Verified LB feedback matches the samples of index [%lu, %lu).",
 180             start, start + count);
 181   }
 182
 183   const std::vector<std::pair<double, double>> kQpsEpsSamples = {
 184       {546.1, 153.1},  {62.1, 54.1},   {578.1, 154.2}, {978.1, 645.1},
 185       {1132.1, 846.4}, {531.5, 315.4}, {874.1, 324.9}};
 186 };
 187
 188 TEST_F(LbFeedbackTest, ZeroDuration) {
 189   PrepareCpuExpectation(kCpuStatsSamples.size());
 190   EXPECT_CALL(*mock_census_view_provider(), FetchViewData())
 191       .WillRepeatedly(
 192           Return(::grpc::load_reporter::CensusViewProvider::ViewDataMap()));
 193   // Verify that divide-by-zero exception doesn't happen.
 194   for (size_t i = 0; i < kCpuStatsSamples.size(); ++i) {
 195     load_reporter_->FetchAndSample();
 196   }
 197   load_reporter_->GenerateLoadBalancingFeedback();
 198 }
 199
 200 TEST_F(LbFeedbackTest, Normal) {
 201   // Prepare view data list using the <QPS, EPS> samples.
 202   std::vector<CensusViewProvider::ViewDataMap> view_data_map_list;
 203   for (const auto& p : LbFeedbackTest::kQpsEpsSamples) {
 204     double qps = p.first;
 205     double eps = p.second;
 206     double ok_count = (qps - eps) * kFetchAndSampleIntervalSeconds;
 207     double error_count = eps * kFetchAndSampleIntervalSeconds;
 208     double ok_count_1 = ok_count / 3.0;
 209     double ok_count_2 = ok_count - ok_count_1;
 210     auto end_count_vd = ::opencensus::stats::testing::TestUtils::MakeViewData(
 211         mock_census_view_provider()->FindViewDescriptor(
 212             ::grpc::load_reporter::kViewEndCount),
 213         {{{kClientIp0 + kLbToken1, kHostname1, kUser1,
 214            ::grpc::load_reporter::kCallStatusOk},
 215           ok_count_1},
 216          {{kClientIp0 + kLbToken1, kHostname1, kUser2,
 217            ::grpc::load_reporter::kCallStatusOk},
 218           ok_count_2},
 219          {{kClientIp0 + kLbToken1, kHostname1, kUser1,
 220            ::grpc::load_reporter::kCallStatusClientError},
 221           error_count}});
 222     // Values for other view data don't matter.
 223     auto end_bytes_sent_vd =
 224         ::opencensus::stats::testing::TestUtils::MakeViewData(
 225             mock_census_view_provider()->FindViewDescriptor(
 226                 ::grpc::load_reporter::kViewEndBytesSent),
 227             {{{kClientIp0 + kLbToken1, kHostname1, kUser1,
 228                ::grpc::load_reporter::kCallStatusOk},
 229               0},
 230              {{kClientIp0 + kLbToken1, kHostname1, kUser2,
 231                ::grpc::load_reporter::kCallStatusOk},
 232               0},
 233              {{kClientIp0 + kLbToken1, kHostname1, kUser1,
 234                ::grpc::load_reporter::kCallStatusClientError},
 235               0}});
 236     auto end_bytes_received_vd =
 237         ::opencensus::stats::testing::TestUtils::MakeViewData(
 238             mock_census_view_provider()->FindViewDescriptor(
 239                 ::grpc::load_reporter::kViewEndBytesReceived),
 240             {{{kClientIp0 + kLbToken1, kHostname1, kUser1,
 241                ::grpc::load_reporter::kCallStatusOk},
 242               0},
 243              {{kClientIp0 + kLbToken1, kHostname1, kUser2,
 244                ::grpc::load_reporter::kCallStatusOk},
 245               0},
 246              {{kClientIp0 + kLbToken1, kHostname1, kUser1,
 247                ::grpc::load_reporter::kCallStatusClientError},
 248               0}});
 249     auto end_latency_vd = ::opencensus::stats::testing::TestUtils::MakeViewData(
 250         mock_census_view_provider()->FindViewDescriptor(
 251             ::grpc::load_reporter::kViewEndLatencyMs),
 252         {{{kClientIp0 + kLbToken1, kHostname1, kUser1,
 253            ::grpc::load_reporter::kCallStatusOk},
 254           0},
 255          {{kClientIp0 + kLbToken1, kHostname1, kUser2,
 256            ::grpc::load_reporter::kCallStatusOk},
 257           0},
 258          {{kClientIp0 + kLbToken1, kHostname1, kUser1,
 259            ::grpc::load_reporter::kCallStatusClientError},
 260           0}});
 261     view_data_map_list.push_back(
 262         {{::grpc::load_reporter::kViewEndCount, end_count_vd},
 263          {::grpc::load_reporter::kViewEndBytesSent, end_bytes_sent_vd},
 264          {::grpc::load_reporter::kViewEndBytesReceived, end_bytes_received_vd},
 265          {::grpc::load_reporter::kViewEndLatencyMs, end_latency_vd}});
 266   }
 267   {
 268     ::testing::InSequence s;
 269     for (size_t i = 0; i < view_data_map_list.size(); ++i) {
 270       EXPECT_CALL(*mock_census_view_provider(), FetchViewData())
 271           .WillOnce(Return(view_data_map_list[i]))
 272           .RetiresOnSaturation();
 273     }
 274   }
 275   PrepareCpuExpectation(kNumFeedbackSamplesInWindow + 2);
 276   // When the load reporter is created, a trivial LB feedback record is added.
 277   // But that's not enough for generating an LB feedback.
 278   // Fetch some view data so that non-trivial LB feedback can be generated.
 279   for (size_t i = 0; i < kNumFeedbackSamplesInWindow / 2; ++i) {
 280     // TODO(juanlishen): Find some fake clock to speed up testing.
 281     sleep(1);
 282     load_reporter_->FetchAndSample();
 283   }
 284   VerifyLbFeedback(load_reporter_->GenerateLoadBalancingFeedback(), 0,
 285                    kNumFeedbackSamplesInWindow / 2);
 286   // Fetch more view data so that the feedback record window is just full (the
 287   // initial record just falls out of the window).
 288   for (size_t i = 0; i < (kNumFeedbackSamplesInWindow + 1) / 2; ++i) {
 289     sleep(1);
 290     load_reporter_->FetchAndSample();
 291   }
 292   VerifyLbFeedback(load_reporter_->GenerateLoadBalancingFeedback(), 0,
 293                    kNumFeedbackSamplesInWindow);
 294   // Further fetching will cause the old records to fall out of the window.
 295   for (size_t i = 0; i < 2; ++i) {
 296     sleep(1);
 297     load_reporter_->FetchAndSample();
 298   }
 299   VerifyLbFeedback(load_reporter_->GenerateLoadBalancingFeedback(), 2,
 300                    kNumFeedbackSamplesInWindow);
 301 }
 302
 303 using LoadReportTest = LoadReporterTest;
 304
 305 TEST_F(LoadReportTest, BasicReport) {
 306   // Make up the first view data map.
 307   CensusViewProvider::ViewDataMap vdm1;
 308   vdm1.emplace(
 309       ::grpc::load_reporter::kViewStartCount,
 310       ::opencensus::stats::testing::TestUtils::MakeViewData(
 311           mock_census_view_provider()->FindViewDescriptor(
 312               ::grpc::load_reporter::kViewStartCount),
 313           {{{kClientIp1 + kLbToken1, kHostname1, kUser1}, 1234},
 314            {{kClientIp2 + kLbToken1, kHostname1, kUser1}, 1225},
 315            {{kClientIp0 + kLbToken1, kHostname1, kUser1}, 10},
 316            {{kClientIp2 + kLbToken1, kHostname1, kUser2}, 464},
 317            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3}, 101},
 318            {{kClientIp1 + kLbToken2, kHostname2, kUser3}, 17},
 319            {{kClientIp2 + kLbId3 + kLbTag2, kHostname2, kUser3}, 23}}));
 320   vdm1.emplace(::grpc::load_reporter::kViewEndCount,
 321                ::opencensus::stats::testing::TestUtils::MakeViewData(
 322                    mock_census_view_provider()->FindViewDescriptor(
 323                        ::grpc::load_reporter::kViewEndCount),
 324                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
 325                       ::grpc::load_reporter::kCallStatusOk},
 326                      641},
 327                     {{kClientIp2 + kLbToken1, kHostname1, kUser1,
 328                       ::grpc::load_reporter::kCallStatusClientError},
 329                      272},
 330                     {{kClientIp2 + kLbToken1, kHostname1, kUser2,
 331                       ::grpc::load_reporter::kCallStatusOk},
 332                      996},
 333                     {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3,
 334                       ::grpc::load_reporter::kCallStatusClientError},
 335                      34},
 336                     {{kClientIp1 + kLbToken2, kHostname2, kUser2,
 337                       ::grpc::load_reporter::kCallStatusOk},
 338                      18}}));
 339   vdm1.emplace(::grpc::load_reporter::kViewEndBytesSent,
 340                ::opencensus::stats::testing::TestUtils::MakeViewData(
 341                    mock_census_view_provider()->FindViewDescriptor(
 342                        ::grpc::load_reporter::kViewEndBytesSent),
 343                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
 344                       ::grpc::load_reporter::kCallStatusOk},
 345                      8977},
 346                     {{kClientIp2 + kLbToken1, kHostname1, kUser1,
 347                       ::grpc::load_reporter::kCallStatusClientError},
 348                      266},
 349                     {{kClientIp2 + kLbToken1, kHostname1, kUser2,
 350                       ::grpc::load_reporter::kCallStatusOk},
 351                      1276},
 352                     {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3,
 353                       ::grpc::load_reporter::kCallStatusClientError},
 354                      77823},
 355                     {{kClientIp1 + kLbToken2, kHostname2, kUser2,
 356                       ::grpc::load_reporter::kCallStatusOk},
 357                      48}}));
 358   vdm1.emplace(::grpc::load_reporter::kViewEndBytesReceived,
 359                ::opencensus::stats::testing::TestUtils::MakeViewData(
 360                    mock_census_view_provider()->FindViewDescriptor(
 361                        ::grpc::load_reporter::kViewEndBytesReceived),
 362                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
 363                       ::grpc::load_reporter::kCallStatusOk},
 364                      2341},
 365                     {{kClientIp2 + kLbToken1, kHostname1, kUser1,
 366                       ::grpc::load_reporter::kCallStatusClientError},
 367                      466},
 368                     {{kClientIp2 + kLbToken1, kHostname1, kUser2,
 369                       ::grpc::load_reporter::kCallStatusOk},
 370                      518},
 371                     {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3,
 372                       ::grpc::load_reporter::kCallStatusClientError},
 373                      81},
 374                     {{kClientIp1 + kLbToken2, kHostname2, kUser2,
 375                       ::grpc::load_reporter::kCallStatusOk},
 376                      27}}));
 377   vdm1.emplace(::grpc::load_reporter::kViewEndLatencyMs,
 378                ::opencensus::stats::testing::TestUtils::MakeViewData(
 379                    mock_census_view_provider()->FindViewDescriptor(
 380                        ::grpc::load_reporter::kViewEndLatencyMs),
 381                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
 382                       ::grpc::load_reporter::kCallStatusOk},
 383                      3.14},
 384                     {{kClientIp2 + kLbToken1, kHostname1, kUser1,
 385                       ::grpc::load_reporter::kCallStatusClientError},
 386                      5.26},
 387                     {{kClientIp2 + kLbToken1, kHostname1, kUser2,
 388                       ::grpc::load_reporter::kCallStatusOk},
 389                      45.4},
 390                     {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3,
 391                       ::grpc::load_reporter::kCallStatusClientError},
 392                      4.4},
 393                     {{kClientIp1 + kLbToken2, kHostname2, kUser2,
 394                       ::grpc::load_reporter::kCallStatusOk},
 395                      2348.0}}));
 396   vdm1.emplace(
 397       ::grpc::load_reporter::kViewOtherCallMetricCount,
 398       ::opencensus::stats::testing::TestUtils::MakeViewData(
 399           mock_census_view_provider()->FindViewDescriptor(
 400               ::grpc::load_reporter::kViewOtherCallMetricCount),
 401           {{{kClientIp1 + kLbToken1, kHostname1, kUser2, kMetric1}, 1},
 402            {{kClientIp1 + kLbToken1, kHostname1, kUser2, kMetric1}, 1},
 403            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric2},
 404             1}}));
 405   vdm1.emplace(
 406       ::grpc::load_reporter::kViewOtherCallMetricValue,
 407       ::opencensus::stats::testing::TestUtils::MakeViewData(
 408           mock_census_view_provider()->FindViewDescriptor(
 409               ::grpc::load_reporter::kViewOtherCallMetricValue),
 410           {{{kClientIp1 + kLbToken1, kHostname1, kUser2, kMetric1}, 1.2},
 411            {{kClientIp1 + kLbToken1, kHostname1, kUser2, kMetric1}, 1.2},
 412            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric2},
 413             3.2}}));
 414   // Make up the second view data map.
 415   CensusViewProvider::ViewDataMap vdm2;
 416   vdm2.emplace(
 417       ::grpc::load_reporter::kViewStartCount,
 418       ::opencensus::stats::testing::TestUtils::MakeViewData(
 419           mock_census_view_provider()->FindViewDescriptor(
 420               ::grpc::load_reporter::kViewStartCount),
 421           {{{kClientIp2 + kLbToken1, kHostname1, kUser1}, 3},
 422            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3}, 778}}));
 423   vdm2.emplace(::grpc::load_reporter::kViewEndCount,
 424                ::opencensus::stats::testing::TestUtils::MakeViewData(
 425                    mock_census_view_provider()->FindViewDescriptor(
 426                        ::grpc::load_reporter::kViewEndCount),
 427                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
 428                       ::grpc::load_reporter::kCallStatusOk},
 429                      24},
 430                     {{kClientIp1 + kLbToken2, kHostname2, kUser3,
 431                       ::grpc::load_reporter::kCallStatusClientError},
 432                      546}}));
 433   vdm2.emplace(::grpc::load_reporter::kViewEndBytesSent,
 434                ::opencensus::stats::testing::TestUtils::MakeViewData(
 435                    mock_census_view_provider()->FindViewDescriptor(
 436                        ::grpc::load_reporter::kViewEndBytesSent),
 437                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
 438                       ::grpc::load_reporter::kCallStatusOk},
 439                      747},
 440                     {{kClientIp1 + kLbToken2, kHostname2, kUser3,
 441                       ::grpc::load_reporter::kCallStatusClientError},
 442                      229}}));
 443   vdm2.emplace(::grpc::load_reporter::kViewEndBytesReceived,
 444                ::opencensus::stats::testing::TestUtils::MakeViewData(
 445                    mock_census_view_provider()->FindViewDescriptor(
 446                        ::grpc::load_reporter::kViewEndBytesReceived),
 447                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
 448                       ::grpc::load_reporter::kCallStatusOk},
 449                      173},
 450                     {{kClientIp1 + kLbToken2, kHostname2, kUser3,
 451                       ::grpc::load_reporter::kCallStatusClientError},
 452                      438}}));
 453   vdm2.emplace(::grpc::load_reporter::kViewEndLatencyMs,
 454                ::opencensus::stats::testing::TestUtils::MakeViewData(
 455                    mock_census_view_provider()->FindViewDescriptor(
 456                        ::grpc::load_reporter::kViewEndLatencyMs),
 457                    {{{kClientIp1 + kLbToken1, kHostname1, kUser1,
 458                       ::grpc::load_reporter::kCallStatusOk},
 459                      187},
 460                     {{kClientIp1 + kLbToken2, kHostname2, kUser3,
 461                       ::grpc::load_reporter::kCallStatusClientError},
 462                      34}}));
 463   vdm2.emplace(
 464       ::grpc::load_reporter::kViewOtherCallMetricCount,
 465       ::opencensus::stats::testing::TestUtils::MakeViewData(
 466           mock_census_view_provider()->FindViewDescriptor(
 467               ::grpc::load_reporter::kViewOtherCallMetricCount),
 468           {{{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric1}, 1},
 469            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric2},
 470             1}}));
 471   vdm2.emplace(
 472       ::grpc::load_reporter::kViewOtherCallMetricValue,
 473       ::opencensus::stats::testing::TestUtils::MakeViewData(
 474           mock_census_view_provider()->FindViewDescriptor(
 475               ::grpc::load_reporter::kViewOtherCallMetricValue),
 476           {{{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric1}, 9.6},
 477            {{kClientIp1 + kLbId2 + kLbTag1, kHostname2, kUser3, kMetric2},
 478             5.7}}));
 479   // Set up mock expectation.
 480   EXPECT_CALL(*mock_census_view_provider(), FetchViewData())
 481       .WillOnce(Return(vdm1))
 482       .WillOnce(Return(vdm2));
 483   PrepareCpuExpectation(2);
 484   // Start testing.
 485   load_reporter_->ReportStreamCreated(kHostname1, kLbId1, kLoadKey1);
 486   load_reporter_->ReportStreamCreated(kHostname2, kLbId2, kLoadKey2);
 487   load_reporter_->ReportStreamCreated(kHostname2, kLbId3, kLoadKey3);
 488   // First fetch.
 489   load_reporter_->FetchAndSample();
 490   load_reporter_->GenerateLoads(kHostname1, kLbId1);
 491   gpr_log(GPR_INFO, "First load generated.");
 492   // Second fetch.
 493   load_reporter_->FetchAndSample();
 494   load_reporter_->GenerateLoads(kHostname2, kLbId2);
 495   gpr_log(GPR_INFO, "Second load generated.");
 496   // TODO(juanlishen): Verify the data.
 497 }
 498
 499 }  // namespace
 500 }  // namespace testing
 501 }  // namespace grpc
 502
 503 int main(int argc, char** argv) {
 504   grpc::testing::TestEnvironment env(argc, argv);
 505   ::testing::InitGoogleTest(&argc, argv);
 506   return RUN_ALL_TESTS();
 507 }