test/core/security/grpc_tls_certificate_distributor_test.cc

   1 //
   2 // Copyright 2020 gRPC authors.
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //     http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 //
  16
  17 #include "src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h"
  18
  19 #include <gmock/gmock.h>
  20 #include <grpc/support/alloc.h>
  21 #include <grpc/support/log.h>
  22 #include <grpc/support/string_util.h>
  23 #include <gtest/gtest.h>
  24
  25 #include <deque>
  26 #include <list>
  27 #include <string>
  28 #include <thread>
  29
  30 #include "src/core/lib/slice/slice_internal.h"
  31 #include "test/core/util/test_config.h"
  32 #include "test/core/util/tls_utils.h"
  33
  34 namespace grpc_core {
  35
  36 namespace testing {
  37
  38 constexpr const char* kCertName1 = "cert_1_name";
  39 constexpr const char* kCertName2 = "cert_2_name";
  40 constexpr const char* kRootCert1Name = "root_cert_1_name";
  41 constexpr const char* kRootCert1Contents = "root_cert_1_contents";
  42 constexpr const char* kRootCert2Name = "root_cert_2_name";
  43 constexpr const char* kRootCert2Contents = "root_cert_2_contents";
  44 constexpr const char* kIdentityCert1Name = "identity_cert_1_name";
  45 constexpr const char* kIdentityCert1PrivateKey = "identity_private_key_1";
  46 constexpr const char* kIdentityCert1Contents = "identity_cert_1_contents";
  47 constexpr const char* kIdentityCert2Name = "identity_cert_2_name";
  48 constexpr const char* kIdentityCert2PrivateKey = "identity_private_key_2";
  49 constexpr const char* kIdentityCert2Contents = "identity_cert_2_contents";
  50 constexpr const char* kErrorMessage = "error_message";
  51 constexpr const char* kRootErrorMessage = "root_error_message";
  52 constexpr const char* kIdentityErrorMessage = "identity_error_message";
  53
  54 class GrpcTlsCertificateDistributorTest : public ::testing::Test {
  55  protected:
  56   // Forward declaration.
  57   class TlsCertificatesTestWatcher;
  58
  59   // CredentialInfo contains the parameters when calling OnCertificatesChanged
  60   // of a watcher. When OnCertificatesChanged is invoked, we will push a
  61   // CredentialInfo to the cert_update_queue of state_, and check in each test
  62   // if the status updates are correct.
  63   struct CredentialInfo {
  64     std::string root_certs;
  65     PemKeyCertPairList key_cert_pairs;
  66     CredentialInfo(std::string root, PemKeyCertPairList key_cert)
  67         : root_certs(std::move(root)), key_cert_pairs(std::move(key_cert)) {}
  68     bool operator==(const CredentialInfo& other) const {
  69       return root_certs == other.root_certs &&
  70              key_cert_pairs == other.key_cert_pairs;
  71     }
  72   };
  73
  74   // ErrorInfo contains the parameters when calling OnError of a watcher. When
  75   // OnError is invoked, we will push a ErrorInfo to the error_queue of state_,
  76   // and check in each test if the status updates are correct.
  77   struct ErrorInfo {
  78     std::string root_cert_str;
  79     std::string identity_cert_str;
  80     ErrorInfo(std::string root, std::string identity)
  81         : root_cert_str(std::move(root)),
  82           identity_cert_str(std::move(identity)) {}
  83     bool operator==(const ErrorInfo& other) const {
  84       return root_cert_str == other.root_cert_str &&
  85              identity_cert_str == other.identity_cert_str;
  86     }
  87   };
  88
  89   struct WatcherState {
  90     TlsCertificatesTestWatcher* watcher = nullptr;
  91     std::deque<CredentialInfo> cert_update_queue;
  92     std::deque<ErrorInfo> error_queue;
  93
  94     std::deque<CredentialInfo> GetCredentialQueue() {
  95       // We move the data member value so the data member will be re-initiated
  96       // with size 0, and ready for the next check.
  97       return std::move(cert_update_queue);
  98     }
  99     std::deque<ErrorInfo> GetErrorQueue() {
 100       // We move the data member value so the data member will be re-initiated
 101       // with size 0, and ready for the next check.
 102       return std::move(error_queue);
 103     }
 104   };
 105
 106   class TlsCertificatesTestWatcher : public grpc_tls_certificate_distributor::
 107                                          TlsCertificatesWatcherInterface {
 108    public:
 109     // ctor sets state->watcher to this.
 110     explicit TlsCertificatesTestWatcher(WatcherState* state) : state_(state) {
 111       state_->watcher = this;
 112     }
 113
 114     // dtor sets state->watcher to nullptr.
 115     ~TlsCertificatesTestWatcher() override { state_->watcher = nullptr; }
 116
 117     void OnCertificatesChanged(
 118         absl::optional<absl::string_view> root_certs,
 119         absl::optional<PemKeyCertPairList> key_cert_pairs) override {
 120       std::string updated_root;
 121       if (root_certs.has_value()) {
 122         updated_root = std::string(*root_certs);
 123       }
 124       PemKeyCertPairList updated_identity;
 125       if (key_cert_pairs.has_value()) {
 126         updated_identity = std::move(*key_cert_pairs);
 127       }
 128       state_->cert_update_queue.emplace_back(std::move(updated_root),
 129                                              std::move(updated_identity));
 130     }
 131
 132     void OnError(grpc_error_handle root_cert_error,
 133                  grpc_error_handle identity_cert_error) override {
 134       GPR_ASSERT(root_cert_error != GRPC_ERROR_NONE ||
 135                  identity_cert_error != GRPC_ERROR_NONE);
 136       std::string root_error_str;
 137       std::string identity_error_str;
 138       if (root_cert_error != GRPC_ERROR_NONE) {
 139         grpc_slice root_error_slice;
 140         GPR_ASSERT(grpc_error_get_str(
 141             root_cert_error, GRPC_ERROR_STR_DESCRIPTION, &root_error_slice));
 142         root_error_str = std::string(StringViewFromSlice(root_error_slice));
 143       }
 144       if (identity_cert_error != GRPC_ERROR_NONE) {
 145         grpc_slice identity_error_slice;
 146         GPR_ASSERT(grpc_error_get_str(identity_cert_error,
 147                                       GRPC_ERROR_STR_DESCRIPTION,
 148                                       &identity_error_slice));
 149         identity_error_str =
 150             std::string(StringViewFromSlice(identity_error_slice));
 151       }
 152       state_->error_queue.emplace_back(std::move(root_error_str),
 153                                        std::move(identity_error_str));
 154       GRPC_ERROR_UNREF(root_cert_error);
 155       GRPC_ERROR_UNREF(identity_cert_error);
 156     }
 157
 158    private:
 159     WatcherState* state_;
 160   };
 161
 162   // CallbackStatus contains the parameters when calling watch_status_callback_
 163   // of the distributor. When a particular callback is invoked, we will push a
 164   // CallbackStatus to a callback_queue_, and check in each test if the status
 165   // updates are correct.
 166   struct CallbackStatus {
 167     std::string cert_name;
 168     bool root_being_watched;
 169     bool identity_being_watched;
 170     CallbackStatus(std::string name, bool root_watched, bool identity_watched)
 171         : cert_name(std::move(name)),
 172           root_being_watched(root_watched),
 173           identity_being_watched(identity_watched) {}
 174     bool operator==(const CallbackStatus& other) const {
 175       return cert_name == other.cert_name &&
 176              root_being_watched == other.root_being_watched &&
 177              identity_being_watched == other.identity_being_watched;
 178     }
 179   };
 180
 181   void SetUp() override {
 182     distributor_.SetWatchStatusCallback([this](std::string cert_name,
 183                                                bool root_being_watched,
 184                                                bool identity_being_watched) {
 185       callback_queue_.emplace_back(std::move(cert_name), root_being_watched,
 186                                    identity_being_watched);
 187     });
 188   }
 189
 190   WatcherState* MakeWatcher(absl::optional<std::string> root_cert_name,
 191                             absl::optional<std::string> identity_cert_name) {
 192     MutexLock lock(&mu_);
 193     watchers_.emplace_back();
 194     // TlsCertificatesTestWatcher ctor takes a pointer to the WatcherState.
 195     // It sets WatcherState::watcher to point to itself.
 196     // The TlsCertificatesTestWatcher dtor will set WatcherState::watcher back
 197     // to nullptr to indicate that it's been destroyed.
 198     auto watcher =
 199         absl::make_unique<TlsCertificatesTestWatcher>(&watchers_.back());
 200     distributor_.WatchTlsCertificates(std::move(watcher),
 201                                       std::move(root_cert_name),
 202                                       std::move(identity_cert_name));
 203     return &watchers_.back();
 204   }
 205
 206   void CancelWatch(WatcherState* state) {
 207     MutexLock lock(&mu_);
 208     distributor_.CancelTlsCertificatesWatch(state->watcher);
 209     EXPECT_EQ(state->watcher, nullptr);
 210   }
 211
 212   std::deque<CallbackStatus> GetCallbackQueue() {
 213     // We move the data member value so the data member will be re-initiated
 214     // with size 0, and ready for the next check.
 215     return std::move(callback_queue_);
 216   }
 217
 218   grpc_tls_certificate_distributor distributor_;
 219   // Use a std::list<> here to avoid the address invalidation caused by internal
 220   // reallocation of std::vector<>.
 221   std::list<WatcherState> watchers_;
 222   std::deque<CallbackStatus> callback_queue_;
 223   // This is to make watchers_ and callback_queue_ thread-safe.
 224   Mutex mu_;
 225 };
 226
 227 TEST_F(GrpcTlsCertificateDistributorTest, BasicCredentialBehaviors) {
 228   EXPECT_FALSE(distributor_.HasRootCerts(kRootCert1Name));
 229   EXPECT_FALSE(distributor_.HasKeyCertPairs(kIdentityCert1Name));
 230   // After setting the certificates to the corresponding cert names, the
 231   // distributor should possess the corresponding certs.
 232   distributor_.SetKeyMaterials(kRootCert1Name, kRootCert1Contents,
 233                                absl::nullopt);
 234   EXPECT_TRUE(distributor_.HasRootCerts(kRootCert1Name));
 235   distributor_.SetKeyMaterials(
 236       kIdentityCert1Name, absl::nullopt,
 237       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 238   EXPECT_TRUE(distributor_.HasKeyCertPairs(kIdentityCert1Name));
 239   // Querying a non-existing cert name should return false.
 240   EXPECT_FALSE(distributor_.HasRootCerts(kRootCert2Name));
 241   EXPECT_FALSE(distributor_.HasKeyCertPairs(kIdentityCert2Name));
 242 }
 243
 244 TEST_F(GrpcTlsCertificateDistributorTest, UpdateCredentialsOnAnySide) {
 245   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 246   EXPECT_THAT(GetCallbackQueue(),
 247               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 248   // SetKeyMaterials should trigger watcher's OnCertificatesChanged method.
 249   distributor_.SetKeyMaterials(
 250       kCertName1, kRootCert1Contents,
 251       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 252   EXPECT_THAT(
 253       watcher_state_1->GetCredentialQueue(),
 254       ::testing::ElementsAre(CredentialInfo(
 255           kRootCert1Contents,
 256           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 257   // Set root certs should trigger watcher's OnCertificatesChanged again.
 258   distributor_.SetKeyMaterials(kCertName1, kRootCert2Contents, absl::nullopt);
 259   EXPECT_THAT(
 260       watcher_state_1->GetCredentialQueue(),
 261       ::testing::ElementsAre(CredentialInfo(
 262           kRootCert2Contents,
 263           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 264   // Set identity certs should trigger watcher's OnCertificatesChanged again.
 265   distributor_.SetKeyMaterials(
 266       kCertName1, absl::nullopt,
 267       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 268   EXPECT_THAT(
 269       watcher_state_1->GetCredentialQueue(),
 270       ::testing::ElementsAre(CredentialInfo(
 271           kRootCert2Contents,
 272           MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents))));
 273   CancelWatch(watcher_state_1);
 274 }
 275
 276 TEST_F(GrpcTlsCertificateDistributorTest, SameIdentityNameDiffRootName) {
 277   // Register watcher 1.
 278   WatcherState* watcher_state_1 =
 279       MakeWatcher(kRootCert1Name, kIdentityCert1Name);
 280   EXPECT_THAT(
 281       GetCallbackQueue(),
 282       ::testing::ElementsAre(CallbackStatus(kRootCert1Name, true, false),
 283                              CallbackStatus(kIdentityCert1Name, false, true)));
 284   // Register watcher 2.
 285   WatcherState* watcher_state_2 =
 286       MakeWatcher(kRootCert2Name, kIdentityCert1Name);
 287   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre(CallbackStatus(
 288                                       kRootCert2Name, true, false)));
 289   // Push credential updates to kRootCert1Name and check if the status works as
 290   // expected.
 291   distributor_.SetKeyMaterials(kRootCert1Name, kRootCert1Contents,
 292                                absl::nullopt);
 293   // Check the updates are delivered to watcher 1.
 294   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 295               ::testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
 296   // Push credential updates to kRootCert2Name.
 297   distributor_.SetKeyMaterials(kRootCert2Name, kRootCert2Contents,
 298                                absl::nullopt);
 299   // Check the updates are delivered to watcher 2.
 300   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 301               ::testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
 302   // Push credential updates to kIdentityCert1Name and check if the status works
 303   // as expected.
 304   distributor_.SetKeyMaterials(
 305       kIdentityCert1Name, absl::nullopt,
 306       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 307   // Check the updates are delivered to watcher 1 and watcher 2.
 308   EXPECT_THAT(
 309       watcher_state_1->GetCredentialQueue(),
 310       ::testing::ElementsAre(CredentialInfo(
 311           kRootCert1Contents,
 312           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 313   EXPECT_THAT(
 314       watcher_state_2->GetCredentialQueue(),
 315       ::testing::ElementsAre(CredentialInfo(
 316           kRootCert2Contents,
 317           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 318   // Cancel watcher 1.
 319   CancelWatch(watcher_state_1);
 320   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre(CallbackStatus(
 321                                       kRootCert1Name, false, false)));
 322   // Cancel watcher 2.
 323   CancelWatch(watcher_state_2);
 324   EXPECT_THAT(
 325       GetCallbackQueue(),
 326       ::testing::ElementsAre(CallbackStatus(kRootCert2Name, false, false),
 327                              CallbackStatus(kIdentityCert1Name, false, false)));
 328 }
 329
 330 TEST_F(GrpcTlsCertificateDistributorTest, SameRootNameDiffIdentityName) {
 331   // Register watcher 1.
 332   WatcherState* watcher_state_1 =
 333       MakeWatcher(kRootCert1Name, kIdentityCert1Name);
 334   EXPECT_THAT(
 335       GetCallbackQueue(),
 336       ::testing::ElementsAre(CallbackStatus(kRootCert1Name, true, false),
 337                              CallbackStatus(kIdentityCert1Name, false, true)));
 338   // Register watcher 2.
 339   WatcherState* watcher_state_2 =
 340       MakeWatcher(kRootCert1Name, kIdentityCert2Name);
 341   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre(CallbackStatus(
 342                                       kIdentityCert2Name, false, true)));
 343   // Push credential updates to kRootCert1Name and check if the status works as
 344   // expected.
 345   distributor_.SetKeyMaterials(kRootCert1Name, kRootCert1Contents,
 346                                absl::nullopt);
 347   // Check the updates are delivered to watcher 1.
 348   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 349               ::testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
 350   // Check the updates are delivered to watcher 2.
 351   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 352               ::testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
 353   // Push credential updates to SetKeyMaterials.
 354   distributor_.SetKeyMaterials(
 355       kIdentityCert1Name, absl::nullopt,
 356       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 357   // Check the updates are delivered to watcher 1.
 358   EXPECT_THAT(
 359       watcher_state_1->GetCredentialQueue(),
 360       ::testing::ElementsAre(CredentialInfo(
 361           kRootCert1Contents,
 362           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 363   // Push credential updates to kIdentityCert2Name.
 364   distributor_.SetKeyMaterials(
 365       kIdentityCert2Name, absl::nullopt,
 366       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 367   // Check the updates are delivered to watcher 2.
 368   EXPECT_THAT(
 369       watcher_state_2->GetCredentialQueue(),
 370       ::testing::ElementsAre(CredentialInfo(
 371           kRootCert1Contents,
 372           MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents))));
 373   // Cancel watcher 1.
 374   CancelWatch(watcher_state_1);
 375   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre(CallbackStatus(
 376                                       kIdentityCert1Name, false, false)));
 377   // Cancel watcher 2.
 378   CancelWatch(watcher_state_2);
 379   EXPECT_THAT(
 380       GetCallbackQueue(),
 381       ::testing::ElementsAre(CallbackStatus(kRootCert1Name, false, false),
 382                              CallbackStatus(kIdentityCert2Name, false, false)));
 383 }
 384
 385 TEST_F(GrpcTlsCertificateDistributorTest,
 386        AddAndCancelFirstWatcherForSameRootAndIdentityCertName) {
 387   // Register watcher 1 watching kCertName1 for both root and identity certs.
 388   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 389   EXPECT_THAT(GetCallbackQueue(),
 390               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 391   // Push credential updates to kCertName1 and check if the status works as
 392   // expected.
 393   distributor_.SetKeyMaterials(
 394       kCertName1, kRootCert1Contents,
 395       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 396   // Check the updates are delivered to watcher 1.
 397   EXPECT_THAT(
 398       watcher_state_1->GetCredentialQueue(),
 399       ::testing::ElementsAre(CredentialInfo(
 400           kRootCert1Contents,
 401           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 402   // Cancel watcher 1.
 403   CancelWatch(watcher_state_1);
 404   EXPECT_THAT(GetCallbackQueue(),
 405               ::testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
 406 }
 407
 408 TEST_F(GrpcTlsCertificateDistributorTest,
 409        AddAndCancelFirstWatcherForIdentityCertNameWithRootBeingWatched) {
 410   // Register watcher 1 watching kCertName1 for root certs.
 411   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, absl::nullopt);
 412   EXPECT_THAT(GetCallbackQueue(),
 413               ::testing::ElementsAre(CallbackStatus(kCertName1, true, false)));
 414   // Register watcher 2 watching kCertName1 for identity certs.
 415   WatcherState* watcher_state_2 = MakeWatcher(absl::nullopt, kCertName1);
 416   EXPECT_THAT(GetCallbackQueue(),
 417               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 418   // Push credential updates to kCertName1 and check if the status works as
 419   // expected.
 420   distributor_.SetKeyMaterials(
 421       kCertName1, kRootCert1Contents,
 422       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 423   // Check the updates are delivered to watcher 1.
 424   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 425               ::testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
 426   // Check the updates are delivered to watcher 2.
 427   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 428               ::testing::ElementsAre(CredentialInfo(
 429                   "", MakeCertKeyPairs(kIdentityCert1PrivateKey,
 430                                        kIdentityCert1Contents))));
 431   // Push root cert updates to kCertName1.
 432   distributor_.SetKeyMaterials(kCertName1, kRootCert2Contents, absl::nullopt);
 433   // Check the updates are delivered to watcher 1.
 434   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 435               ::testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
 436   // Check the updates are not delivered to watcher 2.
 437   EXPECT_THAT(watcher_state_2->GetCredentialQueue(), ::testing::ElementsAre());
 438   // Push identity cert updates to kCertName1.
 439   distributor_.SetKeyMaterials(
 440       kCertName1, absl::nullopt,
 441       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 442   // Check the updates are not delivered to watcher 1.
 443   EXPECT_THAT(watcher_state_1->GetCredentialQueue(), ::testing::ElementsAre());
 444   // Check the updates are delivered to watcher 2.
 445   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 446               ::testing::ElementsAre(CredentialInfo(
 447                   "", MakeCertKeyPairs(kIdentityCert2PrivateKey,
 448                                        kIdentityCert2Contents))));
 449   watcher_state_2->cert_update_queue.clear();
 450   // Cancel watcher 2.
 451   CancelWatch(watcher_state_2);
 452   EXPECT_THAT(GetCallbackQueue(),
 453               ::testing::ElementsAre(CallbackStatus(kCertName1, true, false)));
 454   // Cancel watcher 1.
 455   CancelWatch(watcher_state_1);
 456   EXPECT_THAT(GetCallbackQueue(),
 457               ::testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
 458 }
 459
 460 TEST_F(GrpcTlsCertificateDistributorTest,
 461        AddAndCancelFirstWatcherForRootCertNameWithIdentityBeingWatched) {
 462   // Register watcher 1 watching kCertName1 for identity certs.
 463   WatcherState* watcher_state_1 = MakeWatcher(absl::nullopt, kCertName1);
 464   EXPECT_THAT(GetCallbackQueue(),
 465               ::testing::ElementsAre(CallbackStatus(kCertName1, false, true)));
 466   // Register watcher 2 watching kCertName1 for root certs.
 467   WatcherState* watcher_state_2 = MakeWatcher(kCertName1, absl::nullopt);
 468   EXPECT_THAT(GetCallbackQueue(),
 469               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 470   // Push credential updates to kCertName1 and check if the status works as
 471   // expected.
 472   distributor_.SetKeyMaterials(
 473       kCertName1, kRootCert1Contents,
 474       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 475   // Check the updates are delivered to watcher 1.
 476   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 477               ::testing::ElementsAre(CredentialInfo(
 478                   "", MakeCertKeyPairs(kIdentityCert1PrivateKey,
 479                                        kIdentityCert1Contents))));
 480   // Check the updates are delivered to watcher 2.
 481   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 482               ::testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
 483   // Push root cert updates to kCertName1.
 484   distributor_.SetKeyMaterials(kCertName1, kRootCert2Contents, absl::nullopt);
 485   // Check the updates are delivered to watcher 2.
 486   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 487               ::testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
 488   // Check the updates are not delivered to watcher 1.
 489   EXPECT_THAT(watcher_state_1->GetCredentialQueue(), ::testing::ElementsAre());
 490   // Push identity cert updates to kCertName1.
 491   distributor_.SetKeyMaterials(
 492       kCertName1, absl::nullopt,
 493       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 494   // Check the updates are not delivered to watcher 2.
 495   EXPECT_THAT(watcher_state_2->GetCredentialQueue(), ::testing::ElementsAre());
 496   // Check the updates are delivered to watcher 1.
 497   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 498               ::testing::ElementsAre(CredentialInfo(
 499                   "", MakeCertKeyPairs(kIdentityCert2PrivateKey,
 500                                        kIdentityCert2Contents))));
 501   // Cancel watcher 2.
 502   CancelWatch(watcher_state_2);
 503   EXPECT_THAT(GetCallbackQueue(),
 504               ::testing::ElementsAre(CallbackStatus(kCertName1, false, true)));
 505   // Cancel watcher 1.
 506   CancelWatch(watcher_state_1);
 507   EXPECT_THAT(GetCallbackQueue(),
 508               ::testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
 509 }
 510
 511 TEST_F(GrpcTlsCertificateDistributorTest,
 512        RemoveAllWatchersForCertNameAndAddAgain) {
 513   // Register watcher 1 and watcher 2 watching kCertName1 for root and identity
 514   // certs.
 515   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 516   EXPECT_THAT(GetCallbackQueue(),
 517               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 518   WatcherState* watcher_state_2 = MakeWatcher(kCertName1, kCertName1);
 519   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre());
 520   // Push credential updates to kCertName1.
 521   distributor_.SetKeyMaterials(
 522       kCertName1, kRootCert1Contents,
 523       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 524   // Cancel watcher 2.
 525   CancelWatch(watcher_state_2);
 526   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre());
 527   // Cancel watcher 1.
 528   CancelWatch(watcher_state_1);
 529   EXPECT_THAT(GetCallbackQueue(),
 530               ::testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
 531   // Register watcher 3 watching kCertName for root and identity certs.
 532   WatcherState* watcher_state_3 = MakeWatcher(kCertName1, kCertName1);
 533   EXPECT_THAT(GetCallbackQueue(),
 534               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 535   // Push credential updates to kCertName1.
 536   distributor_.SetKeyMaterials(
 537       kCertName1, kRootCert2Contents,
 538       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 539   // Check the updates are delivered to watcher 3.
 540   EXPECT_THAT(
 541       watcher_state_3->GetCredentialQueue(),
 542       ::testing::ElementsAre(CredentialInfo(
 543           kRootCert2Contents,
 544           MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents))));
 545   // Cancel watcher 3.
 546   CancelWatch(watcher_state_3);
 547   EXPECT_THAT(GetCallbackQueue(),
 548               ::testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
 549 }
 550
 551 TEST_F(GrpcTlsCertificateDistributorTest, ResetCallbackToNull) {
 552   // Register watcher 1 watching kCertName1 for root and identity certs.
 553   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 554   EXPECT_THAT(GetCallbackQueue(),
 555               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 556   // Reset callback to nullptr.
 557   distributor_.SetWatchStatusCallback(nullptr);
 558   // Cancel watcher 1 shouldn't trigger any callback.
 559   CancelWatch(watcher_state_1);
 560   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre());
 561 }
 562
 563 TEST_F(GrpcTlsCertificateDistributorTest, SetKeyMaterialsInCallback) {
 564   distributor_.SetWatchStatusCallback([this](std::string cert_name,
 565                                              bool /*root_being_watched*/,
 566                                              bool /*identity_being_watched*/) {
 567     distributor_.SetKeyMaterials(
 568         cert_name, kRootCert1Contents,
 569         MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 570   });
 571   auto verify_function = [this](std::string cert_name) {
 572     WatcherState* watcher_state_1 = MakeWatcher(cert_name, cert_name);
 573     // Check the updates are delivered to watcher 1.
 574     EXPECT_THAT(
 575         watcher_state_1->GetCredentialQueue(),
 576         ::testing::ElementsAre(CredentialInfo(
 577             kRootCert1Contents, MakeCertKeyPairs(kIdentityCert1PrivateKey,
 578                                                  kIdentityCert1Contents))));
 579     CancelWatch(watcher_state_1);
 580   };
 581   // Start 1000 threads that will register a watcher to a new cert name, verify
 582   // the key materials being set, and then cancel the watcher, to make sure the
 583   // lock mechanism in the distributor is safe.
 584   std::vector<std::thread> threads;
 585   threads.reserve(1000);
 586   for (int i = 0; i < 1000; ++i) {
 587     threads.emplace_back(verify_function, std::to_string(i));
 588   }
 589   for (auto& th : threads) {
 590     th.join();
 591   }
 592 }
 593
 594 TEST_F(GrpcTlsCertificateDistributorTest, WatchACertInfoWithValidCredentials) {
 595   // Push credential updates to kCertName1.
 596   distributor_.SetKeyMaterials(
 597       kCertName1, kRootCert1Contents,
 598       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 599   // Push root credential updates to kCertName2.
 600   distributor_.SetKeyMaterials(kRootCert2Name, kRootCert2Contents,
 601                                absl::nullopt);
 602   // Push identity credential updates to kCertName2.
 603   distributor_.SetKeyMaterials(
 604       kIdentityCert2Name, absl::nullopt,
 605       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 606   // Register watcher 1.
 607   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 608   // watcher 1 should receive the credentials right away.
 609   EXPECT_THAT(
 610       watcher_state_1->GetCredentialQueue(),
 611       ::testing::ElementsAre(CredentialInfo(
 612           kRootCert1Contents,
 613           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 614   CancelWatch(watcher_state_1);
 615   // Register watcher 2.
 616   WatcherState* watcher_state_2 = MakeWatcher(kRootCert2Name, absl::nullopt);
 617   // watcher 2 should receive the root credentials right away.
 618   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 619               ::testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
 620   // Register watcher 3.
 621   WatcherState* watcher_state_3 =
 622       MakeWatcher(absl::nullopt, kIdentityCert2Name);
 623   // watcher 3 should received the identity credentials right away.
 624   EXPECT_THAT(watcher_state_3->GetCredentialQueue(),
 625               ::testing::ElementsAre(CredentialInfo(
 626                   "", MakeCertKeyPairs(kIdentityCert2PrivateKey,
 627                                        kIdentityCert2Contents))));
 628   CancelWatch(watcher_state_2);
 629   CancelWatch(watcher_state_3);
 630 }
 631
 632 TEST_F(GrpcTlsCertificateDistributorTest,
 633        SetErrorForCertForBothRootAndIdentity) {
 634   // Register watcher 1.
 635   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 636   // Calling SetErrorForCert on both cert names should only call one OnError
 637   // on watcher 1.
 638   distributor_.SetErrorForCert(
 639       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 640       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 641   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 642               ::testing::ElementsAre(
 643                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 644   // Calling SetErrorForCert on root cert name should call OnError
 645   // on watcher 1 again.
 646   distributor_.SetErrorForCert(
 647       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage),
 648       absl::nullopt);
 649   EXPECT_THAT(
 650       watcher_state_1->GetErrorQueue(),
 651       ::testing::ElementsAre(ErrorInfo(kErrorMessage, kIdentityErrorMessage)));
 652   // Calling SetErrorForCert on identity cert name should call OnError
 653   // on watcher 1 again.
 654   distributor_.SetErrorForCert(
 655       kCertName1, absl::nullopt,
 656       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage));
 657   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 658               ::testing::ElementsAre(ErrorInfo(kErrorMessage, kErrorMessage)));
 659   distributor_.CancelTlsCertificatesWatch(watcher_state_1->watcher);
 660   EXPECT_EQ(watcher_state_1->watcher, nullptr);
 661 }
 662
 663 TEST_F(GrpcTlsCertificateDistributorTest, SetErrorForCertForRootOrIdentity) {
 664   // Register watcher 1.
 665   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, absl::nullopt);
 666   // Calling SetErrorForCert on root name should only call one OnError
 667   // on watcher 1.
 668   distributor_.SetErrorForCert(
 669       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 670       absl::nullopt);
 671   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 672               ::testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
 673   // Calling SetErrorForCert on identity name should do nothing.
 674   distributor_.SetErrorForCert(
 675       kCertName1, absl::nullopt,
 676       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 677   EXPECT_THAT(watcher_state_1->GetErrorQueue(), ::testing::ElementsAre());
 678   // Calling SetErrorForCert on both names should still get one OnError call.
 679   distributor_.SetErrorForCert(
 680       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 681       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 682   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 683               ::testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
 684   CancelWatch(watcher_state_1);
 685   // Register watcher 2.
 686   WatcherState* watcher_state_2 = MakeWatcher(absl::nullopt, kCertName1);
 687   // Calling SetErrorForCert on identity name should only call one OnError
 688   // on watcher 2.
 689   distributor_.SetErrorForCert(
 690       kCertName1, absl::nullopt,
 691       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 692   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
 693               ::testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
 694   // Calling SetErrorForCert on root name should do nothing.
 695   distributor_.SetErrorForCert(
 696       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 697       absl::nullopt);
 698   EXPECT_THAT(watcher_state_2->GetErrorQueue(), ::testing::ElementsAre());
 699   // Calling SetErrorForCert on both names should still get one OnError call.
 700   distributor_.SetErrorForCert(
 701       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 702       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 703   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
 704               ::testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
 705   CancelWatch(watcher_state_2);
 706 }
 707
 708 TEST_F(GrpcTlsCertificateDistributorTest,
 709        SetErrorForIdentityNameWithPreexistingErrorForRootName) {
 710   // SetErrorForCert for kCertName1.
 711   distributor_.SetErrorForCert(
 712       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 713       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 714   // Register watcher 1 for kCertName1 as root and kCertName2 as identity.
 715   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName2);
 716   // Should trigger OnError call right away since kCertName1 has error.
 717   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 718               ::testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
 719   // Calling SetErrorForCert on kCertName2 should trigger OnError with both
 720   // errors, because kCertName1 also has error.
 721   distributor_.SetErrorForCert(
 722       kCertName2, absl::nullopt,
 723       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 724   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 725               ::testing::ElementsAre(
 726                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 727   CancelWatch(watcher_state_1);
 728 }
 729
 730 TEST_F(GrpcTlsCertificateDistributorTest,
 731        SetErrorForCertForRootNameWithSameNameForIdentityErrored) {
 732   // SetErrorForCert for kCertName1.
 733   distributor_.SetErrorForCert(
 734       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 735       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 736   // Register watcher 1 for kCertName2 as root and kCertName1 as identity.
 737   WatcherState* watcher_state_1 = MakeWatcher(kCertName2, kCertName1);
 738   // Should trigger OnError call right away since kCertName2 has error.
 739   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 740               ::testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
 741   // Calling SetErrorForCert on kCertName2 should trigger OnError with both
 742   // errors, because kCertName1 also has error.
 743   distributor_.SetErrorForCert(
 744       kCertName2, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 745       absl::nullopt);
 746   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 747               ::testing::ElementsAre(
 748                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 749   CancelWatch(watcher_state_1);
 750 }
 751
 752 TEST_F(GrpcTlsCertificateDistributorTest,
 753        SetErrorForIdentityNameWithoutErrorForRootName) {
 754   // Register watcher 1 for kCertName1 as root and kCertName2 as identity.
 755   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName2);
 756   // Should not trigger OnError.
 757   EXPECT_THAT(watcher_state_1->GetErrorQueue(), ::testing::ElementsAre());
 758   // Calling SetErrorForCert on kCertName2 should trigger OnError.
 759   distributor_.SetErrorForCert(
 760       kCertName2, absl::nullopt,
 761       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 762   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 763               ::testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
 764   CancelWatch(watcher_state_1);
 765   // Register watcher 2 for kCertName2 as identity and a non-existing name
 766   // kRootCert1Name as root.
 767   WatcherState* watcher_state_2 = MakeWatcher(kRootCert1Name, kCertName2);
 768   // Should not trigger OnError.
 769   EXPECT_THAT(watcher_state_2->GetErrorQueue(), ::testing::ElementsAre());
 770   // Calling SetErrorForCert on kCertName2 should trigger OnError.
 771   distributor_.SetErrorForCert(
 772       kCertName2, absl::nullopt,
 773       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 774   EXPECT_THAT(watcher_state_2->error_queue,
 775               ::testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
 776   CancelWatch(watcher_state_2);
 777 }
 778
 779 TEST_F(GrpcTlsCertificateDistributorTest,
 780        SetErrorForRootNameWithPreexistingErrorForIdentityName) {
 781   WatcherState* watcher_state_1 = MakeWatcher(kCertName2, kCertName1);
 782   // Should not trigger OnError.
 783   EXPECT_THAT(watcher_state_1->GetErrorQueue(), ::testing::ElementsAre());
 784   // Calling SetErrorForCert on kCertName2 should trigger OnError.
 785   distributor_.SetErrorForCert(
 786       kCertName2, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 787       absl::nullopt);
 788   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 789               ::testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
 790   CancelWatch(watcher_state_1);
 791   // Register watcher 2 for kCertName2 as root and a non-existing name
 792   // kIdentityCert1Name as identity.
 793   WatcherState* watcher_state_2 = MakeWatcher(kCertName2, kIdentityCert1Name);
 794   // Should not trigger OnError.
 795   EXPECT_THAT(watcher_state_2->GetErrorQueue(), ::testing::ElementsAre());
 796   // Calling SetErrorForCert on kCertName2 should trigger OnError.
 797   distributor_.SetErrorForCert(
 798       kCertName2, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 799       absl::nullopt);
 800   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
 801               ::testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
 802   CancelWatch(watcher_state_2);
 803 }
 804
 805 TEST_F(GrpcTlsCertificateDistributorTest,
 806        CancelTheLastWatcherOnAnErroredCertInfo) {
 807   // Register watcher 1.
 808   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 809   // Calling SetErrorForCert on both cert names should only call one OnError
 810   // on watcher 1.
 811   distributor_.SetErrorForCert(
 812       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 813       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 814   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 815               ::testing::ElementsAre(
 816                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 817   // When watcher 1 is removed, the cert info entry should be removed.
 818   CancelWatch(watcher_state_1);
 819   // Register watcher 2 on the same cert name.
 820   WatcherState* watcher_state_2 = MakeWatcher(kCertName1, kCertName1);
 821   // Should not trigger OnError call on watcher 2 right away.
 822   EXPECT_THAT(watcher_state_2->GetErrorQueue(), ::testing::ElementsAre());
 823   CancelWatch(watcher_state_2);
 824 }
 825
 826 TEST_F(GrpcTlsCertificateDistributorTest,
 827        WatchErroredCertInfoWithValidCredentialData) {
 828   // Push credential updates to kCertName1.
 829   distributor_.SetKeyMaterials(
 830       kCertName1, kRootCert1Contents,
 831       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 832   // Calling SetErrorForCert on both cert names.
 833   distributor_.SetErrorForCert(
 834       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 835       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 836   // Register watcher 1.
 837   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 838   // watcher 1 should receive both the old credentials and the error right away.
 839   EXPECT_THAT(
 840       watcher_state_1->GetCredentialQueue(),
 841       ::testing::ElementsAre(CredentialInfo(
 842           kRootCert1Contents,
 843           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 844   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 845               ::testing::ElementsAre(
 846                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 847   CancelWatch(watcher_state_1);
 848 }
 849
 850 TEST_F(GrpcTlsCertificateDistributorTest,
 851        SetErrorForCertThenSuccessfulCredentialUpdates) {
 852   // Calling SetErrorForCert on both cert names.
 853   distributor_.SetErrorForCert(
 854       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 855       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 856   // Push credential updates to kCertName1.
 857   distributor_.SetKeyMaterials(
 858       kCertName1, kRootCert1Contents,
 859       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 860   // Register watcher 1.
 861   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 862   // watcher 1 should only receive credential updates without any error, because
 863   // the previous error is wiped out by a successful update.
 864   EXPECT_THAT(
 865       watcher_state_1->GetCredentialQueue(),
 866       ::testing::ElementsAre(CredentialInfo(
 867           kRootCert1Contents,
 868           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 869   EXPECT_THAT(watcher_state_1->GetErrorQueue(), ::testing::ElementsAre());
 870   CancelWatch(watcher_state_1);
 871 }
 872
 873 TEST_F(GrpcTlsCertificateDistributorTest, WatchCertInfoThenInvokeSetError) {
 874   // Register watcher 1.
 875   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 876   // Register watcher 2.
 877   WatcherState* watcher_state_2 = MakeWatcher(kRootCert1Name, absl::nullopt);
 878   // Register watcher 3.
 879   WatcherState* watcher_state_3 =
 880       MakeWatcher(absl::nullopt, kIdentityCert1Name);
 881   distributor_.SetError(GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage));
 882   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 883               ::testing::ElementsAre(ErrorInfo(kErrorMessage, kErrorMessage)));
 884   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
 885               ::testing::ElementsAre(ErrorInfo(kErrorMessage, "")));
 886   EXPECT_THAT(watcher_state_3->GetErrorQueue(),
 887               ::testing::ElementsAre(ErrorInfo("", kErrorMessage)));
 888   CancelWatch(watcher_state_1);
 889   CancelWatch(watcher_state_2);
 890   CancelWatch(watcher_state_3);
 891 }
 892
 893 TEST_F(GrpcTlsCertificateDistributorTest, WatchErroredCertInfoBySetError) {
 894   // Register watcher 1 watching kCertName1 as root.
 895   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, absl::nullopt);
 896   // Register watcher 2 watching kCertName2 as identity.
 897   WatcherState* watcher_state_2 = MakeWatcher(absl::nullopt, kCertName2);
 898   // Call SetError and then cancel all watchers.
 899   distributor_.SetError(GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage));
 900   CancelWatch(watcher_state_1);
 901   CancelWatch(watcher_state_2);
 902   // Register watcher 3 watching kCertName1 as root and kCertName2 as identity
 903   // should not get the error updates.
 904   WatcherState* watcher_state_3 = MakeWatcher(kCertName1, kCertName2);
 905   EXPECT_THAT(watcher_state_3->GetErrorQueue(), ::testing::ElementsAre());
 906   CancelWatch(watcher_state_3);
 907   // Register watcher 4 watching kCertName2 as root and kCertName1 as identity
 908   // should not get the error updates.
 909   WatcherState* watcher_state_4 = MakeWatcher(kCertName2, kCertName1);
 910   EXPECT_THAT(watcher_state_4->GetErrorQueue(), ::testing::ElementsAre());
 911   CancelWatch(watcher_state_4);
 912 }
 913
 914 TEST_F(GrpcTlsCertificateDistributorTest, SetErrorForCertInCallback) {
 915   distributor_.SetWatchStatusCallback([this](std::string cert_name,
 916                                              bool /*root_being_watched*/,
 917                                              bool /*identity_being_watched*/) {
 918     this->distributor_.SetErrorForCert(
 919         cert_name, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 920         GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 921   });
 922   auto verify_function = [this](std::string cert_name) {
 923     WatcherState* watcher_state_1 = MakeWatcher(cert_name, cert_name);
 924     // Check the errors are delivered to watcher 1.
 925     EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 926                 ::testing::ElementsAre(
 927                     ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 928     CancelWatch(watcher_state_1);
 929   };
 930   // Start 1000 threads that will register a watcher to a new cert name, verify
 931   // the key materials being set, and then cancel the watcher, to make sure the
 932   // lock mechanism in the distributor is safe.
 933   std::vector<std::thread> threads;
 934   threads.reserve(1000);
 935   for (int i = 0; i < 1000; ++i) {
 936     threads.emplace_back(verify_function, std::to_string(i));
 937   }
 938   for (auto& th : threads) {
 939     th.join();
 940   }
 941 }
 942
 943 }  // namespace testing
 944
 945 }  // namespace grpc_core
 946
 947 int main(int argc, char** argv) {
 948   grpc::testing::TestEnvironment env(argc, argv);
 949   ::testing::InitGoogleTest(&argc, argv);
 950   grpc_init();
 951   int ret = RUN_ALL_TESTS();
 952   grpc_shutdown();
 953   return ret;
 954 }