test/core/security/grpc_tls_certificate_distributor_test.cc

   1 //
   2 // Copyright 2020 gRPC authors.
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //     http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 //
  16
  17 #include "src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h"
  18
  19 #include <gmock/gmock.h>
  20 #include <grpc/support/alloc.h>
  21 #include <grpc/support/log.h>
  22 #include <grpc/support/string_util.h>
  23 #include <gtest/gtest.h>
  24
  25 #include <deque>
  26 #include <list>
  27 #include <string>
  28 #include <thread>
  29
  30 #include "src/core/lib/slice/slice_internal.h"
  31 #include "test/core/util/test_config.h"
  32
  33 namespace testing {
  34
  35 constexpr const char* kCertName1 = "cert_1_name";
  36 constexpr const char* kCertName2 = "cert_2_name";
  37 constexpr const char* kRootCert1Name = "root_cert_1_name";
  38 constexpr const char* kRootCert1Contents = "root_cert_1_contents";
  39 constexpr const char* kRootCert2Name = "root_cert_2_name";
  40 constexpr const char* kRootCert2Contents = "root_cert_2_contents";
  41 constexpr const char* kIdentityCert1Name = "identity_cert_1_name";
  42 constexpr const char* kIdentityCert1PrivateKey = "identity_private_key_1";
  43 constexpr const char* kIdentityCert1Contents = "identity_cert_1_contents";
  44 constexpr const char* kIdentityCert2Name = "identity_cert_2_name";
  45 constexpr const char* kIdentityCert2PrivateKey = "identity_private_key_2";
  46 constexpr const char* kIdentityCert2Contents = "identity_cert_2_contents";
  47 constexpr const char* kErrorMessage = "error_message";
  48 constexpr const char* kRootErrorMessage = "root_error_message";
  49 constexpr const char* kIdentityErrorMessage = "identity_error_message";
  50
  51 class GrpcTlsCertificateDistributorTest : public ::testing::Test {
  52  protected:
  53   // Forward declaration.
  54   class TlsCertificatesTestWatcher;
  55
  56   static grpc_tls_certificate_distributor::PemKeyCertPairList MakeCertKeyPairs(
  57       const char* private_key, const char* certs) {
  58     if (strcmp(private_key, "") == 0 && strcmp(certs, "") == 0) {
  59       return {};
  60     }
  61     grpc_ssl_pem_key_cert_pair* ssl_pair =
  62         static_cast<grpc_ssl_pem_key_cert_pair*>(
  63             gpr_malloc(sizeof(grpc_ssl_pem_key_cert_pair)));
  64     ssl_pair->private_key = gpr_strdup(private_key);
  65     ssl_pair->cert_chain = gpr_strdup(certs);
  66     grpc_tls_certificate_distributor::PemKeyCertPairList pem_key_cert_pairs;
  67     pem_key_cert_pairs.emplace_back(ssl_pair);
  68     return pem_key_cert_pairs;
  69   }
  70
  71   // CredentialInfo contains the parameters when calling OnCertificatesChanged
  72   // of a watcher. When OnCertificatesChanged is invoked, we will push a
  73   // CredentialInfo to the cert_update_queue of state_, and check in each test
  74   // if the status updates are correct.
  75   struct CredentialInfo {
  76     std::string root_certs;
  77     grpc_tls_certificate_distributor::PemKeyCertPairList key_cert_pairs;
  78     CredentialInfo(
  79         std::string root,
  80         grpc_tls_certificate_distributor::PemKeyCertPairList key_cert)
  81         : root_certs(std::move(root)), key_cert_pairs(std::move(key_cert)) {}
  82     bool operator==(const CredentialInfo& other) const {
  83       return root_certs == other.root_certs &&
  84              key_cert_pairs == other.key_cert_pairs;
  85     }
  86   };
  87
  88   // ErrorInfo contains the parameters when calling OnError of a watcher. When
  89   // OnError is invoked, we will push a ErrorInfo to the error_queue of state_,
  90   // and check in each test if the status updates are correct.
  91   struct ErrorInfo {
  92     std::string root_cert_str;
  93     std::string identity_cert_str;
  94     ErrorInfo(std::string root, std::string identity)
  95         : root_cert_str(std::move(root)),
  96           identity_cert_str(std::move(identity)) {}
  97     bool operator==(const ErrorInfo& other) const {
  98       return root_cert_str == other.root_cert_str &&
  99              identity_cert_str == other.identity_cert_str;
 100     }
 101   };
 102
 103   struct WatcherState {
 104     TlsCertificatesTestWatcher* watcher = nullptr;
 105     std::deque<CredentialInfo> cert_update_queue;
 106     std::deque<ErrorInfo> error_queue;
 107
 108     std::deque<CredentialInfo> GetCredentialQueue() {
 109       // We move the data member value so the data member will be re-initiated
 110       // with size 0, and ready for the next check.
 111       return std::move(cert_update_queue);
 112     }
 113     std::deque<ErrorInfo> GetErrorQueue() {
 114       // We move the data member value so the data member will be re-initiated
 115       // with size 0, and ready for the next check.
 116       return std::move(error_queue);
 117     }
 118   };
 119
 120   class TlsCertificatesTestWatcher : public grpc_tls_certificate_distributor::
 121                                          TlsCertificatesWatcherInterface {
 122    public:
 123     // ctor sets state->watcher to this.
 124     explicit TlsCertificatesTestWatcher(WatcherState* state) : state_(state) {
 125       state_->watcher = this;
 126     }
 127
 128     // dtor sets state->watcher to nullptr.
 129     ~TlsCertificatesTestWatcher() { state_->watcher = nullptr; }
 130
 131     void OnCertificatesChanged(
 132         absl::optional<absl::string_view> root_certs,
 133         absl::optional<grpc_tls_certificate_distributor::PemKeyCertPairList>
 134             key_cert_pairs) override {
 135       std::string updated_root;
 136       if (root_certs.has_value()) {
 137         updated_root = std::string(*root_certs);
 138       }
 139       grpc_tls_certificate_distributor::PemKeyCertPairList updated_identity;
 140       if (key_cert_pairs.has_value()) {
 141         updated_identity = std::move(*key_cert_pairs);
 142       }
 143       state_->cert_update_queue.emplace_back(std::move(updated_root),
 144                                              std::move(updated_identity));
 145     }
 146
 147     void OnError(grpc_error* root_cert_error,
 148                  grpc_error* identity_cert_error) override {
 149       GPR_ASSERT(root_cert_error != GRPC_ERROR_NONE ||
 150                  identity_cert_error != GRPC_ERROR_NONE);
 151       std::string root_error_str;
 152       std::string identity_error_str;
 153       if (root_cert_error != GRPC_ERROR_NONE) {
 154         grpc_slice root_error_slice;
 155         GPR_ASSERT(grpc_error_get_str(
 156             root_cert_error, GRPC_ERROR_STR_DESCRIPTION, &root_error_slice));
 157         root_error_str =
 158             std::string(grpc_core::StringViewFromSlice(root_error_slice));
 159       }
 160       if (identity_cert_error != GRPC_ERROR_NONE) {
 161         grpc_slice identity_error_slice;
 162         GPR_ASSERT(grpc_error_get_str(identity_cert_error,
 163                                       GRPC_ERROR_STR_DESCRIPTION,
 164                                       &identity_error_slice));
 165         identity_error_str =
 166             std::string(grpc_core::StringViewFromSlice(identity_error_slice));
 167       }
 168       state_->error_queue.emplace_back(std::move(root_error_str),
 169                                        std::move(identity_error_str));
 170       GRPC_ERROR_UNREF(root_cert_error);
 171       GRPC_ERROR_UNREF(identity_cert_error);
 172     }
 173
 174    private:
 175     WatcherState* state_;
 176   };
 177
 178   // CallbackStatus contains the parameters when calling watch_status_callback_
 179   // of the distributor. When a particular callback is invoked, we will push a
 180   // CallbackStatus to a callback_queue_, and check in each test if the status
 181   // updates are correct.
 182   struct CallbackStatus {
 183     std::string cert_name;
 184     bool root_being_watched;
 185     bool identity_being_watched;
 186     CallbackStatus(std::string name, bool root_watched, bool identity_watched)
 187         : cert_name(std::move(name)),
 188           root_being_watched(root_watched),
 189           identity_being_watched(identity_watched) {}
 190     bool operator==(const CallbackStatus& other) const {
 191       return cert_name == other.cert_name &&
 192              root_being_watched == other.root_being_watched &&
 193              identity_being_watched == other.identity_being_watched;
 194     }
 195   };
 196
 197   void SetUp() override {
 198     distributor_.SetWatchStatusCallback([this](std::string cert_name,
 199                                                bool root_being_watched,
 200                                                bool identity_being_watched) {
 201       callback_queue_.emplace_back(std::move(cert_name), root_being_watched,
 202                                    identity_being_watched);
 203     });
 204   }
 205
 206   WatcherState* MakeWatcher(absl::optional<std::string> root_cert_name,
 207                             absl::optional<std::string> identity_cert_name) {
 208     grpc_core::MutexLock lock(&mu_);
 209     watchers_.emplace_back();
 210     // TlsCertificatesTestWatcher ctor takes a pointer to the WatcherState.
 211     // It sets WatcherState::watcher to point to itself.
 212     // The TlsCertificatesTestWatcher dtor will set WatcherState::watcher back
 213     // to nullptr to indicate that it's been destroyed.
 214     auto watcher =
 215         absl::make_unique<TlsCertificatesTestWatcher>(&watchers_.back());
 216     distributor_.WatchTlsCertificates(std::move(watcher),
 217                                       std::move(root_cert_name),
 218                                       std::move(identity_cert_name));
 219     return &watchers_.back();
 220   }
 221
 222   void CancelWatch(WatcherState* state) {
 223     grpc_core::MutexLock lock(&mu_);
 224     distributor_.CancelTlsCertificatesWatch(state->watcher);
 225     EXPECT_EQ(state->watcher, nullptr);
 226   }
 227
 228   std::deque<CallbackStatus> GetCallbackQueue() {
 229     // We move the data member value so the data member will be re-initiated
 230     // with size 0, and ready for the next check.
 231     return std::move(callback_queue_);
 232   }
 233
 234   grpc_tls_certificate_distributor distributor_;
 235   // Use a std::list<> here to avoid the address invalidation caused by internal
 236   // reallocation of std::vector<>.
 237   std::list<WatcherState> watchers_;
 238   std::deque<CallbackStatus> callback_queue_;
 239   // This is to make watchers_ and callback_queue_ thread-safe.
 240   grpc_core::Mutex mu_;
 241 };
 242
 243 TEST_F(GrpcTlsCertificateDistributorTest, BasicCredentialBehaviors) {
 244   EXPECT_FALSE(distributor_.HasRootCerts(kRootCert1Name));
 245   EXPECT_FALSE(distributor_.HasKeyCertPairs(kIdentityCert1Name));
 246   // After setting the certificates to the corresponding cert names, the
 247   // distributor should possess the corresponding certs.
 248   distributor_.SetKeyMaterials(kRootCert1Name, kRootCert1Contents,
 249                                absl::nullopt);
 250   EXPECT_TRUE(distributor_.HasRootCerts(kRootCert1Name));
 251   distributor_.SetKeyMaterials(
 252       kIdentityCert1Name, absl::nullopt,
 253       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 254   EXPECT_TRUE(distributor_.HasKeyCertPairs(kIdentityCert1Name));
 255   // Querying a non-existing cert name should return false.
 256   EXPECT_FALSE(distributor_.HasRootCerts(kRootCert2Name));
 257   EXPECT_FALSE(distributor_.HasKeyCertPairs(kIdentityCert2Name));
 258 }
 259
 260 TEST_F(GrpcTlsCertificateDistributorTest, UpdateCredentialsOnAnySide) {
 261   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 262   EXPECT_THAT(GetCallbackQueue(),
 263               testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 264   // SetKeyMaterials should trigger watcher's OnCertificatesChanged method.
 265   distributor_.SetKeyMaterials(
 266       kCertName1, kRootCert1Contents,
 267       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 268   EXPECT_THAT(
 269       watcher_state_1->GetCredentialQueue(),
 270       testing::ElementsAre(CredentialInfo(
 271           kRootCert1Contents,
 272           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 273   // Set root certs should trigger watcher's OnCertificatesChanged again.
 274   distributor_.SetKeyMaterials(kCertName1, kRootCert2Contents, absl::nullopt);
 275   EXPECT_THAT(
 276       watcher_state_1->GetCredentialQueue(),
 277       testing::ElementsAre(CredentialInfo(
 278           kRootCert2Contents,
 279           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 280   // Set identity certs should trigger watcher's OnCertificatesChanged again.
 281   distributor_.SetKeyMaterials(
 282       kCertName1, absl::nullopt,
 283       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 284   EXPECT_THAT(
 285       watcher_state_1->GetCredentialQueue(),
 286       testing::ElementsAre(CredentialInfo(
 287           kRootCert2Contents,
 288           MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents))));
 289   CancelWatch(watcher_state_1);
 290 }
 291
 292 TEST_F(GrpcTlsCertificateDistributorTest, SameIdentityNameDiffRootName) {
 293   // Register watcher 1.
 294   WatcherState* watcher_state_1 =
 295       MakeWatcher(kRootCert1Name, kIdentityCert1Name);
 296   EXPECT_THAT(
 297       GetCallbackQueue(),
 298       testing::ElementsAre(CallbackStatus(kRootCert1Name, true, false),
 299                            CallbackStatus(kIdentityCert1Name, false, true)));
 300   // Register watcher 2.
 301   WatcherState* watcher_state_2 =
 302       MakeWatcher(kRootCert2Name, kIdentityCert1Name);
 303   EXPECT_THAT(GetCallbackQueue(), testing::ElementsAre(CallbackStatus(
 304                                       kRootCert2Name, true, false)));
 305   // Push credential updates to kRootCert1Name and check if the status works as
 306   // expected.
 307   distributor_.SetKeyMaterials(kRootCert1Name, kRootCert1Contents,
 308                                absl::nullopt);
 309   // Check the updates are delivered to watcher 1.
 310   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 311               testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
 312   // Push credential updates to kRootCert2Name.
 313   distributor_.SetKeyMaterials(kRootCert2Name, kRootCert2Contents,
 314                                absl::nullopt);
 315   // Check the updates are delivered to watcher 2.
 316   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 317               testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
 318   // Push credential updates to kIdentityCert1Name and check if the status works
 319   // as expected.
 320   distributor_.SetKeyMaterials(
 321       kIdentityCert1Name, absl::nullopt,
 322       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 323   // Check the updates are delivered to watcher 1 and watcher 2.
 324   EXPECT_THAT(
 325       watcher_state_1->GetCredentialQueue(),
 326       testing::ElementsAre(CredentialInfo(
 327           kRootCert1Contents,
 328           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 329   EXPECT_THAT(
 330       watcher_state_2->GetCredentialQueue(),
 331       testing::ElementsAre(CredentialInfo(
 332           kRootCert2Contents,
 333           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 334   // Cancel watcher 1.
 335   CancelWatch(watcher_state_1);
 336   EXPECT_THAT(GetCallbackQueue(), testing::ElementsAre(CallbackStatus(
 337                                       kRootCert1Name, false, false)));
 338   // Cancel watcher 2.
 339   CancelWatch(watcher_state_2);
 340   EXPECT_THAT(
 341       GetCallbackQueue(),
 342       testing::ElementsAre(CallbackStatus(kRootCert2Name, false, false),
 343                            CallbackStatus(kIdentityCert1Name, false, false)));
 344 }
 345
 346 TEST_F(GrpcTlsCertificateDistributorTest, SameRootNameDiffIdentityName) {
 347   // Register watcher 1.
 348   WatcherState* watcher_state_1 =
 349       MakeWatcher(kRootCert1Name, kIdentityCert1Name);
 350   EXPECT_THAT(
 351       GetCallbackQueue(),
 352       testing::ElementsAre(CallbackStatus(kRootCert1Name, true, false),
 353                            CallbackStatus(kIdentityCert1Name, false, true)));
 354   // Register watcher 2.
 355   WatcherState* watcher_state_2 =
 356       MakeWatcher(kRootCert1Name, kIdentityCert2Name);
 357   EXPECT_THAT(GetCallbackQueue(), testing::ElementsAre(CallbackStatus(
 358                                       kIdentityCert2Name, false, true)));
 359   // Push credential updates to kRootCert1Name and check if the status works as
 360   // expected.
 361   distributor_.SetKeyMaterials(kRootCert1Name, kRootCert1Contents,
 362                                absl::nullopt);
 363   // Check the updates are delivered to watcher 1.
 364   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 365               testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
 366   // Check the updates are delivered to watcher 2.
 367   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 368               testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
 369   // Push credential updates to SetKeyMaterials.
 370   distributor_.SetKeyMaterials(
 371       kIdentityCert1Name, absl::nullopt,
 372       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 373   // Check the updates are delivered to watcher 1.
 374   EXPECT_THAT(
 375       watcher_state_1->GetCredentialQueue(),
 376       testing::ElementsAre(CredentialInfo(
 377           kRootCert1Contents,
 378           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 379   // Push credential updates to kIdentityCert2Name.
 380   distributor_.SetKeyMaterials(
 381       kIdentityCert2Name, absl::nullopt,
 382       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 383   // Check the updates are delivered to watcher 2.
 384   EXPECT_THAT(
 385       watcher_state_2->GetCredentialQueue(),
 386       testing::ElementsAre(CredentialInfo(
 387           kRootCert1Contents,
 388           MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents))));
 389   // Cancel watcher 1.
 390   CancelWatch(watcher_state_1);
 391   EXPECT_THAT(GetCallbackQueue(), testing::ElementsAre(CallbackStatus(
 392                                       kIdentityCert1Name, false, false)));
 393   // Cancel watcher 2.
 394   CancelWatch(watcher_state_2);
 395   EXPECT_THAT(
 396       GetCallbackQueue(),
 397       testing::ElementsAre(CallbackStatus(kRootCert1Name, false, false),
 398                            CallbackStatus(kIdentityCert2Name, false, false)));
 399 }
 400
 401 TEST_F(GrpcTlsCertificateDistributorTest,
 402        AddAndCancelFirstWatcherForSameRootAndIdentityCertName) {
 403   // Register watcher 1 watching kCertName1 for both root and identity certs.
 404   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 405   EXPECT_THAT(GetCallbackQueue(),
 406               testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 407   // Push credential updates to kCertName1 and check if the status works as
 408   // expected.
 409   distributor_.SetKeyMaterials(
 410       kCertName1, kRootCert1Contents,
 411       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 412   // Check the updates are delivered to watcher 1.
 413   EXPECT_THAT(
 414       watcher_state_1->GetCredentialQueue(),
 415       testing::ElementsAre(CredentialInfo(
 416           kRootCert1Contents,
 417           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 418   // Cancel watcher 1.
 419   CancelWatch(watcher_state_1);
 420   EXPECT_THAT(GetCallbackQueue(),
 421               testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
 422 }
 423
 424 TEST_F(GrpcTlsCertificateDistributorTest,
 425        AddAndCancelFirstWatcherForIdentityCertNameWithRootBeingWatched) {
 426   // Register watcher 1 watching kCertName1 for root certs.
 427   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, absl::nullopt);
 428   EXPECT_THAT(GetCallbackQueue(),
 429               testing::ElementsAre(CallbackStatus(kCertName1, true, false)));
 430   // Register watcher 2 watching kCertName1 for identity certs.
 431   WatcherState* watcher_state_2 = MakeWatcher(absl::nullopt, kCertName1);
 432   EXPECT_THAT(GetCallbackQueue(),
 433               testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 434   // Push credential updates to kCertName1 and check if the status works as
 435   // expected.
 436   distributor_.SetKeyMaterials(
 437       kCertName1, kRootCert1Contents,
 438       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 439   // Check the updates are delivered to watcher 1.
 440   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 441               testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
 442   // Check the updates are delivered to watcher 2.
 443   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 444               testing::ElementsAre(CredentialInfo(
 445                   "", MakeCertKeyPairs(kIdentityCert1PrivateKey,
 446                                        kIdentityCert1Contents))));
 447   // Push root cert updates to kCertName1.
 448   distributor_.SetKeyMaterials(kCertName1, kRootCert2Contents, absl::nullopt);
 449   // Check the updates are delivered to watcher 1.
 450   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 451               testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
 452   // Check the updates are not delivered to watcher 2.
 453   EXPECT_THAT(watcher_state_2->GetCredentialQueue(), testing::ElementsAre());
 454   // Push identity cert updates to kCertName1.
 455   distributor_.SetKeyMaterials(
 456       kCertName1, absl::nullopt,
 457       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 458   // Check the updates are not delivered to watcher 1.
 459   EXPECT_THAT(watcher_state_1->GetCredentialQueue(), testing::ElementsAre());
 460   // Check the updates are delivered to watcher 2.
 461   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 462               testing::ElementsAre(CredentialInfo(
 463                   "", MakeCertKeyPairs(kIdentityCert2PrivateKey,
 464                                        kIdentityCert2Contents))));
 465   watcher_state_2->cert_update_queue.clear();
 466   // Cancel watcher 2.
 467   CancelWatch(watcher_state_2);
 468   EXPECT_THAT(GetCallbackQueue(),
 469               testing::ElementsAre(CallbackStatus(kCertName1, true, false)));
 470   // Cancel watcher 1.
 471   CancelWatch(watcher_state_1);
 472   EXPECT_THAT(GetCallbackQueue(),
 473               testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
 474 }
 475
 476 TEST_F(GrpcTlsCertificateDistributorTest,
 477        AddAndCancelFirstWatcherForRootCertNameWithIdentityBeingWatched) {
 478   // Register watcher 1 watching kCertName1 for identity certs.
 479   WatcherState* watcher_state_1 = MakeWatcher(absl::nullopt, kCertName1);
 480   EXPECT_THAT(GetCallbackQueue(),
 481               testing::ElementsAre(CallbackStatus(kCertName1, false, true)));
 482   // Register watcher 2 watching kCertName1 for root certs.
 483   WatcherState* watcher_state_2 = MakeWatcher(kCertName1, absl::nullopt);
 484   EXPECT_THAT(GetCallbackQueue(),
 485               testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 486   // Push credential updates to kCertName1 and check if the status works as
 487   // expected.
 488   distributor_.SetKeyMaterials(
 489       kCertName1, kRootCert1Contents,
 490       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 491   // Check the updates are delivered to watcher 1.
 492   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 493               testing::ElementsAre(CredentialInfo(
 494                   "", MakeCertKeyPairs(kIdentityCert1PrivateKey,
 495                                        kIdentityCert1Contents))));
 496   // Check the updates are delivered to watcher 2.
 497   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 498               testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
 499   // Push root cert updates to kCertName1.
 500   distributor_.SetKeyMaterials(kCertName1, kRootCert2Contents, absl::nullopt);
 501   // Check the updates are delivered to watcher 2.
 502   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 503               testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
 504   // Check the updates are not delivered to watcher 1.
 505   EXPECT_THAT(watcher_state_1->GetCredentialQueue(), testing::ElementsAre());
 506   // Push identity cert updates to kCertName1.
 507   distributor_.SetKeyMaterials(
 508       kCertName1, absl::nullopt,
 509       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 510   // Check the updates are not delivered to watcher 2.
 511   EXPECT_THAT(watcher_state_2->GetCredentialQueue(), testing::ElementsAre());
 512   // Check the updates are delivered to watcher 1.
 513   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 514               testing::ElementsAre(CredentialInfo(
 515                   "", MakeCertKeyPairs(kIdentityCert2PrivateKey,
 516                                        kIdentityCert2Contents))));
 517   // Cancel watcher 2.
 518   CancelWatch(watcher_state_2);
 519   EXPECT_THAT(GetCallbackQueue(),
 520               testing::ElementsAre(CallbackStatus(kCertName1, false, true)));
 521   // Cancel watcher 1.
 522   CancelWatch(watcher_state_1);
 523   EXPECT_THAT(GetCallbackQueue(),
 524               testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
 525 }
 526
 527 TEST_F(GrpcTlsCertificateDistributorTest,
 528        RemoveAllWatchersForCertNameAndAddAgain) {
 529   // Register watcher 1 and watcher 2 watching kCertName1 for root and identity
 530   // certs.
 531   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 532   EXPECT_THAT(GetCallbackQueue(),
 533               testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 534   WatcherState* watcher_state_2 = MakeWatcher(kCertName1, kCertName1);
 535   EXPECT_THAT(GetCallbackQueue(), testing::ElementsAre());
 536   // Push credential updates to kCertName1.
 537   distributor_.SetKeyMaterials(
 538       kCertName1, kRootCert1Contents,
 539       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 540   // Cancel watcher 2.
 541   CancelWatch(watcher_state_2);
 542   EXPECT_THAT(GetCallbackQueue(), testing::ElementsAre());
 543   // Cancel watcher 1.
 544   CancelWatch(watcher_state_1);
 545   EXPECT_THAT(GetCallbackQueue(),
 546               testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
 547   // Register watcher 3 watching kCertName for root and identity certs.
 548   WatcherState* watcher_state_3 = MakeWatcher(kCertName1, kCertName1);
 549   EXPECT_THAT(GetCallbackQueue(),
 550               testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 551   // Push credential updates to kCertName1.
 552   distributor_.SetKeyMaterials(
 553       kCertName1, kRootCert2Contents,
 554       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 555   // Check the updates are delivered to watcher 3.
 556   EXPECT_THAT(
 557       watcher_state_3->GetCredentialQueue(),
 558       testing::ElementsAre(CredentialInfo(
 559           kRootCert2Contents,
 560           MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents))));
 561   // Cancel watcher 3.
 562   CancelWatch(watcher_state_3);
 563   EXPECT_THAT(GetCallbackQueue(),
 564               testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
 565 }
 566
 567 TEST_F(GrpcTlsCertificateDistributorTest, ResetCallbackToNull) {
 568   // Register watcher 1 watching kCertName1 for root and identity certs.
 569   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 570   EXPECT_THAT(GetCallbackQueue(),
 571               testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
 572   // Reset callback to nullptr.
 573   distributor_.SetWatchStatusCallback(nullptr);
 574   // Cancel watcher 1 shouldn't trigger any callback.
 575   CancelWatch(watcher_state_1);
 576   EXPECT_THAT(GetCallbackQueue(), testing::ElementsAre());
 577 }
 578
 579 TEST_F(GrpcTlsCertificateDistributorTest, SetKeyMaterialsInCallback) {
 580   distributor_.SetWatchStatusCallback([this](std::string cert_name,
 581                                              bool root_being_watched,
 582                                              bool identity_being_watched) {
 583     distributor_.SetKeyMaterials(
 584         cert_name, kRootCert1Contents,
 585         MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 586   });
 587   auto verify_function = [this](std::string cert_name) {
 588     WatcherState* watcher_state_1 = MakeWatcher(cert_name, cert_name);
 589     // Check the updates are delivered to watcher 1.
 590     EXPECT_THAT(
 591         watcher_state_1->GetCredentialQueue(),
 592         testing::ElementsAre(CredentialInfo(
 593             kRootCert1Contents, MakeCertKeyPairs(kIdentityCert1PrivateKey,
 594                                                  kIdentityCert1Contents))));
 595     CancelWatch(watcher_state_1);
 596   };
 597   // Start 1000 threads that will register a watcher to a new cert name, verify
 598   // the key materials being set, and then cancel the watcher, to make sure the
 599   // lock mechanism in the distributor is safe.
 600   std::vector<std::thread> threads;
 601   threads.reserve(1000);
 602   for (int i = 0; i < 1000; ++i) {
 603     threads.emplace_back(verify_function, std::to_string(i));
 604   }
 605   for (auto& th : threads) {
 606     th.join();
 607   }
 608 }
 609
 610 TEST_F(GrpcTlsCertificateDistributorTest, WatchACertInfoWithValidCredentials) {
 611   // Push credential updates to kCertName1.
 612   distributor_.SetKeyMaterials(
 613       kCertName1, kRootCert1Contents,
 614       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 615   // Push root credential updates to kCertName2.
 616   distributor_.SetKeyMaterials(kRootCert2Name, kRootCert2Contents,
 617                                absl::nullopt);
 618   // Push identity credential updates to kCertName2.
 619   distributor_.SetKeyMaterials(
 620       kIdentityCert2Name, absl::nullopt,
 621       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
 622   // Register watcher 1.
 623   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 624   // watcher 1 should receive the credentials right away.
 625   EXPECT_THAT(
 626       watcher_state_1->GetCredentialQueue(),
 627       testing::ElementsAre(CredentialInfo(
 628           kRootCert1Contents,
 629           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 630   CancelWatch(watcher_state_1);
 631   // Register watcher 2.
 632   WatcherState* watcher_state_2 = MakeWatcher(kRootCert2Name, absl::nullopt);
 633   // watcher 2 should receive the root credentials right away.
 634   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 635               testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
 636   // Register watcher 3.
 637   WatcherState* watcher_state_3 =
 638       MakeWatcher(absl::nullopt, kIdentityCert2Name);
 639   // watcher 3 should received the identity credentials right away.
 640   EXPECT_THAT(watcher_state_3->GetCredentialQueue(),
 641               testing::ElementsAre(CredentialInfo(
 642                   "", MakeCertKeyPairs(kIdentityCert2PrivateKey,
 643                                        kIdentityCert2Contents))));
 644   CancelWatch(watcher_state_2);
 645   CancelWatch(watcher_state_3);
 646 }
 647
 648 TEST_F(GrpcTlsCertificateDistributorTest,
 649        SetErrorForCertForBothRootAndIdentity) {
 650   // Register watcher 1.
 651   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 652   // Calling SetErrorForCert on both cert names should only call one OnError
 653   // on watcher 1.
 654   distributor_.SetErrorForCert(
 655       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 656       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 657   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 658               testing::ElementsAre(
 659                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 660   // Calling SetErrorForCert on root cert name should call OnError
 661   // on watcher 1 again.
 662   distributor_.SetErrorForCert(
 663       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage),
 664       absl::nullopt);
 665   EXPECT_THAT(
 666       watcher_state_1->GetErrorQueue(),
 667       testing::ElementsAre(ErrorInfo(kErrorMessage, kIdentityErrorMessage)));
 668   // Calling SetErrorForCert on identity cert name should call OnError
 669   // on watcher 1 again.
 670   distributor_.SetErrorForCert(
 671       kCertName1, absl::nullopt,
 672       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage));
 673   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 674               testing::ElementsAre(ErrorInfo(kErrorMessage, kErrorMessage)));
 675   distributor_.CancelTlsCertificatesWatch(watcher_state_1->watcher);
 676   EXPECT_EQ(watcher_state_1->watcher, nullptr);
 677 }
 678
 679 TEST_F(GrpcTlsCertificateDistributorTest, SetErrorForCertForRootOrIdentity) {
 680   // Register watcher 1.
 681   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, absl::nullopt);
 682   // Calling SetErrorForCert on root name should only call one OnError
 683   // on watcher 1.
 684   distributor_.SetErrorForCert(
 685       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 686       absl::nullopt);
 687   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 688               testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
 689   // Calling SetErrorForCert on identity name should do nothing.
 690   distributor_.SetErrorForCert(
 691       kCertName1, absl::nullopt,
 692       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 693   EXPECT_THAT(watcher_state_1->GetErrorQueue(), testing::ElementsAre());
 694   // Calling SetErrorForCert on both names should still get one OnError call.
 695   distributor_.SetErrorForCert(
 696       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 697       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 698   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 699               testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
 700   CancelWatch(watcher_state_1);
 701   // Register watcher 2.
 702   WatcherState* watcher_state_2 = MakeWatcher(absl::nullopt, kCertName1);
 703   // Calling SetErrorForCert on identity name should only call one OnError
 704   // on watcher 2.
 705   distributor_.SetErrorForCert(
 706       kCertName1, absl::nullopt,
 707       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 708   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
 709               testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
 710   // Calling SetErrorForCert on root name should do nothing.
 711   distributor_.SetErrorForCert(
 712       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 713       absl::nullopt);
 714   EXPECT_THAT(watcher_state_2->GetErrorQueue(), testing::ElementsAre());
 715   // Calling SetErrorForCert on both names should still get one OnError call.
 716   distributor_.SetErrorForCert(
 717       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 718       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 719   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
 720               testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
 721   CancelWatch(watcher_state_2);
 722 }
 723
 724 TEST_F(GrpcTlsCertificateDistributorTest,
 725        SetErrorForIdentityNameWithPreexistingErrorForRootName) {
 726   // SetErrorForCert for kCertName1.
 727   distributor_.SetErrorForCert(
 728       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 729       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 730   // Register watcher 1 for kCertName1 as root and kCertName2 as identity.
 731   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName2);
 732   // Should trigger OnError call right away since kCertName1 has error.
 733   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 734               testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
 735   // Calling SetErrorForCert on kCertName2 should trigger OnError with both
 736   // errors, because kCertName1 also has error.
 737   distributor_.SetErrorForCert(
 738       kCertName2, absl::nullopt,
 739       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 740   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 741               testing::ElementsAre(
 742                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 743   CancelWatch(watcher_state_1);
 744 }
 745
 746 TEST_F(GrpcTlsCertificateDistributorTest,
 747        SetErrorForCertForRootNameWithSameNameForIdentityErrored) {
 748   // SetErrorForCert for kCertName1.
 749   distributor_.SetErrorForCert(
 750       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 751       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 752   // Register watcher 1 for kCertName2 as root and kCertName1 as identity.
 753   WatcherState* watcher_state_1 = MakeWatcher(kCertName2, kCertName1);
 754   // Should trigger OnError call right away since kCertName2 has error.
 755   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 756               testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
 757   // Calling SetErrorForCert on kCertName2 should trigger OnError with both
 758   // errors, because kCertName1 also has error.
 759   distributor_.SetErrorForCert(
 760       kCertName2, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 761       absl::nullopt);
 762   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 763               testing::ElementsAre(
 764                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 765   CancelWatch(watcher_state_1);
 766 }
 767
 768 TEST_F(GrpcTlsCertificateDistributorTest,
 769        SetErrorForIdentityNameWithoutErrorForRootName) {
 770   // Register watcher 1 for kCertName1 as root and kCertName2 as identity.
 771   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName2);
 772   // Should not trigger OnError.
 773   EXPECT_THAT(watcher_state_1->GetErrorQueue(), testing::ElementsAre());
 774   // Calling SetErrorForCert on kCertName2 should trigger OnError.
 775   distributor_.SetErrorForCert(
 776       kCertName2, absl::nullopt,
 777       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 778   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 779               testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
 780   CancelWatch(watcher_state_1);
 781   // Register watcher 2 for kCertName2 as identity and a non-existing name
 782   // kRootCert1Name as root.
 783   WatcherState* watcher_state_2 = MakeWatcher(kRootCert1Name, kCertName2);
 784   // Should not trigger OnError.
 785   EXPECT_THAT(watcher_state_2->GetErrorQueue(), testing::ElementsAre());
 786   // Calling SetErrorForCert on kCertName2 should trigger OnError.
 787   distributor_.SetErrorForCert(
 788       kCertName2, absl::nullopt,
 789       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 790   EXPECT_THAT(watcher_state_2->error_queue,
 791               testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
 792   CancelWatch(watcher_state_2);
 793 }
 794
 795 TEST_F(GrpcTlsCertificateDistributorTest,
 796        SetErrorForRootNameWithPreexistingErrorForIdentityName) {
 797   WatcherState* watcher_state_1 = MakeWatcher(kCertName2, kCertName1);
 798   // Should not trigger OnError.
 799   EXPECT_THAT(watcher_state_1->GetErrorQueue(), testing::ElementsAre());
 800   // Calling SetErrorForCert on kCertName2 should trigger OnError.
 801   distributor_.SetErrorForCert(
 802       kCertName2, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 803       absl::nullopt);
 804   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 805               testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
 806   CancelWatch(watcher_state_1);
 807   // Register watcher 2 for kCertName2 as root and a non-existing name
 808   // kIdentityCert1Name as identity.
 809   WatcherState* watcher_state_2 = MakeWatcher(kCertName2, kIdentityCert1Name);
 810   // Should not trigger OnError.
 811   EXPECT_THAT(watcher_state_2->GetErrorQueue(), testing::ElementsAre());
 812   // Calling SetErrorForCert on kCertName2 should trigger OnError.
 813   distributor_.SetErrorForCert(
 814       kCertName2, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 815       absl::nullopt);
 816   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
 817               testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
 818   CancelWatch(watcher_state_2);
 819 }
 820
 821 TEST_F(GrpcTlsCertificateDistributorTest,
 822        CancelTheLastWatcherOnAnErroredCertInfo) {
 823   // Register watcher 1.
 824   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 825   // Calling SetErrorForCert on both cert names should only call one OnError
 826   // on watcher 1.
 827   distributor_.SetErrorForCert(
 828       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 829       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 830   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 831               testing::ElementsAre(
 832                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 833   // When watcher 1 is removed, the cert info entry should be removed.
 834   CancelWatch(watcher_state_1);
 835   // Register watcher 2 on the same cert name.
 836   WatcherState* watcher_state_2 = MakeWatcher(kCertName1, kCertName1);
 837   // Should not trigger OnError call on watcher 2 right away.
 838   EXPECT_THAT(watcher_state_2->GetErrorQueue(), testing::ElementsAre());
 839   CancelWatch(watcher_state_2);
 840 }
 841
 842 TEST_F(GrpcTlsCertificateDistributorTest,
 843        WatchErroredCertInfoWithValidCredentialData) {
 844   // Push credential updates to kCertName1.
 845   distributor_.SetKeyMaterials(
 846       kCertName1, kRootCert1Contents,
 847       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 848   // Calling SetErrorForCert on both cert names.
 849   distributor_.SetErrorForCert(
 850       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 851       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 852   // Register watcher 1.
 853   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 854   // watcher 1 should receive both the old credentials and the error right away.
 855   EXPECT_THAT(
 856       watcher_state_1->GetCredentialQueue(),
 857       testing::ElementsAre(CredentialInfo(
 858           kRootCert1Contents,
 859           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 860   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 861               testing::ElementsAre(
 862                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 863   CancelWatch(watcher_state_1);
 864 }
 865
 866 TEST_F(GrpcTlsCertificateDistributorTest,
 867        SetErrorForCertThenSuccessfulCredentialUpdates) {
 868   // Calling SetErrorForCert on both cert names.
 869   distributor_.SetErrorForCert(
 870       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 871       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 872   // Push credential updates to kCertName1.
 873   distributor_.SetKeyMaterials(
 874       kCertName1, kRootCert1Contents,
 875       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
 876   // Register watcher 1.
 877   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 878   // watcher 1 should only receive credential updates without any error, because
 879   // the previous error is wiped out by a successful update.
 880   EXPECT_THAT(
 881       watcher_state_1->GetCredentialQueue(),
 882       testing::ElementsAre(CredentialInfo(
 883           kRootCert1Contents,
 884           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
 885   EXPECT_THAT(watcher_state_1->GetErrorQueue(), testing::ElementsAre());
 886   CancelWatch(watcher_state_1);
 887 }
 888
 889 TEST_F(GrpcTlsCertificateDistributorTest, WatchCertInfoThenInvokeSetError) {
 890   // Register watcher 1.
 891   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
 892   // Register watcher 2.
 893   WatcherState* watcher_state_2 = MakeWatcher(kRootCert1Name, absl::nullopt);
 894   // Register watcher 3.
 895   WatcherState* watcher_state_3 =
 896       MakeWatcher(absl::nullopt, kIdentityCert1Name);
 897   distributor_.SetError(GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage));
 898   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 899               testing::ElementsAre(ErrorInfo(kErrorMessage, kErrorMessage)));
 900   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
 901               testing::ElementsAre(ErrorInfo(kErrorMessage, "")));
 902   EXPECT_THAT(watcher_state_3->GetErrorQueue(),
 903               testing::ElementsAre(ErrorInfo("", kErrorMessage)));
 904   CancelWatch(watcher_state_1);
 905   CancelWatch(watcher_state_2);
 906   CancelWatch(watcher_state_3);
 907 }
 908
 909 TEST_F(GrpcTlsCertificateDistributorTest, WatchErroredCertInfoBySetError) {
 910   // Register watcher 1 watching kCertName1 as root.
 911   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, absl::nullopt);
 912   // Register watcher 2 watching kCertName2 as identity.
 913   WatcherState* watcher_state_2 = MakeWatcher(absl::nullopt, kCertName2);
 914   // Call SetError and then cancel all watchers.
 915   distributor_.SetError(GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage));
 916   CancelWatch(watcher_state_1);
 917   CancelWatch(watcher_state_2);
 918   // Register watcher 3 watching kCertName1 as root and kCertName2 as identity
 919   // should not get the error updates.
 920   WatcherState* watcher_state_3 = MakeWatcher(kCertName1, kCertName2);
 921   EXPECT_THAT(watcher_state_3->GetErrorQueue(), testing::ElementsAre());
 922   CancelWatch(watcher_state_3);
 923   // Register watcher 4 watching kCertName2 as root and kCertName1 as identity
 924   // should not get the error updates.
 925   WatcherState* watcher_state_4 = MakeWatcher(kCertName2, kCertName1);
 926   EXPECT_THAT(watcher_state_4->GetErrorQueue(), testing::ElementsAre());
 927   CancelWatch(watcher_state_4);
 928 }
 929
 930 TEST_F(GrpcTlsCertificateDistributorTest, SetErrorForCertInCallback) {
 931   distributor_.SetWatchStatusCallback([this](std::string cert_name,
 932                                              bool root_being_watched,
 933                                              bool identity_being_watched) {
 934     this->distributor_.SetErrorForCert(
 935         cert_name, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
 936         GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
 937   });
 938   auto verify_function = [this](std::string cert_name) {
 939     WatcherState* watcher_state_1 = MakeWatcher(cert_name, cert_name);
 940     // Check the errors are delivered to watcher 1.
 941     EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 942                 testing::ElementsAre(
 943                     ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
 944     CancelWatch(watcher_state_1);
 945   };
 946   // Start 1000 threads that will register a watcher to a new cert name, verify
 947   // the key materials being set, and then cancel the watcher, to make sure the
 948   // lock mechanism in the distributor is safe.
 949   std::vector<std::thread> threads;
 950   threads.reserve(1000);
 951   for (int i = 0; i < 1000; ++i) {
 952     threads.emplace_back(verify_function, std::to_string(i));
 953   }
 954   for (auto& th : threads) {
 955     th.join();
 956   }
 957 }
 958
 959 }  // namespace testing
 960
 961 int main(int argc, char** argv) {
 962   grpc::testing::TestEnvironment env(argc, argv);
 963   ::testing::InitGoogleTest(&argc, argv);
 964   grpc_init();
 965   int ret = RUN_ALL_TESTS();
 966   grpc_shutdown();
 967   return ret;
 968 }