test/core/security/grpc_tls_certificate_provider_test.cc

   1 //
   2 // Copyright 2020 gRPC authors.
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //     http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 //
  16
  17 #include "src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h"
  18
  19 #include <deque>
  20 #include <list>
  21
  22 #include <gmock/gmock.h>
  23 #include <gtest/gtest.h>
  24
  25 #include <grpc/support/alloc.h>
  26 #include <grpc/support/log.h>
  27 #include <grpc/support/string_util.h>
  28
  29 #include "src/core/lib/gpr/tmpfile.h"
  30 #include "src/core/lib/iomgr/load_file.h"
  31 #include "src/core/lib/slice/slice_internal.h"
  32 #include "test/core/util/test_config.h"
  33 #include "test/core/util/tls_utils.h"
  34
  35 #define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem"
  36 #define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem"
  37 #define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key"
  38 #define CA_CERT_PATH_2 "src/core/tsi/test_creds/multi-domain.pem"
  39 #define SERVER_CERT_PATH_2 "src/core/tsi/test_creds/server0.pem"
  40 #define SERVER_KEY_PATH_2 "src/core/tsi/test_creds/server0.key"
  41 #define INVALID_PATH "invalid/path"
  42
  43 namespace grpc_core {
  44
  45 namespace testing {
  46
  47 constexpr const char* kCertName = "cert_name";
  48 constexpr const char* kRootError = "Unable to get latest root certificates.";
  49 constexpr const char* kIdentityError =
  50     "Unable to get latest identity certificates.";
  51
  52 class GrpcTlsCertificateProviderTest : public ::testing::Test {
  53  protected:
  54   // Forward declaration.
  55   class TlsCertificatesTestWatcher;
  56
  57   // CredentialInfo contains the parameters when calling OnCertificatesChanged
  58   // of a watcher. When OnCertificatesChanged is invoked, we will push a
  59   // CredentialInfo to the cert_update_queue of state_, and check in each test
  60   // if the status updates are correct.
  61   struct CredentialInfo {
  62     std::string root_certs;
  63     PemKeyCertPairList key_cert_pairs;
  64     CredentialInfo(std::string root, PemKeyCertPairList key_cert)
  65         : root_certs(std::move(root)), key_cert_pairs(std::move(key_cert)) {}
  66     bool operator==(const CredentialInfo& other) const {
  67       return root_certs == other.root_certs &&
  68              key_cert_pairs == other.key_cert_pairs;
  69     }
  70   };
  71
  72   // ErrorInfo contains the parameters when calling OnError of a watcher. When
  73   // OnError is invoked, we will push a ErrorInfo to the error_queue of state_,
  74   // and check in each test if the status updates are correct.
  75   struct ErrorInfo {
  76     std::string root_cert_str;
  77     std::string identity_cert_str;
  78     ErrorInfo(std::string root, std::string identity)
  79         : root_cert_str(std::move(root)),
  80           identity_cert_str(std::move(identity)) {}
  81     bool operator==(const ErrorInfo& other) const {
  82       return root_cert_str == other.root_cert_str &&
  83              identity_cert_str == other.identity_cert_str;
  84     }
  85   };
  86
  87   struct WatcherState {
  88     TlsCertificatesTestWatcher* watcher = nullptr;
  89     std::deque<CredentialInfo> cert_update_queue;
  90     std::deque<ErrorInfo> error_queue;
  91     Mutex mu;
  92
  93     std::deque<CredentialInfo> GetCredentialQueue() {
  94       // We move the data member value so the data member will be re-initiated
  95       // with size 0, and ready for the next check.
  96       MutexLock lock(&mu);
  97       return std::move(cert_update_queue);
  98     }
  99     std::deque<ErrorInfo> GetErrorQueue() {
 100       // We move the data member value so the data member will be re-initiated
 101       // with size 0, and ready for the next check.
 102       MutexLock lock(&mu);
 103       return std::move(error_queue);
 104     }
 105   };
 106
 107   class TlsCertificatesTestWatcher : public grpc_tls_certificate_distributor::
 108                                          TlsCertificatesWatcherInterface {
 109    public:
 110     // ctor sets state->watcher to this.
 111     explicit TlsCertificatesTestWatcher(WatcherState* state) : state_(state) {
 112       state_->watcher = this;
 113     }
 114
 115     // dtor sets state->watcher to nullptr.
 116     ~TlsCertificatesTestWatcher() override { state_->watcher = nullptr; }
 117
 118     void OnCertificatesChanged(
 119         absl::optional<absl::string_view> root_certs,
 120         absl::optional<PemKeyCertPairList> key_cert_pairs) override {
 121       MutexLock lock(&state_->mu);
 122       std::string updated_root;
 123       if (root_certs.has_value()) {
 124         updated_root = std::string(*root_certs);
 125       }
 126       PemKeyCertPairList updated_identity;
 127       if (key_cert_pairs.has_value()) {
 128         updated_identity = std::move(*key_cert_pairs);
 129       }
 130       state_->cert_update_queue.emplace_back(std::move(updated_root),
 131                                              std::move(updated_identity));
 132     }
 133
 134     void OnError(grpc_error_handle root_cert_error,
 135                  grpc_error_handle identity_cert_error) override {
 136       MutexLock lock(&state_->mu);
 137       GPR_ASSERT(root_cert_error != GRPC_ERROR_NONE ||
 138                  identity_cert_error != GRPC_ERROR_NONE);
 139       std::string root_error_str;
 140       std::string identity_error_str;
 141       if (root_cert_error != GRPC_ERROR_NONE) {
 142         grpc_slice root_error_slice;
 143         GPR_ASSERT(grpc_error_get_str(
 144             root_cert_error, GRPC_ERROR_STR_DESCRIPTION, &root_error_slice));
 145         root_error_str = std::string(StringViewFromSlice(root_error_slice));
 146       }
 147       if (identity_cert_error != GRPC_ERROR_NONE) {
 148         grpc_slice identity_error_slice;
 149         GPR_ASSERT(grpc_error_get_str(identity_cert_error,
 150                                       GRPC_ERROR_STR_DESCRIPTION,
 151                                       &identity_error_slice));
 152         identity_error_str =
 153             std::string(StringViewFromSlice(identity_error_slice));
 154       }
 155       state_->error_queue.emplace_back(std::move(root_error_str),
 156                                        std::move(identity_error_str));
 157       GRPC_ERROR_UNREF(root_cert_error);
 158       GRPC_ERROR_UNREF(identity_cert_error);
 159     }
 160
 161    private:
 162     WatcherState* state_;
 163   };
 164
 165   void SetUp() override {
 166     root_cert_ = GetFileContents(CA_CERT_PATH);
 167     cert_chain_ = GetFileContents(SERVER_CERT_PATH);
 168     private_key_ = GetFileContents(SERVER_KEY_PATH);
 169     root_cert_2_ = GetFileContents(CA_CERT_PATH_2);
 170     cert_chain_2_ = GetFileContents(SERVER_CERT_PATH_2);
 171     private_key_2_ = GetFileContents(SERVER_KEY_PATH_2);
 172   }
 173
 174   WatcherState* MakeWatcher(
 175       RefCountedPtr<grpc_tls_certificate_distributor> distributor,
 176       absl::optional<std::string> root_cert_name,
 177       absl::optional<std::string> identity_cert_name) {
 178     MutexLock lock(&mu_);
 179     distributor_ = distributor;
 180     watchers_.emplace_back();
 181     // TlsCertificatesTestWatcher ctor takes a pointer to the WatcherState.
 182     // It sets WatcherState::watcher to point to itself.
 183     // The TlsCertificatesTestWatcher dtor will set WatcherState::watcher back
 184     // to nullptr to indicate that it's been destroyed.
 185     auto watcher =
 186         absl::make_unique<TlsCertificatesTestWatcher>(&watchers_.back());
 187     distributor_->WatchTlsCertificates(std::move(watcher),
 188                                        std::move(root_cert_name),
 189                                        std::move(identity_cert_name));
 190     return &watchers_.back();
 191   }
 192
 193   void CancelWatch(WatcherState* state) {
 194     MutexLock lock(&mu_);
 195     distributor_->CancelTlsCertificatesWatch(state->watcher);
 196     EXPECT_EQ(state->watcher, nullptr);
 197   }
 198
 199   std::string root_cert_;
 200   std::string private_key_;
 201   std::string cert_chain_;
 202   std::string root_cert_2_;
 203   std::string private_key_2_;
 204   std::string cert_chain_2_;
 205   RefCountedPtr<grpc_tls_certificate_distributor> distributor_;
 206   // Use a std::list<> here to avoid the address invalidation caused by internal
 207   // reallocation of std::vector<>.
 208   std::list<WatcherState> watchers_;
 209   // This is to make watchers_ thread-safe.
 210   Mutex mu_;
 211 };
 212
 213 TEST_F(GrpcTlsCertificateProviderTest, StaticDataCertificateProviderCreation) {
 214   StaticDataCertificateProvider provider(
 215       root_cert_, MakeCertKeyPairs(private_key_.c_str(), cert_chain_.c_str()));
 216   // Watcher watching both root and identity certs.
 217   WatcherState* watcher_state_1 =
 218       MakeWatcher(provider.distributor(), kCertName, kCertName);
 219   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 220               ::testing::ElementsAre(CredentialInfo(
 221                   root_cert_, MakeCertKeyPairs(private_key_.c_str(),
 222                                                cert_chain_.c_str()))));
 223   CancelWatch(watcher_state_1);
 224   // Watcher watching only root certs.
 225   WatcherState* watcher_state_2 =
 226       MakeWatcher(provider.distributor(), kCertName, absl::nullopt);
 227   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 228               ::testing::ElementsAre(CredentialInfo(root_cert_, {})));
 229   CancelWatch(watcher_state_2);
 230   // Watcher watching only identity certs.
 231   WatcherState* watcher_state_3 =
 232       MakeWatcher(provider.distributor(), absl::nullopt, kCertName);
 233   EXPECT_THAT(
 234       watcher_state_3->GetCredentialQueue(),
 235       ::testing::ElementsAre(CredentialInfo(
 236           "", MakeCertKeyPairs(private_key_.c_str(), cert_chain_.c_str()))));
 237   CancelWatch(watcher_state_3);
 238 }
 239
 240 TEST_F(GrpcTlsCertificateProviderTest,
 241        FileWatcherCertificateProviderWithGoodPaths) {
 242   FileWatcherCertificateProvider provider(SERVER_KEY_PATH, SERVER_CERT_PATH,
 243                                           CA_CERT_PATH, 1);
 244   // Watcher watching both root and identity certs.
 245   WatcherState* watcher_state_1 =
 246       MakeWatcher(provider.distributor(), kCertName, kCertName);
 247   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 248               ::testing::ElementsAre(CredentialInfo(
 249                   root_cert_, MakeCertKeyPairs(private_key_.c_str(),
 250                                                cert_chain_.c_str()))));
 251   CancelWatch(watcher_state_1);
 252   // Watcher watching only root certs.
 253   WatcherState* watcher_state_2 =
 254       MakeWatcher(provider.distributor(), kCertName, absl::nullopt);
 255   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
 256               ::testing::ElementsAre(CredentialInfo(root_cert_, {})));
 257   CancelWatch(watcher_state_2);
 258   // Watcher watching only identity certs.
 259   WatcherState* watcher_state_3 =
 260       MakeWatcher(provider.distributor(), absl::nullopt, kCertName);
 261   EXPECT_THAT(
 262       watcher_state_3->GetCredentialQueue(),
 263       ::testing::ElementsAre(CredentialInfo(
 264           "", MakeCertKeyPairs(private_key_.c_str(), cert_chain_.c_str()))));
 265   CancelWatch(watcher_state_3);
 266 }
 267
 268 TEST_F(GrpcTlsCertificateProviderTest,
 269        FileWatcherCertificateProviderWithBadPaths) {
 270   FileWatcherCertificateProvider provider(INVALID_PATH, INVALID_PATH,
 271                                           INVALID_PATH, 1);
 272   // Watcher watching both root and identity certs.
 273   WatcherState* watcher_state_1 =
 274       MakeWatcher(provider.distributor(), kCertName, kCertName);
 275   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 276               ::testing::ElementsAre(ErrorInfo(kRootError, kIdentityError)));
 277   EXPECT_THAT(watcher_state_1->GetCredentialQueue(), ::testing::ElementsAre());
 278   CancelWatch(watcher_state_1);
 279   // Watcher watching only root certs.
 280   WatcherState* watcher_state_2 =
 281       MakeWatcher(provider.distributor(), kCertName, absl::nullopt);
 282   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
 283               ::testing::ElementsAre(ErrorInfo(kRootError, "")));
 284   EXPECT_THAT(watcher_state_2->GetCredentialQueue(), ::testing::ElementsAre());
 285   CancelWatch(watcher_state_2);
 286   // Watcher watching only identity certs.
 287   WatcherState* watcher_state_3 =
 288       MakeWatcher(provider.distributor(), absl::nullopt, kCertName);
 289   EXPECT_THAT(watcher_state_3->GetErrorQueue(),
 290               ::testing::ElementsAre(ErrorInfo("", kIdentityError)));
 291   EXPECT_THAT(watcher_state_3->GetCredentialQueue(), ::testing::ElementsAre());
 292   CancelWatch(watcher_state_3);
 293 }
 294
 295 // The following tests write credential data to temporary files to test the
 296 // transition behavior of the provider.
 297 TEST_F(GrpcTlsCertificateProviderTest,
 298        FileWatcherCertificateProviderOnBothCertsRefreshed) {
 299   // Create temporary files and copy cert data into them.
 300   TmpFile tmp_root_cert(root_cert_);
 301   TmpFile tmp_identity_key(private_key_);
 302   TmpFile tmp_identity_cert(cert_chain_);
 303   // Create FileWatcherCertificateProvider.
 304   FileWatcherCertificateProvider provider(tmp_identity_key.name(),
 305                                           tmp_identity_cert.name(),
 306                                           tmp_root_cert.name(), 1);
 307   WatcherState* watcher_state_1 =
 308       MakeWatcher(provider.distributor(), kCertName, kCertName);
 309   // Expect to see the credential data.
 310   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 311               ::testing::ElementsAre(CredentialInfo(
 312                   root_cert_, MakeCertKeyPairs(private_key_.c_str(),
 313                                                cert_chain_.c_str()))));
 314   // Copy new data to files.
 315   // TODO(ZhenLian): right now it is not completely atomic. Use the real atomic
 316   // update when the directory renaming is added in gpr.
 317   tmp_root_cert.RewriteFile(root_cert_2_);
 318   tmp_identity_key.RewriteFile(private_key_2_);
 319   tmp_identity_cert.RewriteFile(cert_chain_2_);
 320   // Wait 2 seconds for the provider's refresh thread to read the updated files.
 321   gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
 322                                gpr_time_from_seconds(2, GPR_TIMESPAN)));
 323   // Expect to see the new credential data.
 324   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 325               ::testing::ElementsAre(CredentialInfo(
 326                   root_cert_2_, MakeCertKeyPairs(private_key_2_.c_str(),
 327                                                  cert_chain_2_.c_str()))));
 328   // Clean up.
 329   CancelWatch(watcher_state_1);
 330 }
 331
 332 TEST_F(GrpcTlsCertificateProviderTest,
 333        FileWatcherCertificateProviderOnRootCertsRefreshed) {
 334   // Create temporary files and copy cert data into them.
 335   TmpFile tmp_root_cert(root_cert_);
 336   TmpFile tmp_identity_key(private_key_);
 337   TmpFile tmp_identity_cert(cert_chain_);
 338   // Create FileWatcherCertificateProvider.
 339   FileWatcherCertificateProvider provider(tmp_identity_key.name(),
 340                                           tmp_identity_cert.name(),
 341                                           tmp_root_cert.name(), 1);
 342   WatcherState* watcher_state_1 =
 343       MakeWatcher(provider.distributor(), kCertName, kCertName);
 344   // Expect to see the credential data.
 345   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 346               ::testing::ElementsAre(CredentialInfo(
 347                   root_cert_, MakeCertKeyPairs(private_key_.c_str(),
 348                                                cert_chain_.c_str()))));
 349   // Copy new data to files.
 350   // TODO(ZhenLian): right now it is not completely atomic. Use the real atomic
 351   // update when the directory renaming is added in gpr.
 352   tmp_root_cert.RewriteFile(root_cert_2_);
 353   // Wait 2 seconds for the provider's refresh thread to read the updated files.
 354   gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
 355                                gpr_time_from_seconds(2, GPR_TIMESPAN)));
 356   // Expect to see the new credential data.
 357   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 358               ::testing::ElementsAre(CredentialInfo(
 359                   root_cert_2_, MakeCertKeyPairs(private_key_.c_str(),
 360                                                  cert_chain_.c_str()))));
 361   // Clean up.
 362   CancelWatch(watcher_state_1);
 363 }
 364
 365 TEST_F(GrpcTlsCertificateProviderTest,
 366        FileWatcherCertificateProviderOnIdentityCertsRefreshed) {
 367   // Create temporary files and copy cert data into them.
 368   TmpFile tmp_root_cert(root_cert_);
 369   TmpFile tmp_identity_key(private_key_);
 370   TmpFile tmp_identity_cert(cert_chain_);
 371   // Create FileWatcherCertificateProvider.
 372   FileWatcherCertificateProvider provider(tmp_identity_key.name(),
 373                                           tmp_identity_cert.name(),
 374                                           tmp_root_cert.name(), 1);
 375   WatcherState* watcher_state_1 =
 376       MakeWatcher(provider.distributor(), kCertName, kCertName);
 377   // Expect to see the credential data.
 378   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 379               ::testing::ElementsAre(CredentialInfo(
 380                   root_cert_, MakeCertKeyPairs(private_key_.c_str(),
 381                                                cert_chain_.c_str()))));
 382   // Copy new data to files.
 383   // TODO(ZhenLian): right now it is not completely atomic. Use the real atomic
 384   // update when the directory renaming is added in gpr.
 385   tmp_identity_key.RewriteFile(private_key_2_);
 386   tmp_identity_cert.RewriteFile(cert_chain_2_);
 387   // Wait 2 seconds for the provider's refresh thread to read the updated files.
 388   gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
 389                                gpr_time_from_seconds(2, GPR_TIMESPAN)));
 390   // Expect to see the new credential data.
 391   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 392               ::testing::ElementsAre(CredentialInfo(
 393                   root_cert_, MakeCertKeyPairs(private_key_2_.c_str(),
 394                                                cert_chain_2_.c_str()))));
 395   // Clean up.
 396   CancelWatch(watcher_state_1);
 397 }
 398
 399 TEST_F(GrpcTlsCertificateProviderTest,
 400        FileWatcherCertificateProviderWithGoodAtFirstThenDeletedBothCerts) {
 401   // Create temporary files and copy cert data into it.
 402   auto tmp_root_cert = absl::make_unique<TmpFile>(root_cert_);
 403   auto tmp_identity_key = absl::make_unique<TmpFile>(private_key_);
 404   auto tmp_identity_cert = absl::make_unique<TmpFile>(cert_chain_);
 405   // Create FileWatcherCertificateProvider.
 406   FileWatcherCertificateProvider provider(tmp_identity_key->name(),
 407                                           tmp_identity_cert->name(),
 408                                           tmp_root_cert->name(), 1);
 409   WatcherState* watcher_state_1 =
 410       MakeWatcher(provider.distributor(), kCertName, kCertName);
 411   // The initial data is all good, so we expect to have successful credential
 412   // updates.
 413   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 414               ::testing::ElementsAre(CredentialInfo(
 415                   root_cert_, MakeCertKeyPairs(private_key_.c_str(),
 416                                                cert_chain_.c_str()))));
 417   // Delete TmpFile objects, which will remove the corresponding files.
 418   tmp_root_cert.reset();
 419   tmp_identity_key.reset();
 420   tmp_identity_cert.reset();
 421   // Wait 2 seconds for the provider's refresh thread to read the deleted files.
 422   gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
 423                                gpr_time_from_seconds(2, GPR_TIMESPAN)));
 424   // Expect to see errors sent to watchers, and no credential updates.
 425   // We have no ideas on how many errors we will receive, so we only check once.
 426   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 427               ::testing::Contains(ErrorInfo(kRootError, kIdentityError)));
 428   EXPECT_THAT(watcher_state_1->GetCredentialQueue(), ::testing::ElementsAre());
 429   // Clean up.
 430   CancelWatch(watcher_state_1);
 431 }
 432
 433 TEST_F(GrpcTlsCertificateProviderTest,
 434        FileWatcherCertificateProviderWithGoodAtFirstThenDeletedRootCerts) {
 435   // Create temporary files and copy cert data into it.
 436   auto tmp_root_cert = absl::make_unique<TmpFile>(root_cert_);
 437   TmpFile tmp_identity_key(private_key_);
 438   TmpFile tmp_identity_cert(cert_chain_);
 439   // Create FileWatcherCertificateProvider.
 440   FileWatcherCertificateProvider provider(tmp_identity_key.name(),
 441                                           tmp_identity_cert.name(),
 442                                           tmp_root_cert->name(), 1);
 443   WatcherState* watcher_state_1 =
 444       MakeWatcher(provider.distributor(), kCertName, kCertName);
 445   // The initial data is all good, so we expect to have successful credential
 446   // updates.
 447   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 448               ::testing::ElementsAre(CredentialInfo(
 449                   root_cert_, MakeCertKeyPairs(private_key_.c_str(),
 450                                                cert_chain_.c_str()))));
 451   // Delete root TmpFile object, which will remove the corresponding file.
 452   tmp_root_cert.reset();
 453   // Wait 2 seconds for the provider's refresh thread to read the deleted files.
 454   gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
 455                                gpr_time_from_seconds(2, GPR_TIMESPAN)));
 456   // Expect to see errors sent to watchers, and no credential updates.
 457   // We have no ideas on how many errors we will receive, so we only check once.
 458   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 459               ::testing::Contains(ErrorInfo(kRootError, "")));
 460   EXPECT_THAT(watcher_state_1->GetCredentialQueue(), ::testing::ElementsAre());
 461   // Clean up.
 462   CancelWatch(watcher_state_1);
 463 }
 464
 465 TEST_F(GrpcTlsCertificateProviderTest,
 466        FileWatcherCertificateProviderWithGoodAtFirstThenDeletedIdentityCerts) {
 467   // Create temporary files and copy cert data into it.
 468   TmpFile tmp_root_cert(root_cert_);
 469   auto tmp_identity_key = absl::make_unique<TmpFile>(private_key_);
 470   auto tmp_identity_cert = absl::make_unique<TmpFile>(cert_chain_);
 471   // Create FileWatcherCertificateProvider.
 472   FileWatcherCertificateProvider provider(tmp_identity_key->name(),
 473                                           tmp_identity_cert->name(),
 474                                           tmp_root_cert.name(), 1);
 475   WatcherState* watcher_state_1 =
 476       MakeWatcher(provider.distributor(), kCertName, kCertName);
 477   // The initial data is all good, so we expect to have successful credential
 478   // updates.
 479   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
 480               ::testing::ElementsAre(CredentialInfo(
 481                   root_cert_, MakeCertKeyPairs(private_key_.c_str(),
 482                                                cert_chain_.c_str()))));
 483   // Delete identity TmpFile objects, which will remove the corresponding files.
 484   tmp_identity_key.reset();
 485   tmp_identity_cert.reset();
 486   // Wait 2 seconds for the provider's refresh thread to read the deleted files.
 487   gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
 488                                gpr_time_from_seconds(2, GPR_TIMESPAN)));
 489   // Expect to see errors sent to watchers, and no credential updates.
 490   // We have no ideas on how many errors we will receive, so we only check once.
 491   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
 492               ::testing::Contains(ErrorInfo("", kIdentityError)));
 493   EXPECT_THAT(watcher_state_1->GetCredentialQueue(), ::testing::ElementsAre());
 494   // Clean up.
 495   CancelWatch(watcher_state_1);
 496 }
 497
 498 TEST_F(GrpcTlsCertificateProviderTest, FailedKeyCertMatchOnEmptyPrivateKey) {
 499   absl::StatusOr<bool> status =
 500       PrivateKeyAndCertificateMatch(/*private_key=*/"", cert_chain_);
 501   EXPECT_FALSE(status.ok());
 502   EXPECT_EQ(status.status().code(), absl::StatusCode::kInvalidArgument);
 503   EXPECT_EQ(status.status().message(), "Private key string is empty.");
 504 }
 505
 506 TEST_F(GrpcTlsCertificateProviderTest, FailedKeyCertMatchOnEmptyCertificate) {
 507   absl::StatusOr<bool> status =
 508       PrivateKeyAndCertificateMatch(private_key_2_, /*cert_chain=*/"");
 509   EXPECT_FALSE(status.ok());
 510   EXPECT_EQ(status.status().code(), absl::StatusCode::kInvalidArgument);
 511   EXPECT_EQ(status.status().message(), "Certificate string is empty.");
 512 }
 513
 514 TEST_F(GrpcTlsCertificateProviderTest, FailedKeyCertMatchOnInvalidCertFormat) {
 515   absl::StatusOr<bool> status =
 516       PrivateKeyAndCertificateMatch(private_key_2_, "invalid_certificate");
 517   EXPECT_FALSE(status.ok());
 518   EXPECT_EQ(status.status().code(), absl::StatusCode::kInvalidArgument);
 519   EXPECT_EQ(status.status().message(),
 520             "Conversion from PEM string to X509 failed.");
 521 }
 522
 523 TEST_F(GrpcTlsCertificateProviderTest,
 524        FailedKeyCertMatchOnInvalidPrivateKeyFormat) {
 525   absl::StatusOr<bool> status =
 526       PrivateKeyAndCertificateMatch("invalid_private_key", cert_chain_2_);
 527   EXPECT_EQ(status.status().code(), absl::StatusCode::kInvalidArgument);
 528   EXPECT_EQ(status.status().message(),
 529             "Conversion from PEM string to EVP_PKEY failed.");
 530 }
 531
 532 TEST_F(GrpcTlsCertificateProviderTest, SuccessfulKeyCertMatch) {
 533   absl::StatusOr<bool> status =
 534       PrivateKeyAndCertificateMatch(private_key_2_, cert_chain_2_);
 535   EXPECT_TRUE(status.ok());
 536   EXPECT_TRUE(*status);
 537 }
 538
 539 TEST_F(GrpcTlsCertificateProviderTest, FailedKeyCertMatchOnInvalidPair) {
 540   absl::StatusOr<bool> status =
 541       PrivateKeyAndCertificateMatch(private_key_2_, cert_chain_);
 542   EXPECT_TRUE(status.ok());
 543   EXPECT_FALSE(*status);
 544 }
 545
 546 }  // namespace testing
 547 }  // namespace grpc_core
 548
 549 int main(int argc, char** argv) {
 550   grpc::testing::TestEnvironment env(argc, argv);
 551   ::testing::InitGoogleTest(&argc, argv);
 552   grpc_init();
 553   int ret = RUN_ALL_TESTS();
 554   grpc_shutdown();
 555   return ret;
 556 }