From 0ded5caa100a6802eb2f3e45e7b2073f0cef95eb Mon Sep 17 00:00:00 2001 From: Pawel Wieczorek Date: Wed, 27 Aug 2014 09:18:02 +0200 Subject: [PATCH] Implement mechanism assuring integrity of database There is also added mechanism for cleaning up Cynara's database directory upon loading policies to memory. There is added test checking whether mechanism behaves as intended. Change-Id: I926d1aebf394c092e00731b73717e0e1c55bad0c --- packaging/cynara.spec | 3 +- src/storage/CMakeLists.txt | 1 + src/storage/InMemoryStorageBackend.cpp | 46 +++- src/storage/InMemoryStorageBackend.h | 10 +- src/storage/Integrity.cpp | 232 +++++++++++++++++++++ src/storage/Integrity.h | 71 +++++++ test/CMakeLists.txt | 1 + test/db/db6/_ | 0 test/db/db6/_additional | 1 + test/db/db6/_additional~ | 0 test/db/db6/_~ | 0 test/db/db6/buckets | 2 + test/db/db6/buckets~ | 2 + test/db/db6/guard | 0 .../fakeinmemorystoragebackend.h | 2 +- .../inmemeorystoragebackendfixture.h | 2 + .../inmemorystoragebackend.cpp | 38 +++- 17 files changed, 396 insertions(+), 15 deletions(-) create mode 100644 src/storage/Integrity.cpp create mode 100644 src/storage/Integrity.h create mode 100644 test/db/db6/_ create mode 100644 test/db/db6/_additional create mode 100644 test/db/db6/_additional~ create mode 100644 test/db/db6/_~ create mode 100644 test/db/db6/buckets create mode 100644 test/db/db6/buckets~ create mode 100644 test/db/db6/guard diff --git a/packaging/cynara.spec b/packaging/cynara.spec index 213c94e..68d52ce 100644 --- a/packaging/cynara.spec +++ b/packaging/cynara.spec @@ -283,7 +283,7 @@ cp ./conf/creds.conf %{buildroot}/%{conf_path}/creds.conf mkdir -p %{buildroot}/usr/lib/systemd/system/sockets.target.wants mkdir -p %{buildroot}/%{state_path} -mkdir -p %{buildroot}/%{tests_dir} +mkdir -p %{buildroot}/%{tests_dir}/empty_db cp -a db* %{buildroot}/%{tests_dir} ln -s ../cynara.socket %{buildroot}/usr/lib/systemd/system/sockets.target.wants/cynara.socket ln -s ../cynara-admin.socket %{buildroot}/usr/lib/systemd/system/sockets.target.wants/cynara-admin.socket @@ -494,6 +494,7 @@ fi %manifest cynara-tests.manifest %attr(755,root,root) /usr/bin/cynara-tests %attr(755,root,root) %{tests_dir}/db*/* +%dir %attr(755,root,root) %{tests_dir}/empty_db %files -n libcynara-creds-commons %manifest libcynara-creds-commons.manifest diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 620dc2d..7bccc8c 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -25,6 +25,7 @@ SET(CYNARA_LIB_CYNARA_STORAGE_PATH ${CYNARA_PATH}/storage) SET(LIB_CYNARA_STORAGE_SOURCES ${CYNARA_LIB_CYNARA_STORAGE_PATH}/BucketDeserializer.cpp ${CYNARA_LIB_CYNARA_STORAGE_PATH}/InMemoryStorageBackend.cpp + ${CYNARA_LIB_CYNARA_STORAGE_PATH}/Integrity.cpp ${CYNARA_LIB_CYNARA_STORAGE_PATH}/Storage.cpp ${CYNARA_LIB_CYNARA_STORAGE_PATH}/StorageDeserializer.cpp ${CYNARA_LIB_CYNARA_STORAGE_PATH}/StorageSerializer.cpp diff --git a/src/storage/InMemoryStorageBackend.cpp b/src/storage/InMemoryStorageBackend.cpp index af18926..a2366e8 100644 --- a/src/storage/InMemoryStorageBackend.cpp +++ b/src/storage/InMemoryStorageBackend.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include #include @@ -43,6 +42,7 @@ #include #include +#include #include #include @@ -50,17 +50,28 @@ namespace Cynara { -const std::string InMemoryStorageBackend::m_indexFileName = "buckets"; +const std::string InMemoryStorageBackend::m_indexFilename = "buckets"; +const std::string InMemoryStorageBackend::m_backupFilenameSuffix = "~"; +const std::string InMemoryStorageBackend::m_bucketFilenamePrefix = "_"; void InMemoryStorageBackend::load(void) { - std::string indexFilename = m_dbPath + m_indexFileName; + Integrity integrity(m_dbPath, m_indexFilename, m_backupFilenameSuffix, m_bucketFilenamePrefix); + bool isBackupValid = integrity.backupGuardExists(); + std::string bucketSuffix = ""; + std::string indexFilename = m_dbPath + m_indexFilename; + + if (isBackupValid) { + bucketSuffix += m_backupFilenameSuffix; + indexFilename += m_backupFilenameSuffix; + } try { auto indexStream = std::make_shared(); openFileStream(indexStream, indexFilename); StorageDeserializer storageDeserializer(indexStream, - std::bind(&InMemoryStorageBackend::bucketStreamOpener, this, std::placeholders::_1)); + std::bind(&InMemoryStorageBackend::bucketStreamOpener, this, + std::placeholders::_1, bucketSuffix)); storageDeserializer.initBuckets(buckets()); storageDeserializer.loadBuckets(buckets()); @@ -74,6 +85,13 @@ void InMemoryStorageBackend::load(void) { LOGN("Creating defaultBucket."); this->buckets().insert({ defaultPolicyBucketId, PolicyBucket(defaultPolicyBucketId) }); } + + if (isBackupValid) { + integrity.revalidatePrimaryDatabase(buckets()); + } + //in case there were unnecessary files in db directory + integrity.deleteNonIndexedFiles(std::bind(&InMemoryStorageBackend::hasBucket, this, + std::placeholders::_1)); } void InMemoryStorageBackend::save(void) { @@ -90,11 +108,19 @@ void InMemoryStorageBackend::save(void) { } auto indexStream = std::make_shared(); - openDumpFileStream(indexStream, m_dbPath + m_indexFileName); + std::string indexFilename = m_dbPath + m_indexFilename; + openDumpFileStream(indexStream, indexFilename + m_backupFilenameSuffix); StorageSerializer storageSerializer(indexStream); storageSerializer.dump(buckets(), std::bind(&InMemoryStorageBackend::bucketDumpStreamOpener, this, std::placeholders::_1)); + + Integrity integrity(m_dbPath, m_indexFilename, m_backupFilenameSuffix, m_bucketFilenamePrefix); + + integrity.syncDatabase(buckets(), true); + integrity.createBackupGuard(); + integrity.revalidatePrimaryDatabase(buckets()); + //guard is removed during revalidation } PolicyBucket InMemoryStorageBackend::searchDefaultBucket(const PolicyKey &key) { @@ -181,8 +207,9 @@ void InMemoryStorageBackend::openFileStream(std::shared_ptr strea // stream.exceptions(std::ifstream::failbit | std::ifstream::badbit); stream->open(filename); - if (!stream->is_open()) + if (!stream->is_open()) { throw FileNotFoundException(filename); + } } void InMemoryStorageBackend::openDumpFileStream(std::shared_ptr stream, @@ -195,8 +222,8 @@ void InMemoryStorageBackend::openDumpFileStream(std::shared_ptr s } std::shared_ptr InMemoryStorageBackend::bucketStreamOpener( - const PolicyBucketId &bucketId) { - std::string bucketFilename = m_dbPath + "_" + bucketId; + const PolicyBucketId &bucketId, const std::string &filenameSuffix) { + std::string bucketFilename = m_dbPath + m_bucketFilenamePrefix + bucketId + filenameSuffix; auto bucketStream = std::make_shared(); try { openFileStream(bucketStream, bucketFilename); @@ -208,7 +235,8 @@ std::shared_ptr InMemoryStorageBackend::bucketStreamOpener( std::shared_ptr InMemoryStorageBackend::bucketDumpStreamOpener( const PolicyBucketId &bucketId) { - std::string bucketFilename = m_dbPath + "_" + bucketId; + std::string bucketFilename = m_dbPath + m_bucketFilenamePrefix + + bucketId + m_backupFilenameSuffix; auto bucketStream = std::make_shared(); openDumpFileStream(bucketStream, bucketFilename); diff --git a/src/storage/InMemoryStorageBackend.h b/src/storage/InMemoryStorageBackend.h index cc35f95..d03dd6c 100644 --- a/src/storage/InMemoryStorageBackend.h +++ b/src/storage/InMemoryStorageBackend.h @@ -62,15 +62,19 @@ public: protected: InMemoryStorageBackend() {} void openFileStream(std::shared_ptr stream, const std::string &filename); - std::shared_ptr bucketStreamOpener(const PolicyBucketId &bucketId); + std::shared_ptr bucketStreamOpener(const PolicyBucketId &bucketId, + const std::string &fileNameSuffix); - void openDumpFileStream(std::shared_ptr stream, const std::string &filename); + virtual void openDumpFileStream(std::shared_ptr stream, + const std::string &filename); std::shared_ptr bucketDumpStreamOpener(const PolicyBucketId &bucketId); private: std::string m_dbPath; Buckets m_buckets; - static const std::string m_indexFileName; + static const std::string m_indexFilename; + static const std::string m_backupFilenameSuffix; + static const std::string m_bucketFilenamePrefix; protected: virtual Buckets &buckets(void) { diff --git a/src/storage/Integrity.cpp b/src/storage/Integrity.cpp new file mode 100644 index 0000000..8057e9c --- /dev/null +++ b/src/storage/Integrity.cpp @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2014 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @file src/storage/Integrity.cpp + * @author Pawel Wieczorek + * @version 0.1 + * @brief Implementation of Cynara::Integrity + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "Integrity.h" + +namespace Cynara { + +const std::string Integrity::m_guardFilename = "guard"; + +bool Integrity::backupGuardExists(void) const { + struct stat buffer; + std::string guardFilename = m_dbPath + m_guardFilename; + + int ret = stat(guardFilename.c_str(), &buffer); + + if (ret == 0) { + return true; + } else { + int err = errno; + if (err != ENOENT) { + LOGE("'stat' function error [%d] : <%s>", err, strerror(err)); + throw UnexpectedErrorException(err, strerror(err)); + } + return false; + } +} + +void Integrity::createBackupGuard(void) const { + syncElement(m_dbPath + m_guardFilename, O_CREAT | O_EXCL | O_WRONLY | O_TRUNC); + syncDirectory(m_dbPath); +} + +void Integrity::syncDatabase(const Buckets &buckets, bool syncBackup) { + std::string suffix = ""; + + if (syncBackup) { + suffix += m_backupFilenameSuffix; + } + + for (const auto &bucketIter : buckets) { + const auto &bucketId = bucketIter.first; + const auto &bucketFilename = m_dbPath + m_bucketFilenamePrefix + bucketId + suffix; + + syncElement(bucketFilename); + } + + syncElement(m_dbPath + m_indexFilename + suffix); + syncDirectory(m_dbPath); +} + +void Integrity::revalidatePrimaryDatabase(const Buckets &buckets) { + createPrimaryHardLinks(buckets); + syncDatabase(buckets, false); + + deleteHardLink(m_dbPath + m_guardFilename); + syncDirectory(m_dbPath); + + deleteBackupHardLinks(buckets); +} + +void Integrity::deleteNonIndexedFiles(BucketPresenceTester tester) { + DIR *dirPtr = nullptr; + struct dirent *direntPtr; + + if ((dirPtr = opendir(m_dbPath.c_str())) == nullptr) { + int err = errno; + LOGE("'opendir' function error [%d] : <%s>", err, strerror(err)); + throw UnexpectedErrorException(err, strerror(err)); + return; + } + + while (errno = 0, (direntPtr = readdir(dirPtr)) != nullptr) { + std::string filename = direntPtr->d_name; + //ignore all special files (working dir, parent dir, index) + if ("." == filename || ".." == filename || "buckets" == filename) { + continue; + } + + std::string bucketId; + auto nameLength = filename.length(); + auto prefixLength = m_bucketFilenamePrefix.length(); + + //remove if it is impossible that it is a bucket file + if (nameLength < prefixLength) { + deleteHardLink(m_dbPath + filename); + continue; + } + + //remove if there is no bucket filename prefix + //0 is returned from string::compare() if strings are equal + if (0 != filename.compare(0, prefixLength, m_bucketFilenamePrefix)) { + deleteHardLink(m_dbPath + filename); + continue; + } + + //remove if bucket is not in index + bucketId = filename.substr(prefixLength); + if (!tester(bucketId)) { + deleteHardLink(m_dbPath + filename); + } + } + + if (errno) { + int err = errno; + LOGE("'readdir' function error [%d] : <%s>", err, strerror(err)); + throw UnexpectedErrorException(err, strerror(err)); + return; + } +} + +void Integrity::syncElement(const std::string &filename, int flags, mode_t mode) { + int fileFd = TEMP_FAILURE_RETRY(open(filename.c_str(), flags, mode)); + + if (fileFd < 0) { + int err = errno; + if (err != EEXIST) { + LOGE("'open' function error [%d] : <%s>", err, strerror(err)); + throw UnexpectedErrorException(err, strerror(err)); + } else { + throw CannotCreateFileException(filename); + } + } + + int ret = fsync(fileFd); + + if (ret < 0) { + int err = errno; + LOGE("'fsync' function error [%d] : <%s>", err, strerror(err)); + throw UnexpectedErrorException(err, strerror(err)); + } + + ret = close(fileFd); + + if (ret < 0) { + int err = errno; + LOGE("'close' function error [%d] : <%s>", err, strerror(err)); + throw UnexpectedErrorException(err, strerror(err)); + } +} + +// from: man 2 fsync +// Calling fsync() does not necessarily ensure that the entry in the directory containing +// the file has also reached disk. For that an explicit fsync() on a file descriptor for +// the directory is also needed. +void Integrity::syncDirectory(const std::string &dirname, mode_t mode) { + syncElement(dirname, O_DIRECTORY, mode); +} + +void Integrity::createPrimaryHardLinks(const Buckets &buckets) { + for (const auto &bucketIter : buckets) { + const auto &bucketId = bucketIter.first; + const auto &bucketFilename = m_dbPath + m_bucketFilenamePrefix + bucketId; + + deleteHardLink(bucketFilename); + createHardLink(bucketFilename + m_backupFilenameSuffix, bucketFilename); + } + + const auto &indexFilename = m_dbPath + m_indexFilename; + + deleteHardLink(indexFilename); + createHardLink(indexFilename + m_backupFilenameSuffix, indexFilename); +} + +void Integrity::deleteBackupHardLinks(const Buckets &buckets) { + for (const auto &bucketIter : buckets) { + const auto &bucketId = bucketIter.first; + const auto &bucketFilename = m_dbPath + m_bucketFilenamePrefix + + bucketId + m_backupFilenameSuffix; + + deleteHardLink(bucketFilename); + } + + deleteHardLink(m_dbPath + m_indexFilename + m_backupFilenameSuffix); +} + +void Integrity::createHardLink(const std::string &oldName, const std::string &newName) { + int ret = link(oldName.c_str(), newName.c_str()); + + if (ret < 0) { + int err = errno; + throw UnexpectedErrorException(err, strerror(err)); + LOGN("Trying to link to non-existent file: <%s>", oldName.c_str()); + } +} + +void Integrity::deleteHardLink(const std::string &filename) { + int ret = unlink(filename.c_str()); + + if (ret < 0) { + int err = errno; + if (err != ENOENT) { + LOGE("'unlink' function error [%d] : <%s>", err, strerror(err)); + throw UnexpectedErrorException(err, strerror(err)); + } else { + LOGN("Trying to unlink non-existent file: <%s>", filename.c_str()); + } + } +} + +} /* namespace Cynara */ diff --git a/src/storage/Integrity.h b/src/storage/Integrity.h new file mode 100644 index 0000000..4b677b0 --- /dev/null +++ b/src/storage/Integrity.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2014 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @file src/storage/Integrity.h + * @author Pawel Wieczorek + * @version 0.1 + * @brief Headers for Cynara::Integrity + */ + +#ifndef SRC_STORAGE_INTEGRITY_H_ +#define SRC_STORAGE_INTEGRITY_H_ + +#include +#include + +#include + +namespace Cynara { + +class Integrity +{ +public: + typedef std::function BucketPresenceTester; + Integrity(const std::string &path, const std::string &index, const std::string &backupSuffix, + const std::string &bucketPrefix) + : m_dbPath(path), m_indexFilename(index), m_backupFilenameSuffix(backupSuffix), + m_bucketFilenamePrefix(bucketPrefix) { + } + virtual ~Integrity() {}; + + virtual bool backupGuardExists(void) const; + virtual void createBackupGuard(void) const; + virtual void syncDatabase(const Buckets &buckets, bool syncBackup); + virtual void revalidatePrimaryDatabase(const Buckets &buckets); + virtual void deleteNonIndexedFiles(BucketPresenceTester tester); + +protected: + static void syncElement(const std::string &filename, int flags = O_RDONLY, + mode_t mode = S_IRUSR | S_IWUSR); + static void syncDirectory(const std::string &dirname, mode_t mode = S_IRUSR | S_IWUSR); + + void createPrimaryHardLinks(const Buckets &buckets); + void deleteBackupHardLinks(const Buckets &buckets); + + static void createHardLink(const std::string &oldName, const std::string &newName); + static void deleteHardLink(const std::string &filename); + +private: + const std::string m_dbPath; + const std::string m_indexFilename; + const std::string m_backupFilenameSuffix; + const std::string m_bucketFilenamePrefix; + static const std::string m_guardFilename; +}; + +} // namespace Cynara + +#endif /* SRC_STORAGE_INTEGRITY_H_ */ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ffddd87..063098f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -32,6 +32,7 @@ SET(CYNARA_SOURCES_FOR_TESTS ${CYNARA_SRC}/helpers/creds-commons/creds-commons.cpp ${CYNARA_SRC}/storage/BucketDeserializer.cpp ${CYNARA_SRC}/storage/InMemoryStorageBackend.cpp + ${CYNARA_SRC}/storage/Integrity.cpp ${CYNARA_SRC}/storage/Storage.cpp ${CYNARA_SRC}/storage/StorageDeserializer.cpp ${CYNARA_SRC}/storage/StorageSerializer.cpp diff --git a/test/db/db6/_ b/test/db/db6/_ new file mode 100644 index 0000000..e69de29 diff --git a/test/db/db6/_additional b/test/db/db6/_additional new file mode 100644 index 0000000..cbeab67 --- /dev/null +++ b/test/db/db6/_additional @@ -0,0 +1 @@ +client1;user1;privilege1;0x0; diff --git a/test/db/db6/_additional~ b/test/db/db6/_additional~ new file mode 100644 index 0000000..e69de29 diff --git a/test/db/db6/_~ b/test/db/db6/_~ new file mode 100644 index 0000000..e69de29 diff --git a/test/db/db6/buckets b/test/db/db6/buckets new file mode 100644 index 0000000..19d7a93 --- /dev/null +++ b/test/db/db6/buckets @@ -0,0 +1,2 @@ +;0x0 +additional;0xFFFF diff --git a/test/db/db6/buckets~ b/test/db/db6/buckets~ new file mode 100644 index 0000000..19d7a93 --- /dev/null +++ b/test/db/db6/buckets~ @@ -0,0 +1,2 @@ +;0x0 +additional;0xFFFF diff --git a/test/db/db6/guard b/test/db/db6/guard new file mode 100644 index 0000000..e69de29 diff --git a/test/storage/inmemorystoragebackend/fakeinmemorystoragebackend.h b/test/storage/inmemorystoragebackend/fakeinmemorystoragebackend.h index 1c80dea..52b4fe3 100644 --- a/test/storage/inmemorystoragebackend/fakeinmemorystoragebackend.h +++ b/test/storage/inmemorystoragebackend/fakeinmemorystoragebackend.h @@ -29,7 +29,7 @@ class FakeInMemoryStorageBackend : public Cynara::InMemoryStorageBackend { public: using Cynara::InMemoryStorageBackend::InMemoryStorageBackend; - MOCK_METHOD0(buckets, Cynara::Buckets&()); + MOCK_METHOD0(buckets, Cynara::Buckets&(void)); }; diff --git a/test/storage/inmemorystoragebackend/inmemeorystoragebackendfixture.h b/test/storage/inmemorystoragebackend/inmemeorystoragebackendfixture.h index 3f08dea..2ec2920 100644 --- a/test/storage/inmemorystoragebackend/inmemeorystoragebackendfixture.h +++ b/test/storage/inmemorystoragebackend/inmemeorystoragebackendfixture.h @@ -67,6 +67,8 @@ protected: // TODO: consider defaulting accessor with ON_CALL Cynara::Buckets m_buckets; + static const std::string m_indexFileName; + static const std::string m_backupFileNameSuffix; }; diff --git a/test/storage/inmemorystoragebackend/inmemorystoragebackend.cpp b/test/storage/inmemorystoragebackend/inmemorystoragebackend.cpp index 11c0968..c0fd640 100644 --- a/test/storage/inmemorystoragebackend/inmemorystoragebackend.cpp +++ b/test/storage/inmemorystoragebackend/inmemorystoragebackend.cpp @@ -40,6 +40,9 @@ using namespace Cynara; +const std::string InMemeoryStorageBackendFixture::m_indexFileName = "buckets"; +const std::string InMemeoryStorageBackendFixture::m_backupFileNameSuffix = "~"; + TEST_F(InMemeoryStorageBackendFixture, defaultPolicyIsDeny) { using ::testing::ReturnRef; @@ -192,7 +195,7 @@ TEST_F(InMemeoryStorageBackendFixture, deletePolicyFromNonexistentBucket) { // Database dir is empty TEST_F(InMemeoryStorageBackendFixture, load_no_db) { using ::testing::ReturnRef; - auto testDbPath = std::string(CYNARA_TESTS_DIR) + "/db1/"; + auto testDbPath = std::string(CYNARA_TESTS_DIR) + "/empty_db/"; FakeInMemoryStorageBackend backend(testDbPath); EXPECT_CALL(backend, buckets()).WillRepeatedly(ReturnRef(m_buckets)); backend.load(); @@ -252,3 +255,36 @@ TEST_F(InMemeoryStorageBackendFixture, second_bucket_corrupted) { backend.load(); ASSERT_DB_VIRGIN(m_buckets); } + +/** + * @brief Database was corrupted, restore from backup (which is present) + * @test Scenario: + * - There still is guard file in earlier prepared Cynara's policy database directory (db6) + * - Execution of load() should use backup files (present, with different contents than primaries) + * - Loaded database is checked - backup files were empty, so should recently loaded policies + */ +TEST_F(InMemeoryStorageBackendFixture, load_from_backup) { + using ::testing::ReturnRef; + using ::testing::IsEmpty; + using ::testing::InSequence; + + auto testDbPath = std::string(CYNARA_TESTS_DIR) + "/db6/"; + FakeInMemoryStorageBackend backend(testDbPath); + + { + // Calls are expected in a specific order + InSequence s; + EXPECT_CALL(backend, buckets()).WillRepeatedly(ReturnRef(m_buckets)); + backend.load(); + } + + std::vector bucketIds = { "", "additional" }; + for(const auto &bucketId : bucketIds) { + SCOPED_TRACE(bucketId); + const auto bucketIter = m_buckets.find(bucketId); + ASSERT_NE(m_buckets.end(), bucketIter); + + const auto &bucketPolicies = bucketIter->second; + ASSERT_THAT(bucketPolicies, IsEmpty()); + } +} -- 2.7.4