third_party/pigweed/repo/pw_blob_store/blob_store_test.cc

   1 // Copyright 2020 The Pigweed Authors
   2 //
   3 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
   4 // use this file except in compliance with the License. You may obtain a copy of
   5 // the License at
   6 //
   7 //     https://www.apache.org/licenses/LICENSE-2.0
   8 //
   9 // Unless required by applicable law or agreed to in writing, software
  10 // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 // License for the specific language governing permissions and limitations under
  13 // the License.
  14
  15 #include "pw_blob_store/blob_store.h"
  16
  17 #include <array>
  18 #include <cstddef>
  19 #include <cstring>
  20 #include <span>
  21
  22 #include "gtest/gtest.h"
  23 #include "pw_kvs/crc16_checksum.h"
  24 #include "pw_kvs/fake_flash_memory.h"
  25 #include "pw_kvs/flash_memory.h"
  26 #include "pw_kvs/test_key_value_store.h"
  27 #include "pw_log/log.h"
  28 #include "pw_random/xor_shift.h"
  29
  30 namespace pw::blob_store {
  31 namespace {
  32
  33 class BlobStoreTest : public ::testing::Test {
  34  protected:
  35   BlobStoreTest() : flash_(kFlashAlignment), partition_(&flash_) {}
  36
  37   void InitFlashTo(std::span<const std::byte> contents) {
  38     partition_.Erase();
  39     std::memcpy(flash_.buffer().data(), contents.data(), contents.size());
  40   }
  41
  42   void InitSourceBufferToRandom(uint64_t seed,
  43                                 size_t init_size_bytes = kBlobDataSize) {
  44     ASSERT_LE(init_size_bytes, source_buffer_.size());
  45     random::XorShiftStarRng64 rng(seed);
  46
  47     std::memset(source_buffer_.data(),
  48                 static_cast<int>(flash_.erased_memory_content()),
  49                 source_buffer_.size());
  50     rng.Get(std::span(source_buffer_).first(init_size_bytes));
  51   }
  52
  53   void InitSourceBufferToFill(char fill,
  54                               size_t fill_size_bytes = kBlobDataSize) {
  55     ASSERT_LE(fill_size_bytes, source_buffer_.size());
  56     std::memset(source_buffer_.data(),
  57                 static_cast<int>(flash_.erased_memory_content()),
  58                 source_buffer_.size());
  59     std::memset(source_buffer_.data(), fill, fill_size_bytes);
  60   }
  61
  62   // Fill the source buffer with random pattern based on given seed, written to
  63   // BlobStore in specified chunk size.
  64   void WriteTestBlock(size_t write_size_bytes = kBlobDataSize) {
  65     ASSERT_LE(write_size_bytes, source_buffer_.size());
  66     constexpr size_t kBufferSize = 256;
  67     kvs::ChecksumCrc16 checksum;
  68
  69     ConstByteSpan write_data =
  70         std::span(source_buffer_).first(write_size_bytes);
  71
  72     char name[16] = {};
  73     snprintf(name, sizeof(name), "TestBlobBlock");
  74
  75     BlobStoreBuffer<kBufferSize> blob(
  76         name, partition_, &checksum, kvs::TestKvs(), kBufferSize);
  77     EXPECT_EQ(OkStatus(), blob.Init());
  78
  79     BlobStore::BlobWriter writer(blob);
  80     EXPECT_EQ(OkStatus(), writer.Open());
  81     ASSERT_EQ(OkStatus(), writer.Write(write_data));
  82     EXPECT_EQ(OkStatus(), writer.Close());
  83
  84     // Use reader to check for valid data.
  85     BlobStore::BlobReader reader(blob);
  86     ASSERT_EQ(OkStatus(), reader.Open());
  87     Result<ConstByteSpan> result = reader.GetMemoryMappedBlob();
  88     ASSERT_TRUE(result.ok());
  89     EXPECT_EQ(write_size_bytes, result.value().size_bytes());
  90     VerifyFlash(result.value().first(write_size_bytes));
  91     VerifyFlash(flash_.buffer().first(write_size_bytes));
  92     EXPECT_EQ(OkStatus(), reader.Close());
  93   }
  94
  95   // Open a new blob instance and read the blob using the given read chunk size.
  96   void ChunkReadTest(size_t read_chunk_size) {
  97     kvs::ChecksumCrc16 checksum;
  98
  99     VerifyFlash(flash_.buffer());
 100
 101     char name[16] = "TestBlobBlock";
 102     constexpr size_t kBufferSize = 16;
 103     BlobStoreBuffer<kBufferSize> blob(
 104         name, partition_, &checksum, kvs::TestKvs(), kBufferSize);
 105     EXPECT_EQ(OkStatus(), blob.Init());
 106
 107     // Use reader to check for valid data.
 108     BlobStore::BlobReader reader1(blob);
 109     ASSERT_EQ(OkStatus(), reader1.Open());
 110     Result<ConstByteSpan> possible_blob = reader1.GetMemoryMappedBlob();
 111     ASSERT_TRUE(possible_blob.ok());
 112     VerifyFlash(possible_blob.value());
 113     EXPECT_EQ(OkStatus(), reader1.Close());
 114
 115     BlobStore::BlobReader reader(blob);
 116     ASSERT_EQ(OkStatus(), reader.Open());
 117
 118     std::array<std::byte, kBlobDataSize> read_buffer;
 119
 120     ByteSpan read_span = read_buffer;
 121     while (read_span.size_bytes() > 0) {
 122       size_t read_size = std::min(read_span.size_bytes(), read_chunk_size);
 123
 124       PW_LOG_DEBUG("Do write of %u bytes, %u bytes remain",
 125                    static_cast<unsigned>(read_size),
 126                    static_cast<unsigned>(read_span.size_bytes()));
 127
 128       ASSERT_EQ(read_span.size_bytes(), reader.ConservativeReadLimit());
 129       auto result = reader.Read(read_span.first(read_size));
 130       ASSERT_EQ(result.status(), OkStatus());
 131       read_span = read_span.subspan(read_size);
 132     }
 133     EXPECT_EQ(OkStatus(), reader.Close());
 134
 135     VerifyFlash(read_buffer);
 136   }
 137
 138   void VerifyFlash(ConstByteSpan verify_bytes, size_t offset = 0) {
 139     // Should be defined as same size.
 140     EXPECT_EQ(source_buffer_.size(), flash_.buffer().size_bytes());
 141
 142     // Can't allow it to march off the end of source_buffer_.
 143     ASSERT_LE((verify_bytes.size_bytes() + offset), source_buffer_.size());
 144
 145     for (size_t i = 0; i < verify_bytes.size_bytes(); i++) {
 146       ASSERT_EQ(source_buffer_[i + offset], verify_bytes[i]);
 147     }
 148   }
 149
 150   static constexpr size_t kFlashAlignment = 16;
 151   static constexpr size_t kSectorSize = 2048;
 152   static constexpr size_t kSectorCount = 2;
 153   static constexpr size_t kBlobDataSize = (kSectorCount * kSectorSize);
 154
 155   kvs::FakeFlashMemoryBuffer<kSectorSize, kSectorCount> flash_;
 156   kvs::FlashPartition partition_;
 157   std::array<std::byte, kBlobDataSize> source_buffer_;
 158 };
 159
 160 TEST_F(BlobStoreTest, Init_Ok) {
 161   // TODO: Do init test with flash/kvs explicitly in the different possible
 162   // entry states.
 163   constexpr size_t kBufferSize = 256;
 164   BlobStoreBuffer<kBufferSize> blob(
 165       "Blob_OK", partition_, nullptr, kvs::TestKvs(), kBufferSize);
 166   EXPECT_EQ(OkStatus(), blob.Init());
 167 }
 168
 169 TEST_F(BlobStoreTest, IsOpen) {
 170   constexpr size_t kBufferSize = 256;
 171   BlobStoreBuffer<kBufferSize> blob(
 172       "Blob_open", partition_, nullptr, kvs::TestKvs(), kBufferSize);
 173   EXPECT_EQ(OkStatus(), blob.Init());
 174
 175   BlobStore::DeferredWriter deferred_writer(blob);
 176   EXPECT_EQ(false, deferred_writer.IsOpen());
 177   EXPECT_EQ(OkStatus(), deferred_writer.Open());
 178   EXPECT_EQ(true, deferred_writer.IsOpen());
 179   EXPECT_EQ(OkStatus(), deferred_writer.Close());
 180   EXPECT_EQ(false, deferred_writer.IsOpen());
 181
 182   BlobStore::BlobWriter writer(blob);
 183   EXPECT_EQ(false, writer.IsOpen());
 184   EXPECT_EQ(OkStatus(), writer.Open());
 185   EXPECT_EQ(true, writer.IsOpen());
 186
 187   // Need to write something, so the blob reader is able to open.
 188   std::array<std::byte, 64> tmp_buffer = {};
 189   EXPECT_EQ(OkStatus(), writer.Write(tmp_buffer));
 190   EXPECT_EQ(OkStatus(), writer.Close());
 191   EXPECT_EQ(false, writer.IsOpen());
 192
 193   BlobStore::BlobReader reader(blob);
 194   EXPECT_EQ(false, reader.IsOpen());
 195   ASSERT_EQ(OkStatus(), reader.Open());
 196   EXPECT_EQ(true, reader.IsOpen());
 197   EXPECT_EQ(OkStatus(), reader.Close());
 198   EXPECT_EQ(false, reader.IsOpen());
 199 }
 200
 201 TEST_F(BlobStoreTest, Discard) {
 202   InitSourceBufferToRandom(0x8675309);
 203   WriteTestBlock();
 204   constexpr char blob_title[] = "TestBlobBlock";
 205   std::array<std::byte, 64> tmp_buffer = {};
 206
 207   kvs::ChecksumCrc16 checksum;
 208
 209   // TODO: Do this test with flash/kvs in the different entry state
 210   // combinations.
 211
 212   constexpr size_t kBufferSize = 256;
 213   BlobStoreBuffer<kBufferSize> blob(
 214       blob_title, partition_, &checksum, kvs::TestKvs(), kBufferSize);
 215   EXPECT_EQ(OkStatus(), blob.Init());
 216
 217   BlobStore::BlobWriter writer(blob);
 218
 219   EXPECT_EQ(OkStatus(), writer.Open());
 220   EXPECT_EQ(OkStatus(), writer.Write(tmp_buffer));
 221
 222   // The write does an implicit erase so there should be no key for this blob.
 223   EXPECT_EQ(Status::NotFound(),
 224             kvs::TestKvs().Get(blob_title, tmp_buffer).status());
 225   EXPECT_EQ(OkStatus(), writer.Close());
 226
 227   EXPECT_EQ(OkStatus(), kvs::TestKvs().Get(blob_title, tmp_buffer).status());
 228
 229   EXPECT_EQ(OkStatus(), writer.Open());
 230   EXPECT_EQ(OkStatus(), writer.Discard());
 231   EXPECT_EQ(OkStatus(), writer.Close());
 232
 233   EXPECT_EQ(Status::NotFound(),
 234             kvs::TestKvs().Get(blob_title, tmp_buffer).status());
 235 }
 236
 237 TEST_F(BlobStoreTest, MultipleErase) {
 238   constexpr size_t kBufferSize = 256;
 239   BlobStoreBuffer<kBufferSize> blob(
 240       "Blob_OK", partition_, nullptr, kvs::TestKvs(), kBufferSize);
 241   EXPECT_EQ(OkStatus(), blob.Init());
 242
 243   BlobStore::BlobWriter writer(blob);
 244   EXPECT_EQ(OkStatus(), writer.Open());
 245
 246   EXPECT_EQ(OkStatus(), writer.Erase());
 247   EXPECT_EQ(OkStatus(), writer.Erase());
 248   EXPECT_EQ(OkStatus(), writer.Erase());
 249 }
 250
 251 TEST_F(BlobStoreTest, OffsetRead) {
 252   InitSourceBufferToRandom(0x11309);
 253   WriteTestBlock();
 254
 255   constexpr size_t kOffset = 10;
 256   ASSERT_LT(kOffset, kBlobDataSize);
 257
 258   kvs::ChecksumCrc16 checksum;
 259
 260   char name[16] = "TestBlobBlock";
 261   constexpr size_t kBufferSize = 16;
 262   BlobStoreBuffer<kBufferSize> blob(
 263       name, partition_, &checksum, kvs::TestKvs(), kBufferSize);
 264   EXPECT_EQ(OkStatus(), blob.Init());
 265   BlobStore::BlobReader reader(blob);
 266   ASSERT_EQ(OkStatus(), reader.Open(kOffset));
 267
 268   std::array<std::byte, kBlobDataSize - kOffset> read_buffer;
 269   ByteSpan read_span = read_buffer;
 270   ASSERT_EQ(read_span.size_bytes(), reader.ConservativeReadLimit());
 271
 272   auto result = reader.Read(read_span);
 273   ASSERT_EQ(result.status(), OkStatus());
 274   EXPECT_EQ(OkStatus(), reader.Close());
 275   VerifyFlash(read_buffer, kOffset);
 276 }
 277
 278 TEST_F(BlobStoreTest, InvalidReadOffset) {
 279   InitSourceBufferToRandom(0x11309);
 280   WriteTestBlock();
 281
 282   constexpr size_t kOffset = kBlobDataSize;
 283
 284   kvs::ChecksumCrc16 checksum;
 285
 286   char name[16] = "TestBlobBlock";
 287   constexpr size_t kBufferSize = 16;
 288   BlobStoreBuffer<kBufferSize> blob(
 289       name, partition_, &checksum, kvs::TestKvs(), kBufferSize);
 290   EXPECT_EQ(OkStatus(), blob.Init());
 291   BlobStore::BlobReader reader(blob);
 292   ASSERT_EQ(Status::InvalidArgument(), reader.Open(kOffset));
 293 }
 294
 295 // Test reading with a read buffer larger than the available data in the
 296 TEST_F(BlobStoreTest, ReadBufferIsLargerThanData) {
 297   InitSourceBufferToRandom(0x57326);
 298
 299   constexpr size_t kWriteBytes = 64;
 300   WriteTestBlock(kWriteBytes);
 301
 302   kvs::ChecksumCrc16 checksum;
 303
 304   char name[16] = "TestBlobBlock";
 305   constexpr size_t kBufferSize = 16;
 306   BlobStoreBuffer<kBufferSize> blob(
 307       name, partition_, &checksum, kvs::TestKvs(), kBufferSize);
 308   EXPECT_EQ(OkStatus(), blob.Init());
 309   BlobStore::BlobReader reader(blob);
 310   ASSERT_EQ(OkStatus(), reader.Open());
 311   EXPECT_EQ(kWriteBytes, reader.ConservativeReadLimit());
 312
 313   std::array<std::byte, kWriteBytes + 10> read_buffer;
 314   ByteSpan read_span = read_buffer;
 315
 316   auto result = reader.Read(read_span);
 317   ASSERT_EQ(result.status(), OkStatus());
 318   EXPECT_EQ(OkStatus(), reader.Close());
 319 }
 320
 321 TEST_F(BlobStoreTest, ChunkRead1) {
 322   InitSourceBufferToRandom(0x8675309);
 323   WriteTestBlock();
 324   ChunkReadTest(1);
 325 }
 326
 327 TEST_F(BlobStoreTest, ChunkRead3) {
 328   InitSourceBufferToFill(0);
 329   WriteTestBlock();
 330   ChunkReadTest(3);
 331 }
 332
 333 TEST_F(BlobStoreTest, ChunkRead4) {
 334   InitSourceBufferToFill(1);
 335   WriteTestBlock();
 336   ChunkReadTest(4);
 337 }
 338
 339 TEST_F(BlobStoreTest, ChunkRead5) {
 340   InitSourceBufferToFill(0xff);
 341   WriteTestBlock();
 342   ChunkReadTest(5);
 343 }
 344
 345 TEST_F(BlobStoreTest, ChunkRead16) {
 346   InitSourceBufferToRandom(0x86);
 347   WriteTestBlock();
 348   ChunkReadTest(16);
 349 }
 350
 351 TEST_F(BlobStoreTest, ChunkRead64) {
 352   InitSourceBufferToRandom(0x9);
 353   WriteTestBlock();
 354   ChunkReadTest(64);
 355 }
 356
 357 TEST_F(BlobStoreTest, ChunkReadFull) {
 358   InitSourceBufferToRandom(0x9);
 359   WriteTestBlock();
 360   ChunkReadTest(kBlobDataSize);
 361 }
 362
 363 TEST_F(BlobStoreTest, PartialBufferThenClose) {
 364   // Do write of only a partial chunk, which will only have bytes in buffer
 365   // (none written to flash) at close.
 366   size_t data_bytes = 12;
 367   InitSourceBufferToRandom(0x111, data_bytes);
 368   WriteTestBlock(data_bytes);
 369
 370   // Do write with several full chunks and then some partial.
 371   data_bytes = 158;
 372   InitSourceBufferToRandom(0x3222, data_bytes);
 373 }
 374
 375 // Test to do write/close, write/close multiple times.
 376 TEST_F(BlobStoreTest, MultipleWrites) {
 377   InitSourceBufferToRandom(0x1121);
 378   WriteTestBlock();
 379   InitSourceBufferToRandom(0x515);
 380   WriteTestBlock();
 381   InitSourceBufferToRandom(0x4321);
 382   WriteTestBlock();
 383 }
 384
 385 }  // namespace
 386 }  // namespace pw::blob_store