1 // Copyright 2020 The Pigweed Authors
3 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
4 // use this file except in compliance with the License. You may obtain a copy of
7 // https://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 // License for the specific language governing permissions and limitations under
18 #include "pw_assert/light.h"
19 #include "pw_kvs/checksum.h"
20 #include "pw_kvs/flash_memory.h"
21 #include "pw_kvs/key_value_store.h"
22 #include "pw_status/status.h"
23 #include "pw_stream/stream.h"
25 namespace pw::blob_store {
27 // BlobStore is a storage container for a single blob of data. BlobStore is a
28 // FlashPartition-backed persistent storage system with integrated data
29 // integrity checking that serves as a lightweight alternative to a file
32 // Write and read are only done using the BlobWriter and BlobReader classes.
34 // Once a blob write is closed, reopening to write will discard the previous
38 // 0) Create BlobWriter instance
39 // 1) BlobWriter::Open().
40 // 2) Add data using BlobWriter::Write().
41 // 3) BlobWriter::Close().
44 // 0) Create BlobReader instance
45 // 1) BlobReader::Open().
46 // 2) Read data using BlobReader::Read() or
47 // BlobReader::GetMemoryMappedBlob().
48 // 3) BlobReader::Close().
51 // Implement the stream::Writer and erase interface for a BlobStore. If not
52 // already erased, the Write will do any needed erase.
54 // Only one writter (of either type) is allowed to be open at a time.
55 // Additionally, writters are unable to open if a reader is already open.
56 class BlobWriter : public stream::Writer {
58 constexpr BlobWriter(BlobStore& store) : store_(store), open_(false) {}
59 BlobWriter(const BlobWriter&) = delete;
60 BlobWriter& operator=(const BlobWriter&) = delete;
61 virtual ~BlobWriter() {
67 // Open a blob for writing/erasing. Open will invalidate any existing blob
68 // that may be stored. Can not open when already open. Only one writer is
69 // allowed to be open at a time. Returns:
72 // UNAVAILABLE - Unable to open, another writer or reader instance is
76 Status status = store_.OpenWrite();
83 // Finalize a blob write. Flush all remaining buffered data to storage and
84 // store blob metadata. Close fails in the closed state, do NOT retry Close
85 // on error. An error may or may not result in an invalid blob stored.
89 // DATA_LOSS - Error writing data or fail to verify written data.
93 return store_.CloseWrite();
96 bool IsOpen() { return open_; }
98 // Erase the blob partition and reset state for a new blob. Explicit calls
99 // to Erase are optional, beginning a write will do any needed Erase.
103 // UNAVAILABLE - Unable to erase while reader is open.
104 // [error status] - flash erase failed.
107 return store_.Erase();
110 // Discard the current blob. Any written bytes to this point are considered
114 // FAILED_PRECONDITION - not open.
117 return store_.Invalidate();
120 // Probable (not guaranteed) minimum number of bytes at this time that can
121 // be written. This is not necessarily the full number of bytes remaining in
122 // the blob. Returns zero if, in the current state, Write would return
123 // status other than OK. See stream.h for additional details.
124 size_t ConservativeWriteLimit() const override {
126 return store_.WriteBytesRemaining();
129 size_t CurrentSizeBytes() {
131 return store_.write_address_;
135 Status DoWrite(ConstByteSpan data) override {
137 return store_.Write(data);
144 // Implement the stream::Writer and erase interface with deferred action for a
145 // BlobStore. If not already erased, the Flush will do any needed erase.
147 // Only one writter (of either type) is allowed to be open at a time.
148 // Additionally, writters are unable to open if a reader is already open.
149 class DeferredWriter final : public BlobWriter {
151 constexpr DeferredWriter(BlobStore& store) : BlobWriter(store) {}
152 DeferredWriter(const DeferredWriter&) = delete;
153 DeferredWriter& operator=(const DeferredWriter&) = delete;
154 virtual ~DeferredWriter() {}
156 // Flush data in the write buffer. Only a multiple of flash_write_size_bytes
157 // are written in the flush. Any remainder is held until later for either
158 // a flush with flash_write_size_bytes buffered or the writer is closed.
161 return store_.Flush();
164 // Probable (not guaranteed) minimum number of bytes at this time that can
165 // be written. This is not necessarily the full number of bytes remaining in
166 // the blob. Returns zero if, in the current state, Write would return
167 // status other than OK. See stream.h for additional details.
168 size_t ConservativeWriteLimit() const override {
170 // Deferred writes need to fit in the write buffer.
171 return store_.WriteBufferBytesFree();
175 Status DoWrite(ConstByteSpan data) override {
177 return store_.AddToWriteBuffer(data);
181 // Implement stream::Reader interface for BlobStore. Multiple readers may be
182 // open at the same time, but readers may not be open with a writer open.
183 class BlobReader final : public stream::Reader {
185 constexpr BlobReader(BlobStore& store)
186 : store_(store), open_(false), offset_(0) {}
187 BlobReader(const BlobReader&) = delete;
188 BlobReader& operator=(const BlobReader&) = delete;
195 // Open to do a blob read at the given offset in to the blob. Can not open
196 // when already open. Multiple readers can be open at the same time.
200 // FAILED_PRECONDITION - No readable blob available.
201 // INVALID_ARGUMENT - Invalid offset.
202 // UNAVAILABLE - Unable to open, already open.
203 Status Open(size_t offset = 0) {
205 if (!store_.ValidToRead()) {
206 return Status::FailedPrecondition();
208 if (offset >= store_.ReadableDataBytes()) {
209 return Status::InvalidArgument();
213 Status status = store_.OpenRead();
220 // Finish reading a blob. Close fails in the closed state, do NOT retry
221 // Close on error. Returns:
227 return store_.CloseRead();
230 bool IsOpen() { return open_; }
232 // Probable (not guaranteed) minimum number of bytes at this time that can
233 // be read. Returns zero if, in the current state, Read would return status
234 // other than OK. See stream.h for additional details.
235 size_t ConservativeReadLimit() const override {
237 return store_.ReadableDataBytes() - offset_;
240 // Get a span with the MCU pointer and size of the data. Returns:
242 // OK with span - Valid span respresenting the blob data
243 // FAILED_PRECONDITION - Reader not open.
244 // UNIMPLEMENTED - Memory mapped access not supported for this blob.
245 Result<ConstByteSpan> GetMemoryMappedBlob() {
247 return store_.GetMemoryMappedBlob();
251 StatusWithSize DoRead(ByteSpan dest) override {
253 StatusWithSize status = store_.Read(offset_, dest);
255 offset_ += status.size();
266 // name - Name of blob store, used for metadata KVS key
267 // partition - Flash partiton to use for this blob. Blob uses the entire
268 // partition for blob data.
269 // checksum_algo - Optional checksum for blob integrity checking. Use nullptr
271 // kvs - KVS used for storing blob metadata.
272 // write_buffer - Used for buffering writes. Needs to be at least
273 // flash_write_size_bytes.
274 // flash_write_size_bytes - Size in bytes to use for flash write operations.
275 // This should be chosen to balance optimal write size and required buffer
276 // size. Must be greater than or equal to flash write alignment, less than
277 // or equal to flash sector size.
278 BlobStore(std::string_view name,
279 kvs::FlashPartition& partition,
280 kvs::ChecksumAlgorithm* checksum_algo,
281 kvs::KeyValueStore& kvs,
282 ByteSpan write_buffer,
283 size_t flash_write_size_bytes)
285 partition_(partition),
286 checksum_algo_(checksum_algo),
288 write_buffer_(write_buffer),
289 flash_write_size_bytes_(flash_write_size_bytes),
292 flash_erased_(false),
299 BlobStore(const BlobStore&) = delete;
300 BlobStore& operator=(const BlobStore&) = delete;
302 // Initialize the blob instance. Checks if storage is erased or has any stored
303 // blob data. Returns:
308 // Maximum number of data bytes this BlobStore is able to store.
309 size_t MaxDataSizeBytes() const;
312 typedef uint32_t ChecksumValue;
314 Status LoadMetadata();
316 // Open to do a blob write. Returns:
319 // UNAVAILABLE - Unable to open writer, another writer or reader instance is
323 // Open to do a blob read. Returns:
326 // FAILED_PRECONDITION - Unable to open, no valid blob available.
329 // Finalize a blob write. Flush all remaining buffered data to storage and
330 // store blob metadata. Returns:
332 // OK - success, valid complete blob.
333 // DATA_LOSS - Error during write (this close or previous write/flush). Blob
334 // is closed and marked as invalid.
338 // Write/append data to the in-progress blob write. Data is written
339 // sequentially, with each append added directly after the previous. Data is
340 // not guaranteed to be fully written out to storage on Write return. Returns:
342 // OK - successful write/enqueue of data.
343 // RESOURCE_EXHAUSTED - unable to write all of requested data at this time. No
345 // OUT_OF_RANGE - Writer has been exhausted, similar to EOF. No data written,
346 // no more will be written.
347 // DATA_LOSS - Error during write (this write or previous write/flush). No
348 // more will be written by following Write calls for current blob (until
349 // erase/new blob started).
350 Status Write(ConstByteSpan data);
352 // Similar to Write, but instead immediately writing out to flash, it only
353 // buffers the data. A flush or Close is reqired to get bytes writen out to
356 // OK - successful write/enqueue of data.
357 // RESOURCE_EXHAUSTED - unable to write all of requested data at this time. No
359 // OUT_OF_RANGE - Writer has been exhausted, similar to EOF. No data written,
360 // no more will be written.
361 // DATA_LOSS - Error during a previous write/flush. No more will be written by
362 // following Write calls for current blob (until erase/new blob started).
363 Status AddToWriteBuffer(ConstByteSpan data);
365 // Flush data in the write buffer. Only a multiple of flash_write_size_bytes
366 // are written in the flush. Any remainder is held until later for either a
367 // flush with flash_write_size_bytes buffered or the writer is closed.
369 // OK - successful write/enqueue of data.
370 // DATA_LOSS - Error during write (this flush or previous write/flush). No
371 // more will be written by following Write calls for current blob (until
372 // erase/new blob started).
375 // Flush a chunk of data in the write buffer smaller than
376 // flash_write_size_bytes. This is only for the final flush as part of the
377 // CloseWrite. The partial chunk is padded to flash_write_size_bytes and a
378 // flash_write_size_bytes chunk is written to flash.
380 // OK - successful write/enqueue of data.
381 // DATA_LOSS - Error during write (this flush or previous write/flush). No
382 // more will be written by following Write calls for current blob (until
383 // erase/new blob started).
384 Status FlushFinalPartialChunk();
386 // Commit data to flash and update flash_address_ with data bytes written. The
387 // only time data_bytes should be manually specified is for a CloseWrite with
388 // an unaligned-size chunk remaining in the buffer that has been zero padded
390 Status CommitToFlash(ConstByteSpan source, size_t data_bytes = 0);
392 // Blob is valid/OK to write to. Blob is considered valid to write if no data
393 // has been written due to the auto/implicit erase on write start.
395 // true - Blob is valid and OK to write to.
396 // false - Blob has previously had an error and not valid for writing new
398 bool ValidToWrite() { return (valid_data_ == true) || (write_address_ == 0); }
400 bool WriteBufferEmpty() const { return flash_address_ == write_address_; }
402 size_t WriteBufferBytesUsed() const;
404 size_t WriteBufferBytesFree() const;
406 Status EraseIfNeeded();
408 // Blob is valid/OK and has data to read.
409 bool ValidToRead() const { return (valid_data_ && ReadableDataBytes() > 0); }
411 // Read valid data. Attempts to read the lesser of output.size_bytes() or
412 // available bytes worth of data. Returns:
414 // OK with span of bytes read - success, between 1 and dest.size_bytes() were
416 // INVALID_ARGUMENT - offset is invalid.
417 // FAILED_PRECONDITION - Reader unable/not in state to read data.
418 // RESOURCE_EXHAUSTED - unable to read any bytes at this time. No bytes read.
419 // Try again once bytes become available.
420 // OUT_OF_RANGE - Reader has been exhausted, similar to EOF. No bytes read, no
421 // more will be read.
422 StatusWithSize Read(size_t offset, ByteSpan dest) const;
424 // Get a span with the MCU pointer and size of the data. Returns:
426 // OK with span - Valid span respresenting the blob data
427 // FAILED_PRECONDITION - Blob not in a state to read data
428 // UNIMPLEMENTED - Memory mapped access not supported for this blob.
429 Result<ConstByteSpan> GetMemoryMappedBlob() const;
431 // Size of blob/readable data, in bytes.
432 size_t ReadableDataBytes() const;
434 size_t WriteBytesRemaining() const {
435 return MaxDataSizeBytes() - write_address_;
442 void ResetChecksum() {
443 if (checksum_algo_ != nullptr) {
444 checksum_algo_->Reset();
448 Status ValidateChecksum();
450 Status CalculateChecksumFromFlash(size_t bytes_to_check);
452 const std::string_view MetadataKey() { return name_; }
454 // Changes to the metadata format should also get a different key signature to
455 // avoid new code improperly reading old format metadata.
456 struct BlobMetadata {
457 // The checksum of the blob data stored in flash.
458 ChecksumValue checksum;
460 // Number of blob data bytes stored in flash.
461 size_t data_size_bytes;
463 constexpr void reset() {
466 .data_size_bytes = 0,
471 std::string_view name_;
472 kvs::FlashPartition& partition_;
473 // checksum_algo_ of nullptr indicates no checksum algorithm.
474 kvs::ChecksumAlgorithm* const checksum_algo_;
475 kvs::KeyValueStore& kvs_;
476 ByteSpan write_buffer_;
478 // Size in bytes of flash write operations. This should be chosen to balance
479 // optimal write size and required buffer size. Must be GE flash write
480 // alignment, LE flash sector size.
481 const size_t flash_write_size_bytes_;
484 // Internal state for Blob store
486 // TODO: Consolidate blob state to a single struct
488 // Initialization has been done.
491 // Bytes stored are valid and good. Blob is OK to read and write to. Set as
492 // soon as blob is erased. Even when bytes written is still 0, they are valid.
495 // Blob partition is currently erased and ready to write a new blob.
498 // BlobWriter instance is currently open
501 // Count of open BlobReader instances
502 size_t readers_open_;
504 // Metadata for the blob.
505 BlobMetadata metadata_;
507 // Current index for end of overal blob data. Represents current byte size of
508 // blob data since the FlashPartition starts at address 0.
509 kvs::FlashPartition::Address write_address_;
511 // Current index of end of data written to flash. Number of buffered data
512 // bytes is write_address_ - flash_address_.
513 kvs::FlashPartition::Address flash_address_;
516 // Creates a BlobStore with the buffer of kBufferSizeBytes.
518 // kBufferSizeBytes - Size in bytes of write buffer to create.
519 // name - Name of blob store, used for metadata KVS key
520 // partition - Flash partiton to use for this blob. Blob uses the entire
521 // partition for blob data.
522 // checksum_algo - Optional checksum for blob integrity checking. Use nullptr
524 // kvs - KVS used for storing blob metadata.
525 // write_buffer - Used for buffering writes. Needs to be at least
526 // flash_write_size_bytes.
527 // flash_write_size_bytes - Size in bytes to use for flash write operations.
528 // This should be chosen to balance optimal write size and required buffer
529 // size. Must be greater than or equal to flash write alignment, less than
530 // or equal to flash sector size.
532 template <size_t kBufferSizeBytes>
533 class BlobStoreBuffer : public BlobStore {
535 explicit BlobStoreBuffer(std::string_view name,
536 kvs::FlashPartition& partition,
537 kvs::ChecksumAlgorithm* checksum_algo,
538 kvs::KeyValueStore& kvs,
539 size_t flash_write_size_bytes)
545 flash_write_size_bytes) {}
548 std::array<std::byte, kBufferSizeBytes> buffer_;
551 } // namespace pw::blob_store