1 // Copyright 2020 The Pigweed Authors
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
4 // use this file except in compliance with the License. You may obtain a copy of
5 // the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 // License for the specific language governing permissions and limitations under
13 // the License.
14
15 #include "pw_blob_store/blob_store.h"
16
17 #include <array>
18 #include <cstddef>
19 #include <cstring>
20 #include <span>
21
22 #include "gtest/gtest.h"
23 #include "pw_kvs/crc16_checksum.h"
24 #include "pw_kvs/fake_flash_memory.h"
25 #include "pw_kvs/flash_memory.h"
26 #include "pw_kvs/test_key_value_store.h"
27 #include "pw_log/log.h"
28 #include "pw_random/xor_shift.h"
29
30 namespace pw::blob_store {
31 namespace {
32
33 class BlobStoreTest : public ::testing::Test {
34 protected:
BlobStoreTest()35 BlobStoreTest() : flash_(kFlashAlignment), partition_(&flash_) {}
36
InitFlashTo(std::span<const std::byte> contents)37 void InitFlashTo(std::span<const std::byte> contents) {
38 partition_.Erase();
39 std::memcpy(flash_.buffer().data(), contents.data(), contents.size());
40 }
41
InitSourceBufferToRandom(uint64_t seed,size_t init_size_bytes=kBlobDataSize)42 void InitSourceBufferToRandom(uint64_t seed,
43 size_t init_size_bytes = kBlobDataSize) {
44 ASSERT_LE(init_size_bytes, source_buffer_.size());
45 random::XorShiftStarRng64 rng(seed);
46
47 std::memset(source_buffer_.data(),
48 static_cast<int>(flash_.erased_memory_content()),
49 source_buffer_.size());
50 rng.Get(std::span(source_buffer_).first(init_size_bytes));
51 }
52
InitSourceBufferToFill(char fill,size_t fill_size_bytes=kBlobDataSize)53 void InitSourceBufferToFill(char fill,
54 size_t fill_size_bytes = kBlobDataSize) {
55 ASSERT_LE(fill_size_bytes, source_buffer_.size());
56 std::memset(source_buffer_.data(),
57 static_cast<int>(flash_.erased_memory_content()),
58 source_buffer_.size());
59 std::memset(source_buffer_.data(), fill, fill_size_bytes);
60 }
61
62 // Fill the source buffer with random pattern based on given seed, written to
63 // BlobStore in specified chunk size.
WriteTestBlock(size_t write_size_bytes=kBlobDataSize)64 void WriteTestBlock(size_t write_size_bytes = kBlobDataSize) {
65 ASSERT_LE(write_size_bytes, source_buffer_.size());
66 constexpr size_t kBufferSize = 256;
67 kvs::ChecksumCrc16 checksum;
68
69 ConstByteSpan write_data =
70 std::span(source_buffer_).first(write_size_bytes);
71
72 char name[16] = {};
73 snprintf(name, sizeof(name), "TestBlobBlock");
74
75 BlobStoreBuffer<kBufferSize> blob(
76 name, partition_, &checksum, kvs::TestKvs(), kBufferSize);
77 EXPECT_EQ(OkStatus(), blob.Init());
78
79 BlobStore::BlobWriter writer(blob);
80 EXPECT_EQ(OkStatus(), writer.Open());
81 ASSERT_EQ(OkStatus(), writer.Write(write_data));
82 EXPECT_EQ(OkStatus(), writer.Close());
83
84 // Use reader to check for valid data.
85 BlobStore::BlobReader reader(blob);
86 ASSERT_EQ(OkStatus(), reader.Open());
87 Result<ConstByteSpan> result = reader.GetMemoryMappedBlob();
88 ASSERT_TRUE(result.ok());
89 EXPECT_EQ(write_size_bytes, result.value().size_bytes());
90 VerifyFlash(result.value().first(write_size_bytes));
91 VerifyFlash(flash_.buffer().first(write_size_bytes));
92 EXPECT_EQ(OkStatus(), reader.Close());
93 }
94
95 // Open a new blob instance and read the blob using the given read chunk size.
ChunkReadTest(size_t read_chunk_size)96 void ChunkReadTest(size_t read_chunk_size) {
97 kvs::ChecksumCrc16 checksum;
98
99 VerifyFlash(flash_.buffer());
100
101 char name[16] = "TestBlobBlock";
102 constexpr size_t kBufferSize = 16;
103 BlobStoreBuffer<kBufferSize> blob(
104 name, partition_, &checksum, kvs::TestKvs(), kBufferSize);
105 EXPECT_EQ(OkStatus(), blob.Init());
106
107 // Use reader to check for valid data.
108 BlobStore::BlobReader reader1(blob);
109 ASSERT_EQ(OkStatus(), reader1.Open());
110 Result<ConstByteSpan> possible_blob = reader1.GetMemoryMappedBlob();
111 ASSERT_TRUE(possible_blob.ok());
112 VerifyFlash(possible_blob.value());
113 EXPECT_EQ(OkStatus(), reader1.Close());
114
115 BlobStore::BlobReader reader(blob);
116 ASSERT_EQ(OkStatus(), reader.Open());
117
118 std::array<std::byte, kBlobDataSize> read_buffer;
119
120 ByteSpan read_span = read_buffer;
121 while (read_span.size_bytes() > 0) {
122 size_t read_size = std::min(read_span.size_bytes(), read_chunk_size);
123
124 PW_LOG_DEBUG("Do write of %u bytes, %u bytes remain",
125 static_cast<unsigned>(read_size),
126 static_cast<unsigned>(read_span.size_bytes()));
127
128 ASSERT_EQ(read_span.size_bytes(), reader.ConservativeReadLimit());
129 auto result = reader.Read(read_span.first(read_size));
130 ASSERT_EQ(result.status(), OkStatus());
131 read_span = read_span.subspan(read_size);
132 }
133 EXPECT_EQ(OkStatus(), reader.Close());
134
135 VerifyFlash(read_buffer);
136 }
137
VerifyFlash(ConstByteSpan verify_bytes,size_t offset=0)138 void VerifyFlash(ConstByteSpan verify_bytes, size_t offset = 0) {
139 // Should be defined as same size.
140 EXPECT_EQ(source_buffer_.size(), flash_.buffer().size_bytes());
141
142 // Can't allow it to march off the end of source_buffer_.
143 ASSERT_LE((verify_bytes.size_bytes() + offset), source_buffer_.size());
144
145 for (size_t i = 0; i < verify_bytes.size_bytes(); i++) {
146 ASSERT_EQ(source_buffer_[i + offset], verify_bytes[i]);
147 }
148 }
149
150 static constexpr size_t kFlashAlignment = 16;
151 static constexpr size_t kSectorSize = 2048;
152 static constexpr size_t kSectorCount = 2;
153 static constexpr size_t kBlobDataSize = (kSectorCount * kSectorSize);
154
155 kvs::FakeFlashMemoryBuffer<kSectorSize, kSectorCount> flash_;
156 kvs::FlashPartition partition_;
157 std::array<std::byte, kBlobDataSize> source_buffer_;
158 };
159
TEST_F(BlobStoreTest,Init_Ok)160 TEST_F(BlobStoreTest, Init_Ok) {
161 // TODO: Do init test with flash/kvs explicitly in the different possible
162 // entry states.
163 constexpr size_t kBufferSize = 256;
164 BlobStoreBuffer<kBufferSize> blob(
165 "Blob_OK", partition_, nullptr, kvs::TestKvs(), kBufferSize);
166 EXPECT_EQ(OkStatus(), blob.Init());
167 }
168
TEST_F(BlobStoreTest,IsOpen)169 TEST_F(BlobStoreTest, IsOpen) {
170 constexpr size_t kBufferSize = 256;
171 BlobStoreBuffer<kBufferSize> blob(
172 "Blob_open", partition_, nullptr, kvs::TestKvs(), kBufferSize);
173 EXPECT_EQ(OkStatus(), blob.Init());
174
175 BlobStore::DeferredWriter deferred_writer(blob);
176 EXPECT_EQ(false, deferred_writer.IsOpen());
177 EXPECT_EQ(OkStatus(), deferred_writer.Open());
178 EXPECT_EQ(true, deferred_writer.IsOpen());
179 EXPECT_EQ(OkStatus(), deferred_writer.Close());
180 EXPECT_EQ(false, deferred_writer.IsOpen());
181
182 BlobStore::BlobWriter writer(blob);
183 EXPECT_EQ(false, writer.IsOpen());
184 EXPECT_EQ(OkStatus(), writer.Open());
185 EXPECT_EQ(true, writer.IsOpen());
186
187 // Need to write something, so the blob reader is able to open.
188 std::array<std::byte, 64> tmp_buffer = {};
189 EXPECT_EQ(OkStatus(), writer.Write(tmp_buffer));
190 EXPECT_EQ(OkStatus(), writer.Close());
191 EXPECT_EQ(false, writer.IsOpen());
192
193 BlobStore::BlobReader reader(blob);
194 EXPECT_EQ(false, reader.IsOpen());
195 ASSERT_EQ(OkStatus(), reader.Open());
196 EXPECT_EQ(true, reader.IsOpen());
197 EXPECT_EQ(OkStatus(), reader.Close());
198 EXPECT_EQ(false, reader.IsOpen());
199 }
200
TEST_F(BlobStoreTest,Discard)201 TEST_F(BlobStoreTest, Discard) {
202 InitSourceBufferToRandom(0x8675309);
203 WriteTestBlock();
204 constexpr char blob_title[] = "TestBlobBlock";
205 std::array<std::byte, 64> tmp_buffer = {};
206
207 kvs::ChecksumCrc16 checksum;
208
209 // TODO: Do this test with flash/kvs in the different entry state
210 // combinations.
211
212 constexpr size_t kBufferSize = 256;
213 BlobStoreBuffer<kBufferSize> blob(
214 blob_title, partition_, &checksum, kvs::TestKvs(), kBufferSize);
215 EXPECT_EQ(OkStatus(), blob.Init());
216
217 BlobStore::BlobWriter writer(blob);
218
219 EXPECT_EQ(OkStatus(), writer.Open());
220 EXPECT_EQ(OkStatus(), writer.Write(tmp_buffer));
221
222 // The write does an implicit erase so there should be no key for this blob.
223 EXPECT_EQ(Status::NotFound(),
224 kvs::TestKvs().Get(blob_title, tmp_buffer).status());
225 EXPECT_EQ(OkStatus(), writer.Close());
226
227 EXPECT_EQ(OkStatus(), kvs::TestKvs().Get(blob_title, tmp_buffer).status());
228
229 EXPECT_EQ(OkStatus(), writer.Open());
230 EXPECT_EQ(OkStatus(), writer.Discard());
231 EXPECT_EQ(OkStatus(), writer.Close());
232
233 EXPECT_EQ(Status::NotFound(),
234 kvs::TestKvs().Get(blob_title, tmp_buffer).status());
235 }
236
TEST_F(BlobStoreTest,MultipleErase)237 TEST_F(BlobStoreTest, MultipleErase) {
238 constexpr size_t kBufferSize = 256;
239 BlobStoreBuffer<kBufferSize> blob(
240 "Blob_OK", partition_, nullptr, kvs::TestKvs(), kBufferSize);
241 EXPECT_EQ(OkStatus(), blob.Init());
242
243 BlobStore::BlobWriter writer(blob);
244 EXPECT_EQ(OkStatus(), writer.Open());
245
246 EXPECT_EQ(OkStatus(), writer.Erase());
247 EXPECT_EQ(OkStatus(), writer.Erase());
248 EXPECT_EQ(OkStatus(), writer.Erase());
249 }
250
TEST_F(BlobStoreTest,OffsetRead)251 TEST_F(BlobStoreTest, OffsetRead) {
252 InitSourceBufferToRandom(0x11309);
253 WriteTestBlock();
254
255 constexpr size_t kOffset = 10;
256 ASSERT_LT(kOffset, kBlobDataSize);
257
258 kvs::ChecksumCrc16 checksum;
259
260 char name[16] = "TestBlobBlock";
261 constexpr size_t kBufferSize = 16;
262 BlobStoreBuffer<kBufferSize> blob(
263 name, partition_, &checksum, kvs::TestKvs(), kBufferSize);
264 EXPECT_EQ(OkStatus(), blob.Init());
265 BlobStore::BlobReader reader(blob);
266 ASSERT_EQ(OkStatus(), reader.Open(kOffset));
267
268 std::array<std::byte, kBlobDataSize - kOffset> read_buffer;
269 ByteSpan read_span = read_buffer;
270 ASSERT_EQ(read_span.size_bytes(), reader.ConservativeReadLimit());
271
272 auto result = reader.Read(read_span);
273 ASSERT_EQ(result.status(), OkStatus());
274 EXPECT_EQ(OkStatus(), reader.Close());
275 VerifyFlash(read_buffer, kOffset);
276 }
277
TEST_F(BlobStoreTest,InvalidReadOffset)278 TEST_F(BlobStoreTest, InvalidReadOffset) {
279 InitSourceBufferToRandom(0x11309);
280 WriteTestBlock();
281
282 constexpr size_t kOffset = kBlobDataSize;
283
284 kvs::ChecksumCrc16 checksum;
285
286 char name[16] = "TestBlobBlock";
287 constexpr size_t kBufferSize = 16;
288 BlobStoreBuffer<kBufferSize> blob(
289 name, partition_, &checksum, kvs::TestKvs(), kBufferSize);
290 EXPECT_EQ(OkStatus(), blob.Init());
291 BlobStore::BlobReader reader(blob);
292 ASSERT_EQ(Status::InvalidArgument(), reader.Open(kOffset));
293 }
294
295 // Test reading with a read buffer larger than the available data in the
TEST_F(BlobStoreTest,ReadBufferIsLargerThanData)296 TEST_F(BlobStoreTest, ReadBufferIsLargerThanData) {
297 InitSourceBufferToRandom(0x57326);
298
299 constexpr size_t kWriteBytes = 64;
300 WriteTestBlock(kWriteBytes);
301
302 kvs::ChecksumCrc16 checksum;
303
304 char name[16] = "TestBlobBlock";
305 constexpr size_t kBufferSize = 16;
306 BlobStoreBuffer<kBufferSize> blob(
307 name, partition_, &checksum, kvs::TestKvs(), kBufferSize);
308 EXPECT_EQ(OkStatus(), blob.Init());
309 BlobStore::BlobReader reader(blob);
310 ASSERT_EQ(OkStatus(), reader.Open());
311 EXPECT_EQ(kWriteBytes, reader.ConservativeReadLimit());
312
313 std::array<std::byte, kWriteBytes + 10> read_buffer;
314 ByteSpan read_span = read_buffer;
315
316 auto result = reader.Read(read_span);
317 ASSERT_EQ(result.status(), OkStatus());
318 EXPECT_EQ(OkStatus(), reader.Close());
319 }
320
TEST_F(BlobStoreTest,ChunkRead1)321 TEST_F(BlobStoreTest, ChunkRead1) {
322 InitSourceBufferToRandom(0x8675309);
323 WriteTestBlock();
324 ChunkReadTest(1);
325 }
326
TEST_F(BlobStoreTest,ChunkRead3)327 TEST_F(BlobStoreTest, ChunkRead3) {
328 InitSourceBufferToFill(0);
329 WriteTestBlock();
330 ChunkReadTest(3);
331 }
332
TEST_F(BlobStoreTest,ChunkRead4)333 TEST_F(BlobStoreTest, ChunkRead4) {
334 InitSourceBufferToFill(1);
335 WriteTestBlock();
336 ChunkReadTest(4);
337 }
338
TEST_F(BlobStoreTest,ChunkRead5)339 TEST_F(BlobStoreTest, ChunkRead5) {
340 InitSourceBufferToFill(0xff);
341 WriteTestBlock();
342 ChunkReadTest(5);
343 }
344
TEST_F(BlobStoreTest,ChunkRead16)345 TEST_F(BlobStoreTest, ChunkRead16) {
346 InitSourceBufferToRandom(0x86);
347 WriteTestBlock();
348 ChunkReadTest(16);
349 }
350
TEST_F(BlobStoreTest,ChunkRead64)351 TEST_F(BlobStoreTest, ChunkRead64) {
352 InitSourceBufferToRandom(0x9);
353 WriteTestBlock();
354 ChunkReadTest(64);
355 }
356
TEST_F(BlobStoreTest,ChunkReadFull)357 TEST_F(BlobStoreTest, ChunkReadFull) {
358 InitSourceBufferToRandom(0x9);
359 WriteTestBlock();
360 ChunkReadTest(kBlobDataSize);
361 }
362
TEST_F(BlobStoreTest,PartialBufferThenClose)363 TEST_F(BlobStoreTest, PartialBufferThenClose) {
364 // Do write of only a partial chunk, which will only have bytes in buffer
365 // (none written to flash) at close.
366 size_t data_bytes = 12;
367 InitSourceBufferToRandom(0x111, data_bytes);
368 WriteTestBlock(data_bytes);
369
370 // Do write with several full chunks and then some partial.
371 data_bytes = 158;
372 InitSourceBufferToRandom(0x3222, data_bytes);
373 }
374
375 // Test to do write/close, write/close multiple times.
TEST_F(BlobStoreTest,MultipleWrites)376 TEST_F(BlobStoreTest, MultipleWrites) {
377 InitSourceBufferToRandom(0x1121);
378 WriteTestBlock();
379 InitSourceBufferToRandom(0x515);
380 WriteTestBlock();
381 InitSourceBufferToRandom(0x4321);
382 WriteTestBlock();
383 }
384
385 } // namespace
386 } // namespace pw::blob_store
387