1 // Copyright (C) 2023 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "icing/join/qualified-id-join-index-impl-v2.h"
16
17 #include <cstdint>
18 #include <memory>
19 #include <numeric>
20 #include <string>
21 #include <string_view>
22 #include <utility>
23 #include <vector>
24
25 #include "icing/text_classifier/lib3/utils/base/status.h"
26 #include "icing/text_classifier/lib3/utils/base/statusor.h"
27 #include "gmock/gmock.h"
28 #include "gtest/gtest.h"
29 #include "icing/absl_ports/str_cat.h"
30 #include "icing/file/filesystem.h"
31 #include "icing/file/persistent-storage.h"
32 #include "icing/file/posting_list/posting-list-identifier.h"
33 #include "icing/join/document-id-to-join-info.h"
34 #include "icing/join/qualified-id-join-index.h"
35 #include "icing/schema/joinable-property.h"
36 #include "icing/store/document-filter-data.h"
37 #include "icing/store/document-id.h"
38 #include "icing/store/key-mapper.h"
39 #include "icing/store/namespace-fingerprint-identifier.h"
40 #include "icing/store/namespace-id.h"
41 #include "icing/store/persistent-hash-map-key-mapper.h"
42 #include "icing/testing/common-matchers.h"
43 #include "icing/testing/tmp-directory.h"
44 #include "icing/util/crc32.h"
45 #include "icing/util/status-macros.h"
46
47 namespace icing {
48 namespace lib {
49
50 namespace {
51
52 using ::testing::ElementsAre;
53 using ::testing::Eq;
54 using ::testing::HasSubstr;
55 using ::testing::IsEmpty;
56 using ::testing::IsTrue;
57 using ::testing::Lt;
58 using ::testing::Ne;
59 using ::testing::Not;
60 using ::testing::Pointee;
61 using ::testing::SizeIs;
62
63 using Crcs = PersistentStorage::Crcs;
64 using Info = QualifiedIdJoinIndexImplV2::Info;
65
66 static constexpr int32_t kCorruptedValueOffset = 3;
67
68 class QualifiedIdJoinIndexImplV2Test : public ::testing::Test {
69 protected:
SetUp()70 void SetUp() override {
71 base_dir_ = GetTestTempDir() + "/icing";
72 ASSERT_THAT(filesystem_.CreateDirectoryRecursively(base_dir_.c_str()),
73 IsTrue());
74
75 working_path_ = base_dir_ + "/qualified_id_join_index_impl_v2_test";
76 }
77
TearDown()78 void TearDown() override {
79 filesystem_.DeleteDirectoryRecursively(base_dir_.c_str());
80 }
81
82 Filesystem filesystem_;
83 std::string base_dir_;
84 std::string working_path_;
85 };
86
87 libtextclassifier3::StatusOr<
88 std::vector<QualifiedIdJoinIndexImplV2::JoinDataType>>
GetJoinData(const QualifiedIdJoinIndexImplV2 & index,SchemaTypeId schema_type_id,JoinablePropertyId joinable_property_id)89 GetJoinData(const QualifiedIdJoinIndexImplV2& index,
90 SchemaTypeId schema_type_id,
91 JoinablePropertyId joinable_property_id) {
92 ICING_ASSIGN_OR_RETURN(
93 std::unique_ptr<QualifiedIdJoinIndex::JoinDataIteratorBase> iter,
94 index.GetIterator(schema_type_id, joinable_property_id));
95
96 std::vector<QualifiedIdJoinIndexImplV2::JoinDataType> result;
97 while (iter->Advance().ok()) {
98 result.push_back(iter->GetCurrent());
99 }
100
101 return result;
102 }
103
TEST_F(QualifiedIdJoinIndexImplV2Test,InvalidWorkingPath)104 TEST_F(QualifiedIdJoinIndexImplV2Test, InvalidWorkingPath) {
105 EXPECT_THAT(QualifiedIdJoinIndexImplV2::Create(
106 filesystem_, "/dev/null/qualified_id_join_index_impl_v2_test",
107 /*pre_mapping_fbv=*/false),
108 StatusIs(libtextclassifier3::StatusCode::INTERNAL));
109 }
110
TEST_F(QualifiedIdJoinIndexImplV2Test,InitializeNewFiles)111 TEST_F(QualifiedIdJoinIndexImplV2Test, InitializeNewFiles) {
112 {
113 // Create new qualified id join index
114 ASSERT_FALSE(filesystem_.DirectoryExists(working_path_.c_str()));
115 ICING_ASSERT_OK_AND_ASSIGN(
116 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
117 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
118 /*pre_mapping_fbv=*/false));
119 EXPECT_THAT(index, Pointee(IsEmpty()));
120
121 ICING_ASSERT_OK(index->PersistToDisk());
122 }
123
124 // Metadata file should be initialized correctly for both info and crcs
125 // sections.
126 const std::string metadata_file_path =
127 absl_ports::StrCat(working_path_, "/metadata");
128 auto metadata_buffer = std::make_unique<uint8_t[]>(
129 QualifiedIdJoinIndexImplV2::kMetadataFileSize);
130 ASSERT_THAT(
131 filesystem_.PRead(metadata_file_path.c_str(), metadata_buffer.get(),
132 QualifiedIdJoinIndexImplV2::kMetadataFileSize,
133 /*offset=*/0),
134 IsTrue());
135
136 // Check info section
137 const Info* info = reinterpret_cast<const Info*>(
138 metadata_buffer.get() +
139 QualifiedIdJoinIndexImplV2::kInfoMetadataBufferOffset);
140 EXPECT_THAT(info->magic, Eq(Info::kMagic));
141 EXPECT_THAT(info->num_data, Eq(0));
142 EXPECT_THAT(info->last_added_document_id, Eq(kInvalidDocumentId));
143
144 // Check crcs section
145 const Crcs* crcs = reinterpret_cast<const Crcs*>(
146 metadata_buffer.get() +
147 QualifiedIdJoinIndexImplV2::kCrcsMetadataBufferOffset);
148 // There are some initial info in KeyMapper, so storages_crc should be
149 // non-zero.
150 EXPECT_THAT(crcs->component_crcs.storages_crc, Ne(0));
151 EXPECT_THAT(crcs->component_crcs.info_crc,
152 Eq(Crc32(std::string_view(reinterpret_cast<const char*>(info),
153 sizeof(Info)))
154 .Get()));
155 EXPECT_THAT(crcs->all_crc,
156 Eq(Crc32(std::string_view(
157 reinterpret_cast<const char*>(&crcs->component_crcs),
158 sizeof(Crcs::ComponentCrcs)))
159 .Get()));
160 }
161
TEST_F(QualifiedIdJoinIndexImplV2Test,InitializationShouldFailWithoutPersistToDiskOrDestruction)162 TEST_F(QualifiedIdJoinIndexImplV2Test,
163 InitializationShouldFailWithoutPersistToDiskOrDestruction) {
164 NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/12);
165 NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34);
166 NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/56);
167 NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/78);
168
169 // Create new qualified id join index
170 ICING_ASSERT_OK_AND_ASSIGN(
171 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
172 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
173 /*pre_mapping_fbv=*/false));
174
175 // Insert some data.
176 ICING_ASSERT_OK(index->Put(
177 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5,
178 /*ref_namespace_fingerprint_ids=*/{id2, id1}));
179 ICING_ASSERT_OK(index->PersistToDisk());
180 ICING_ASSERT_OK(index->Put(
181 /*schema_type_id=*/3, /*joinable_property_id=*/10, /*document_id=*/6,
182 /*ref_namespace_fingerprint_ids=*/{id3}));
183 ICING_ASSERT_OK(index->Put(
184 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/12,
185 /*ref_namespace_fingerprint_ids=*/{id4}));
186
187 // Without calling PersistToDisk, checksums will not be recomputed or synced
188 // to disk, so initializing another instance on the same files should fail.
189 EXPECT_THAT(QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
190 /*pre_mapping_fbv=*/false),
191 StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION));
192 }
193
TEST_F(QualifiedIdJoinIndexImplV2Test,InitializationShouldSucceedWithPersistToDisk)194 TEST_F(QualifiedIdJoinIndexImplV2Test,
195 InitializationShouldSucceedWithPersistToDisk) {
196 NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/12);
197 NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34);
198 NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/56);
199 NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/78);
200
201 // Create new qualified id join index
202 ICING_ASSERT_OK_AND_ASSIGN(
203 std::unique_ptr<QualifiedIdJoinIndexImplV2> index1,
204 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
205 /*pre_mapping_fbv=*/false));
206
207 // Insert some data.
208 ICING_ASSERT_OK(index1->Put(
209 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5,
210 /*ref_namespace_fingerprint_ids=*/{id2, id1}));
211 ICING_ASSERT_OK(index1->Put(
212 /*schema_type_id=*/3, /*joinable_property_id=*/10, /*document_id=*/6,
213 /*ref_namespace_fingerprint_ids=*/{id3}));
214 ICING_ASSERT_OK(index1->Put(
215 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/12,
216 /*ref_namespace_fingerprint_ids=*/{id4}));
217 ASSERT_THAT(index1, Pointee(SizeIs(4)));
218
219 // After calling PersistToDisk, all checksums should be recomputed and synced
220 // correctly to disk, so initializing another instance on the same files
221 // should succeed, and we should be able to get the same contents.
222 ICING_EXPECT_OK(index1->PersistToDisk());
223
224 ICING_ASSERT_OK_AND_ASSIGN(
225 std::unique_ptr<QualifiedIdJoinIndexImplV2> index2,
226 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
227 /*pre_mapping_fbv=*/false));
228 EXPECT_THAT(index2, Pointee(SizeIs(4)));
229 EXPECT_THAT(
230 GetJoinData(*index2, /*schema_type_id=*/2, /*joinable_property_id=*/1),
231 IsOkAndHolds(
232 ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
233 /*document_id=*/12, /*join_info=*/id4),
234 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
235 /*document_id=*/5, /*join_info=*/id2),
236 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
237 /*document_id=*/5, /*join_info=*/id1))));
238 EXPECT_THAT(
239 GetJoinData(*index2, /*schema_type_id=*/3, /*joinable_property_id=*/10),
240 IsOkAndHolds(
241 ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
242 /*document_id=*/6, /*join_info=*/id3))));
243 }
244
TEST_F(QualifiedIdJoinIndexImplV2Test,InitializationShouldSucceedAfterDestruction)245 TEST_F(QualifiedIdJoinIndexImplV2Test,
246 InitializationShouldSucceedAfterDestruction) {
247 NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/12);
248 NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34);
249 NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/56);
250 NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/78);
251
252 {
253 // Create new qualified id join index
254 ICING_ASSERT_OK_AND_ASSIGN(
255 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
256 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
257 /*pre_mapping_fbv=*/false));
258
259 // Insert some data.
260 ICING_ASSERT_OK(index->Put(
261 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5,
262 /*ref_namespace_fingerprint_ids=*/{id2, id1}));
263 ICING_ASSERT_OK(index->Put(
264 /*schema_type_id=*/3, /*joinable_property_id=*/10, /*document_id=*/6,
265 /*ref_namespace_fingerprint_ids=*/{id3}));
266 ICING_ASSERT_OK(index->Put(
267 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/12,
268 /*ref_namespace_fingerprint_ids=*/{id4}));
269 ASSERT_THAT(index, Pointee(SizeIs(4)));
270 }
271
272 {
273 // The previous instance went out of scope and was destructed. Although we
274 // didn't call PersistToDisk explicitly, the destructor should invoke it and
275 // thus initializing another instance on the same files should succeed, and
276 // we should be able to get the same contents.
277 ICING_ASSERT_OK_AND_ASSIGN(
278 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
279 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
280 /*pre_mapping_fbv=*/false));
281 EXPECT_THAT(index, Pointee(SizeIs(4)));
282 EXPECT_THAT(
283 GetJoinData(*index, /*schema_type_id=*/2, /*joinable_property_id=*/1),
284 IsOkAndHolds(
285 ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
286 /*document_id=*/12, /*join_info=*/id4),
287 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
288 /*document_id=*/5, /*join_info=*/id2),
289 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
290 /*document_id=*/5, /*join_info=*/id1))));
291 EXPECT_THAT(
292 GetJoinData(*index, /*schema_type_id=*/3, /*joinable_property_id=*/10),
293 IsOkAndHolds(
294 ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
295 /*document_id=*/6, /*join_info=*/id3))));
296 }
297 }
298
TEST_F(QualifiedIdJoinIndexImplV2Test,InitializeExistingFilesWithDifferentMagicShouldFail)299 TEST_F(QualifiedIdJoinIndexImplV2Test,
300 InitializeExistingFilesWithDifferentMagicShouldFail) {
301 {
302 // Create new qualified id join index
303 ICING_ASSERT_OK_AND_ASSIGN(
304 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
305 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
306 /*pre_mapping_fbv=*/false));
307 ICING_ASSERT_OK(index->Put(
308 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5,
309 /*ref_namespace_fingerprint_ids=*/
310 {NamespaceFingerprintIdentifier(/*namespace_id=*/1,
311 /*fingerprint=*/12)}));
312
313 ICING_ASSERT_OK(index->PersistToDisk());
314 }
315
316 {
317 const std::string metadata_file_path =
318 absl_ports::StrCat(working_path_, "/metadata");
319 ScopedFd metadata_sfd(filesystem_.OpenForWrite(metadata_file_path.c_str()));
320 ASSERT_THAT(metadata_sfd.is_valid(), IsTrue());
321
322 auto metadata_buffer = std::make_unique<uint8_t[]>(
323 QualifiedIdJoinIndexImplV2::kMetadataFileSize);
324 ASSERT_THAT(filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(),
325 QualifiedIdJoinIndexImplV2::kMetadataFileSize,
326 /*offset=*/0),
327 IsTrue());
328
329 // Manually change magic and update checksum
330 Crcs* crcs = reinterpret_cast<Crcs*>(
331 metadata_buffer.get() +
332 QualifiedIdJoinIndexImplV2::kCrcsMetadataBufferOffset);
333 Info* info = reinterpret_cast<Info*>(
334 metadata_buffer.get() +
335 QualifiedIdJoinIndexImplV2::kInfoMetadataBufferOffset);
336 info->magic += kCorruptedValueOffset;
337 crcs->component_crcs.info_crc = info->ComputeChecksum().Get();
338 crcs->all_crc = crcs->component_crcs.ComputeChecksum().Get();
339 ASSERT_THAT(filesystem_.PWrite(
340 metadata_sfd.get(), /*offset=*/0, metadata_buffer.get(),
341 QualifiedIdJoinIndexImplV2::kMetadataFileSize),
342 IsTrue());
343 }
344
345 // Attempt to create the qualified id join index with different magic. This
346 // should fail.
347 EXPECT_THAT(QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
348 /*pre_mapping_fbv=*/false),
349 StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION,
350 HasSubstr("Incorrect magic value")));
351 }
352
TEST_F(QualifiedIdJoinIndexImplV2Test,InitializeExistingFilesWithWrongAllCrcShouldFail)353 TEST_F(QualifiedIdJoinIndexImplV2Test,
354 InitializeExistingFilesWithWrongAllCrcShouldFail) {
355 {
356 // Create new qualified id join index
357 ICING_ASSERT_OK_AND_ASSIGN(
358 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
359 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
360 /*pre_mapping_fbv=*/false));
361 ICING_ASSERT_OK(index->Put(
362 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5,
363 /*ref_namespace_fingerprint_ids=*/
364 {NamespaceFingerprintIdentifier(/*namespace_id=*/1,
365 /*fingerprint=*/12)}));
366
367 ICING_ASSERT_OK(index->PersistToDisk());
368 }
369
370 {
371 const std::string metadata_file_path =
372 absl_ports::StrCat(working_path_, "/metadata");
373 ScopedFd metadata_sfd(filesystem_.OpenForWrite(metadata_file_path.c_str()));
374 ASSERT_THAT(metadata_sfd.is_valid(), IsTrue());
375
376 auto metadata_buffer = std::make_unique<uint8_t[]>(
377 QualifiedIdJoinIndexImplV2::kMetadataFileSize);
378 ASSERT_THAT(filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(),
379 QualifiedIdJoinIndexImplV2::kMetadataFileSize,
380 /*offset=*/0),
381 IsTrue());
382
383 // Manually corrupt all_crc
384 Crcs* crcs = reinterpret_cast<Crcs*>(
385 metadata_buffer.get() +
386 QualifiedIdJoinIndexImplV2::kCrcsMetadataBufferOffset);
387 crcs->all_crc += kCorruptedValueOffset;
388
389 ASSERT_THAT(filesystem_.PWrite(
390 metadata_sfd.get(), /*offset=*/0, metadata_buffer.get(),
391 QualifiedIdJoinIndexImplV2::kMetadataFileSize),
392 IsTrue());
393 }
394
395 // Attempt to create the qualified id join index with metadata containing
396 // corrupted all_crc. This should fail.
397 EXPECT_THAT(QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
398 /*pre_mapping_fbv=*/false),
399 StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION,
400 HasSubstr("Invalid all crc")));
401 }
402
TEST_F(QualifiedIdJoinIndexImplV2Test,InitializeExistingFilesWithCorruptedInfoShouldFail)403 TEST_F(QualifiedIdJoinIndexImplV2Test,
404 InitializeExistingFilesWithCorruptedInfoShouldFail) {
405 {
406 // Create new qualified id join index
407 ICING_ASSERT_OK_AND_ASSIGN(
408 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
409 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
410 /*pre_mapping_fbv=*/false));
411 ICING_ASSERT_OK(index->Put(
412 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5,
413 /*ref_namespace_fingerprint_ids=*/
414 {NamespaceFingerprintIdentifier(/*namespace_id=*/1,
415 /*fingerprint=*/12)}));
416
417 ICING_ASSERT_OK(index->PersistToDisk());
418 }
419
420 {
421 const std::string metadata_file_path =
422 absl_ports::StrCat(working_path_, "/metadata");
423 ScopedFd metadata_sfd(filesystem_.OpenForWrite(metadata_file_path.c_str()));
424 ASSERT_THAT(metadata_sfd.is_valid(), IsTrue());
425
426 auto metadata_buffer = std::make_unique<uint8_t[]>(
427 QualifiedIdJoinIndexImplV2::kMetadataFileSize);
428 ASSERT_THAT(filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(),
429 QualifiedIdJoinIndexImplV2::kMetadataFileSize,
430 /*offset=*/0),
431 IsTrue());
432
433 // Modify info, but don't update the checksum. This would be similar to
434 // corruption of info.
435 Info* info = reinterpret_cast<Info*>(
436 metadata_buffer.get() +
437 QualifiedIdJoinIndexImplV2::kInfoMetadataBufferOffset);
438 info->last_added_document_id += kCorruptedValueOffset;
439
440 ASSERT_THAT(filesystem_.PWrite(
441 metadata_sfd.get(), /*offset=*/0, metadata_buffer.get(),
442 QualifiedIdJoinIndexImplV2::kMetadataFileSize),
443 IsTrue());
444 }
445
446 // Attempt to create the qualified id join index with info that doesn't match
447 // its checksum. This should fail.
448 EXPECT_THAT(QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
449 /*pre_mapping_fbv=*/false),
450 StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION,
451 HasSubstr("Invalid info crc")));
452 }
453
TEST_F(QualifiedIdJoinIndexImplV2Test,InitializeExistingFilesWithCorruptedSchemaJoinableIdToPostingListMapperShouldFail)454 TEST_F(
455 QualifiedIdJoinIndexImplV2Test,
456 InitializeExistingFilesWithCorruptedSchemaJoinableIdToPostingListMapperShouldFail) {
457 {
458 // Create new qualified id join index
459 ICING_ASSERT_OK_AND_ASSIGN(
460 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
461 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
462 /*pre_mapping_fbv=*/false));
463 ICING_ASSERT_OK(index->Put(
464 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5,
465 /*ref_namespace_fingerprint_ids=*/
466 {NamespaceFingerprintIdentifier(/*namespace_id=*/1,
467 /*fingerprint=*/12)}));
468
469 ICING_ASSERT_OK(index->PersistToDisk());
470 }
471
472 // Corrupt schema_joinable_id_to_posting_list_mapper manually.
473 {
474 std::string mapper_working_path = absl_ports::StrCat(
475 working_path_, "/schema_joinable_id_to_posting_list_mapper");
476 ICING_ASSERT_OK_AND_ASSIGN(
477 std::unique_ptr<KeyMapper<PostingListIdentifier>> mapper,
478 PersistentHashMapKeyMapper<PostingListIdentifier>::Create(
479 filesystem_, std::move(mapper_working_path),
480 /*pre_mapping_fbv=*/false));
481 ICING_ASSERT_OK_AND_ASSIGN(Crc32 old_crc, mapper->ComputeChecksum());
482 ICING_ASSERT_OK(mapper->Put("foo", PostingListIdentifier::kInvalid));
483 ICING_ASSERT_OK(mapper->PersistToDisk());
484 ICING_ASSERT_OK_AND_ASSIGN(Crc32 new_crc, mapper->ComputeChecksum());
485 ASSERT_THAT(old_crc, Not(Eq(new_crc)));
486 }
487
488 // Attempt to create the qualified id join index with corrupted
489 // doc_join_info_mapper. This should fail.
490 EXPECT_THAT(QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
491 /*pre_mapping_fbv=*/false),
492 StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION,
493 HasSubstr("Invalid storages crc")));
494 }
495
TEST_F(QualifiedIdJoinIndexImplV2Test,InvalidPut)496 TEST_F(QualifiedIdJoinIndexImplV2Test, InvalidPut) {
497 NamespaceFingerprintIdentifier id(/*namespace_id=*/1, /*fingerprint=*/12);
498
499 // Create new qualified id join index
500 ICING_ASSERT_OK_AND_ASSIGN(
501 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
502 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
503 /*pre_mapping_fbv=*/false));
504
505 EXPECT_THAT(
506 index->Put(/*schema_type_id=*/-1, /*joinable_property_id=*/1,
507 /*document_id=*/5, /*ref_namespace_fingerprint_ids=*/{id}),
508 StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
509 EXPECT_THAT(
510 index->Put(/*schema_type_id=*/2, /*joinable_property_id=*/-1,
511 /*document_id=*/5, /*ref_namespace_fingerprint_ids=*/{id}),
512 StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
513 EXPECT_THAT(index->Put(/*schema_type_id=*/2, /*joinable_property_id=*/1,
514 /*document_id=*/kInvalidDocumentId,
515 /*ref_namespace_fingerprint_ids=*/{id}),
516 StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
517 }
518
TEST_F(QualifiedIdJoinIndexImplV2Test,InvalidGetIterator)519 TEST_F(QualifiedIdJoinIndexImplV2Test, InvalidGetIterator) {
520 // Create new qualified id join index
521 ICING_ASSERT_OK_AND_ASSIGN(
522 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
523 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
524 /*pre_mapping_fbv=*/false));
525
526 EXPECT_THAT(
527 index->GetIterator(/*schema_type_id=*/-1, /*joinable_property_id=*/1),
528 StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
529 EXPECT_THAT(
530 index->GetIterator(/*schema_type_id=*/2, /*joinable_property_id=*/-1),
531 StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT));
532 }
533
TEST_F(QualifiedIdJoinIndexImplV2Test,PutEmptyRefNamespaceFingerprintIdsShouldReturnOk)534 TEST_F(QualifiedIdJoinIndexImplV2Test,
535 PutEmptyRefNamespaceFingerprintIdsShouldReturnOk) {
536 SchemaTypeId schema_type_id = 2;
537 JoinablePropertyId joinable_property_id = 1;
538
539 // Create new qualified id join index
540 ICING_ASSERT_OK_AND_ASSIGN(
541 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
542 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
543 /*pre_mapping_fbv=*/false));
544
545 EXPECT_THAT(
546 index->Put(schema_type_id, joinable_property_id, /*document_id=*/5,
547 /*ref_namespace_fingerprint_ids=*/{}),
548 IsOk());
549 EXPECT_THAT(index, Pointee(IsEmpty()));
550
551 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id),
552 IsOkAndHolds(IsEmpty()));
553 EXPECT_THAT(GetJoinData(*index, schema_type_id + 1, joinable_property_id),
554 IsOkAndHolds(IsEmpty()));
555 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id + 1),
556 IsOkAndHolds(IsEmpty()));
557 }
558
TEST_F(QualifiedIdJoinIndexImplV2Test,PutAndGetSingleSchemaTypeAndJoinableProperty)559 TEST_F(QualifiedIdJoinIndexImplV2Test,
560 PutAndGetSingleSchemaTypeAndJoinableProperty) {
561 SchemaTypeId schema_type_id = 2;
562 JoinablePropertyId joinable_property_id = 1;
563
564 NamespaceFingerprintIdentifier id1(/*namespace_id=*/3, /*fingerprint=*/12);
565 NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34);
566 NamespaceFingerprintIdentifier id3(/*namespace_id=*/2, /*fingerprint=*/56);
567 NamespaceFingerprintIdentifier id4(/*namespace_id=*/0, /*fingerprint=*/78);
568
569 {
570 // Create new qualified id join index
571 ICING_ASSERT_OK_AND_ASSIGN(
572 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
573 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
574 /*pre_mapping_fbv=*/false));
575
576 EXPECT_THAT(
577 index->Put(schema_type_id, joinable_property_id, /*document_id=*/5,
578 /*ref_namespace_fingerprint_ids=*/{id2, id1}),
579 IsOk());
580 EXPECT_THAT(
581 index->Put(schema_type_id, joinable_property_id, /*document_id=*/6,
582 /*ref_namespace_fingerprint_ids=*/{id3}),
583 IsOk());
584 EXPECT_THAT(
585 index->Put(schema_type_id, joinable_property_id, /*document_id=*/12,
586 /*ref_namespace_fingerprint_ids=*/{id4}),
587 IsOk());
588 EXPECT_THAT(index, Pointee(SizeIs(4)));
589
590 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id),
591 IsOkAndHolds(ElementsAre(
592 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
593 /*document_id=*/12, /*join_info=*/id4),
594 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
595 /*document_id=*/6, /*join_info=*/id3),
596 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
597 /*document_id=*/5, /*join_info=*/id1),
598 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
599 /*document_id=*/5, /*join_info=*/id2))));
600 EXPECT_THAT(GetJoinData(*index, schema_type_id + 1, joinable_property_id),
601 IsOkAndHolds(IsEmpty()));
602 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id + 1),
603 IsOkAndHolds(IsEmpty()));
604
605 ICING_ASSERT_OK(index->PersistToDisk());
606 }
607
608 // Verify we can get all of them after destructing and re-initializing.
609 ICING_ASSERT_OK_AND_ASSIGN(
610 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
611 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
612 /*pre_mapping_fbv=*/false));
613 EXPECT_THAT(index, Pointee(SizeIs(4)));
614 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id),
615 IsOkAndHolds(ElementsAre(
616 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
617 /*document_id=*/12, /*join_info=*/id4),
618 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
619 /*document_id=*/6, /*join_info=*/id3),
620 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
621 /*document_id=*/5, /*join_info=*/id1),
622 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
623 /*document_id=*/5, /*join_info=*/id2))));
624 EXPECT_THAT(GetJoinData(*index, schema_type_id + 1, joinable_property_id),
625 IsOkAndHolds(IsEmpty()));
626 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id + 1),
627 IsOkAndHolds(IsEmpty()));
628 }
629
TEST_F(QualifiedIdJoinIndexImplV2Test,PutAndGetMultipleSchemaTypesAndJoinableProperties)630 TEST_F(QualifiedIdJoinIndexImplV2Test,
631 PutAndGetMultipleSchemaTypesAndJoinableProperties) {
632 SchemaTypeId schema_type_id1 = 2;
633 SchemaTypeId schema_type_id2 = 4;
634
635 JoinablePropertyId joinable_property_id1 = 1;
636 JoinablePropertyId joinable_property_id2 = 10;
637
638 NamespaceFingerprintIdentifier id1(/*namespace_id=*/3, /*fingerprint=*/12);
639 NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34);
640 NamespaceFingerprintIdentifier id3(/*namespace_id=*/2, /*fingerprint=*/56);
641 NamespaceFingerprintIdentifier id4(/*namespace_id=*/0, /*fingerprint=*/78);
642
643 {
644 // Create new qualified id join index
645 ICING_ASSERT_OK_AND_ASSIGN(
646 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
647 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
648 /*pre_mapping_fbv=*/false));
649
650 EXPECT_THAT(
651 index->Put(schema_type_id1, joinable_property_id1, /*document_id=*/5,
652 /*ref_namespace_fingerprint_ids=*/{id1}),
653 IsOk());
654 EXPECT_THAT(
655 index->Put(schema_type_id1, joinable_property_id2, /*document_id=*/5,
656 /*ref_namespace_fingerprint_ids=*/{id2}),
657 IsOk());
658 EXPECT_THAT(
659 index->Put(schema_type_id2, joinable_property_id1, /*document_id=*/12,
660 /*ref_namespace_fingerprint_ids=*/{id3}),
661 IsOk());
662 EXPECT_THAT(
663 index->Put(schema_type_id2, joinable_property_id2, /*document_id=*/12,
664 /*ref_namespace_fingerprint_ids=*/{id4}),
665 IsOk());
666 EXPECT_THAT(index, Pointee(SizeIs(4)));
667
668 EXPECT_THAT(GetJoinData(*index, schema_type_id1, joinable_property_id1),
669 IsOkAndHolds(ElementsAre(
670 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
671 /*document_id=*/5, /*join_info=*/id1))));
672 EXPECT_THAT(GetJoinData(*index, schema_type_id1, joinable_property_id2),
673 IsOkAndHolds(ElementsAre(
674 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
675 /*document_id=*/5, /*join_info=*/id2))));
676 EXPECT_THAT(GetJoinData(*index, schema_type_id2, joinable_property_id1),
677 IsOkAndHolds(ElementsAre(
678 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
679 /*document_id=*/12, /*join_info=*/id3))));
680 EXPECT_THAT(GetJoinData(*index, schema_type_id2, joinable_property_id2),
681 IsOkAndHolds(ElementsAre(
682 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
683 /*document_id=*/12, /*join_info=*/id4))));
684
685 ICING_ASSERT_OK(index->PersistToDisk());
686 }
687
688 // Verify we can get all of them after destructing and re-initializing.
689 ICING_ASSERT_OK_AND_ASSIGN(
690 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
691 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
692 /*pre_mapping_fbv=*/false));
693 EXPECT_THAT(index, Pointee(SizeIs(4)));
694 EXPECT_THAT(GetJoinData(*index, schema_type_id1, joinable_property_id1),
695 IsOkAndHolds(ElementsAre(
696 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
697 /*document_id=*/5, /*join_info=*/id1))));
698 EXPECT_THAT(GetJoinData(*index, schema_type_id1, joinable_property_id2),
699 IsOkAndHolds(ElementsAre(
700 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
701 /*document_id=*/5, /*join_info=*/id2))));
702 EXPECT_THAT(GetJoinData(*index, schema_type_id2, joinable_property_id1),
703 IsOkAndHolds(ElementsAre(
704 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
705 /*document_id=*/12, /*join_info=*/id3))));
706 EXPECT_THAT(GetJoinData(*index, schema_type_id2, joinable_property_id2),
707 IsOkAndHolds(ElementsAre(
708 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
709 /*document_id=*/12, /*join_info=*/id4))));
710 }
711
TEST_F(QualifiedIdJoinIndexImplV2Test,SetLastAddedDocumentId)712 TEST_F(QualifiedIdJoinIndexImplV2Test, SetLastAddedDocumentId) {
713 ICING_ASSERT_OK_AND_ASSIGN(
714 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
715 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
716 /*pre_mapping_fbv=*/false));
717
718 EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId));
719
720 constexpr DocumentId kDocumentId = 100;
721 index->set_last_added_document_id(kDocumentId);
722 EXPECT_THAT(index->last_added_document_id(), Eq(kDocumentId));
723
724 constexpr DocumentId kNextDocumentId = 123;
725 index->set_last_added_document_id(kNextDocumentId);
726 EXPECT_THAT(index->last_added_document_id(), Eq(kNextDocumentId));
727 }
728
TEST_F(QualifiedIdJoinIndexImplV2Test,SetLastAddedDocumentIdShouldIgnoreNewDocumentIdNotGreaterThanTheCurrent)729 TEST_F(
730 QualifiedIdJoinIndexImplV2Test,
731 SetLastAddedDocumentIdShouldIgnoreNewDocumentIdNotGreaterThanTheCurrent) {
732 ICING_ASSERT_OK_AND_ASSIGN(
733 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
734 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
735 /*pre_mapping_fbv=*/false));
736
737 constexpr DocumentId kDocumentId = 123;
738 index->set_last_added_document_id(kDocumentId);
739 ASSERT_THAT(index->last_added_document_id(), Eq(kDocumentId));
740
741 constexpr DocumentId kNextDocumentId = 100;
742 ASSERT_THAT(kNextDocumentId, Lt(kDocumentId));
743 index->set_last_added_document_id(kNextDocumentId);
744 // last_added_document_id() should remain unchanged.
745 EXPECT_THAT(index->last_added_document_id(), Eq(kDocumentId));
746 }
747
TEST_F(QualifiedIdJoinIndexImplV2Test,Optimize)748 TEST_F(QualifiedIdJoinIndexImplV2Test, Optimize) {
749 // General test for Optimize().
750 ICING_ASSERT_OK_AND_ASSIGN(
751 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
752 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
753 /*pre_mapping_fbv=*/false));
754
755 SchemaTypeId schema_type_id1 = 2;
756 SchemaTypeId schema_type_id2 = 5;
757
758 JoinablePropertyId joinable_property_id1 = 11;
759 JoinablePropertyId joinable_property_id2 = 15;
760
761 NamespaceFingerprintIdentifier id1(/*namespace_id=*/2, /*fingerprint=*/101);
762 NamespaceFingerprintIdentifier id2(/*namespace_id=*/3, /*fingerprint=*/102);
763 NamespaceFingerprintIdentifier id3(/*namespace_id=*/4, /*fingerprint=*/103);
764 NamespaceFingerprintIdentifier id4(/*namespace_id=*/0, /*fingerprint=*/104);
765 NamespaceFingerprintIdentifier id5(/*namespace_id=*/0, /*fingerprint=*/105);
766 NamespaceFingerprintIdentifier id6(/*namespace_id=*/1, /*fingerprint=*/106);
767 NamespaceFingerprintIdentifier id7(/*namespace_id=*/3, /*fingerprint=*/107);
768 NamespaceFingerprintIdentifier id8(/*namespace_id=*/2, /*fingerprint=*/108);
769
770 EXPECT_THAT(
771 index->Put(schema_type_id1, joinable_property_id1, /*document_id=*/3,
772 /*ref_namespace_fingerprint_ids=*/{id1, id2, id3}),
773 IsOk());
774 EXPECT_THAT(
775 index->Put(schema_type_id2, joinable_property_id2, /*document_id=*/5,
776 /*ref_namespace_fingerprint_ids=*/{id4}),
777 IsOk());
778 EXPECT_THAT(
779 index->Put(schema_type_id2, joinable_property_id2, /*document_id=*/8,
780 /*ref_namespace_fingerprint_ids=*/{id5, id6}),
781 IsOk());
782 EXPECT_THAT(
783 index->Put(schema_type_id1, joinable_property_id1, /*document_id=*/13,
784 /*ref_namespace_fingerprint_ids=*/{id7}),
785 IsOk());
786 EXPECT_THAT(
787 index->Put(schema_type_id1, joinable_property_id1, /*document_id=*/21,
788 /*ref_namespace_fingerprint_ids=*/{id8}),
789 IsOk());
790 index->set_last_added_document_id(21);
791
792 ASSERT_THAT(index, Pointee(SizeIs(8)));
793
794 // Delete doc id = 5, 13, compress and keep the rest.
795 std::vector<DocumentId> document_id_old_to_new(22, kInvalidDocumentId);
796 document_id_old_to_new[3] = 0;
797 document_id_old_to_new[8] = 1;
798 document_id_old_to_new[21] = 2;
799
800 // Delete namespace id 1, 2 (and invalidate id1, id6, id8). Reorder namespace
801 // ids [0, 3, 4] to [1, 2, 0].
802 std::vector<NamespaceId> namespace_id_old_to_new(5, kInvalidNamespaceId);
803 namespace_id_old_to_new[0] = 1;
804 namespace_id_old_to_new[3] = 2;
805 namespace_id_old_to_new[4] = 0;
806
807 DocumentId new_last_added_document_id = 2;
808 EXPECT_THAT(index->Optimize(document_id_old_to_new, namespace_id_old_to_new,
809 new_last_added_document_id),
810 IsOk());
811 EXPECT_THAT(index, Pointee(SizeIs(3)));
812 EXPECT_THAT(index->last_added_document_id(), Eq(new_last_added_document_id));
813
814 // Verify GetIterator API should work normally after Optimize().
815 // 1) schema_type_id1, joinable_property_id1:
816 // - old_doc_id=21, old_ref_namespace_id=2: NOT FOUND
817 // - old_doc_id=13, old_ref_namespace_id=3: NOT FOUND
818 // - old_doc_id=3, old_ref_namespace_id=4:
819 // become new_doc_id=0, new_ref_namespace_id=0
820 // - old_doc_id=3, old_ref_namespace_id=3:
821 // become new_doc_id=0, new_ref_namespace_id=2
822 // - old_doc_id=3, old_ref_namespace_id=2: NOT FOUND
823 //
824 // For new_doc_id=0, it should reorder due to posting list restriction.
825 EXPECT_THAT(
826 GetJoinData(*index, schema_type_id1, joinable_property_id1),
827 IsOkAndHolds(ElementsAre(
828 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
829 /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier(
830 /*namespace_id=*/2, /*fingerprint=*/102)),
831 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
832 /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier(
833 /*namespace_id=*/0, /*fingerprint=*/103)))));
834
835 // 2) schema_type_id2, joinable_property_id2:
836 // - old_doc_id=8, old_ref_namespace_id=1: NOT FOUND
837 // - old_doc_id=8, old_ref_namespace_id=0:
838 // become new_doc_id=1, new_ref_namespace_id=1
839 // - old_doc_id=5, old_ref_namespace_id=0: NOT FOUND
840 EXPECT_THAT(
841 GetJoinData(*index, schema_type_id2, joinable_property_id2),
842 IsOkAndHolds(
843 ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
844 /*document_id=*/1, /*join_info=*/NamespaceFingerprintIdentifier(
845 /*namespace_id=*/1, /*fingerprint=*/105)))));
846
847 // Verify Put API should work normally after Optimize().
848 NamespaceFingerprintIdentifier id9(/*namespace_id=*/1, /*fingerprint=*/109);
849 EXPECT_THAT(
850 index->Put(schema_type_id1, joinable_property_id1, /*document_id=*/99,
851 /*ref_namespace_fingerprint_ids=*/{id9}),
852 IsOk());
853 index->set_last_added_document_id(99);
854
855 EXPECT_THAT(index, Pointee(SizeIs(4)));
856 EXPECT_THAT(index->last_added_document_id(), Eq(99));
857 EXPECT_THAT(
858 GetJoinData(*index, schema_type_id1, joinable_property_id1),
859 IsOkAndHolds(ElementsAre(
860 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
861 /*document_id=*/99, /*join_info=*/id9),
862 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
863 /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier(
864 /*namespace_id=*/2, /*fingerprint=*/102)),
865 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
866 /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier(
867 /*namespace_id=*/0, /*fingerprint=*/103)))));
868 }
869
TEST_F(QualifiedIdJoinIndexImplV2Test,OptimizeDocumentIdChange)870 TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeDocumentIdChange) {
871 // Specific test for Optimize(): document id compaction.
872
873 ICING_ASSERT_OK_AND_ASSIGN(
874 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
875 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
876 /*pre_mapping_fbv=*/false));
877
878 SchemaTypeId schema_type_id = 2;
879 JoinablePropertyId joinable_property_id = 1;
880
881 NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/101);
882 NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/102);
883 NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/103);
884 NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/104);
885 NamespaceFingerprintIdentifier id5(/*namespace_id=*/1, /*fingerprint=*/105);
886 NamespaceFingerprintIdentifier id6(/*namespace_id=*/1, /*fingerprint=*/106);
887
888 EXPECT_THAT(
889 index->Put(schema_type_id, joinable_property_id, /*document_id=*/3,
890 /*ref_namespace_fingerprint_ids=*/{id1, id2}),
891 IsOk());
892 EXPECT_THAT(
893 index->Put(schema_type_id, joinable_property_id, /*document_id=*/5,
894 /*ref_namespace_fingerprint_ids=*/{id3}),
895 IsOk());
896 EXPECT_THAT(
897 index->Put(schema_type_id, joinable_property_id, /*document_id=*/8,
898 /*ref_namespace_fingerprint_ids=*/{id4}),
899 IsOk());
900 EXPECT_THAT(
901 index->Put(schema_type_id, joinable_property_id, /*document_id=*/13,
902 /*ref_namespace_fingerprint_ids=*/{id5}),
903 IsOk());
904 EXPECT_THAT(
905 index->Put(schema_type_id, joinable_property_id, /*document_id=*/21,
906 /*ref_namespace_fingerprint_ids=*/{id6}),
907 IsOk());
908 index->set_last_added_document_id(21);
909
910 ASSERT_THAT(index, Pointee(SizeIs(6)));
911
912 // Delete doc id = 5, 8, compress and keep the rest.
913 std::vector<DocumentId> document_id_old_to_new(22, kInvalidDocumentId);
914 document_id_old_to_new[3] = 0;
915 document_id_old_to_new[13] = 1;
916 document_id_old_to_new[21] = 2;
917
918 // No change for namespace id.
919 std::vector<NamespaceId> namespace_id_old_to_new = {0, 1};
920
921 DocumentId new_last_added_document_id = 2;
922 EXPECT_THAT(index->Optimize(document_id_old_to_new, namespace_id_old_to_new,
923 new_last_added_document_id),
924 IsOk());
925 EXPECT_THAT(index, Pointee(SizeIs(4)));
926 EXPECT_THAT(index->last_added_document_id(), Eq(new_last_added_document_id));
927
928 // Verify GetIterator API should work normally after Optimize().
929 // - old_doc_id=21, join_info=id6: become doc_id=2, join_info=id6
930 // - old_doc_id=13, join_info=id5: become doc_id=1, join_info=id5
931 // - old_doc_id=8, join_info=id4: NOT FOUND
932 // - old_doc_id=5, join_info=id3: NOT FOUND
933 // - old_doc_id=3, join_info=id2: become doc_id=0, join_info=id2
934 // - old_doc_id=3, join_info=id1: become doc_id=0, join_info=id1
935 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id),
936 IsOkAndHolds(ElementsAre(
937 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
938 /*document_id=*/2, /*join_info=*/id6),
939 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
940 /*document_id=*/1, /*join_info=*/id5),
941 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
942 /*document_id=*/0, /*join_info=*/id2),
943 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
944 /*document_id=*/0, /*join_info=*/id1))));
945
946 // Verify Put API should work normally after Optimize().
947 NamespaceFingerprintIdentifier id7(/*namespace_id=*/1, /*fingerprint=*/107);
948 EXPECT_THAT(
949 index->Put(schema_type_id, joinable_property_id, /*document_id=*/99,
950 /*ref_namespace_fingerprint_ids=*/{id7}),
951 IsOk());
952 index->set_last_added_document_id(99);
953
954 EXPECT_THAT(index, Pointee(SizeIs(5)));
955 EXPECT_THAT(index->last_added_document_id(), Eq(99));
956 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id),
957 IsOkAndHolds(ElementsAre(
958 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
959 /*document_id=*/99, /*join_info=*/id7),
960 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
961 /*document_id=*/2, /*join_info=*/id6),
962 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
963 /*document_id=*/1, /*join_info=*/id5),
964 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
965 /*document_id=*/0, /*join_info=*/id2),
966 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
967 /*document_id=*/0, /*join_info=*/id1))));
968 }
969
TEST_F(QualifiedIdJoinIndexImplV2Test,OptimizeOutOfRangeDocumentId)970 TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeOutOfRangeDocumentId) {
971 // Specific test for Optimize() for out of range document id.
972
973 ICING_ASSERT_OK_AND_ASSIGN(
974 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
975 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
976 /*pre_mapping_fbv=*/false));
977
978 SchemaTypeId schema_type_id = 2;
979 JoinablePropertyId joinable_property_id = 1;
980 NamespaceFingerprintIdentifier id(/*namespace_id=*/1, /*fingerprint=*/101);
981
982 EXPECT_THAT(
983 index->Put(schema_type_id, joinable_property_id, /*document_id=*/99,
984 /*ref_namespace_fingerprint_ids=*/{id}),
985 IsOk());
986 index->set_last_added_document_id(99);
987
988 // Create document_id_old_to_new with size = 1. Optimize should handle out of
989 // range DocumentId properly.
990 std::vector<DocumentId> document_id_old_to_new = {kInvalidDocumentId};
991 std::vector<NamespaceId> namespace_id_old_to_new = {0, 1};
992
993 // There shouldn't be any error due to vector index.
994 EXPECT_THAT(
995 index->Optimize(document_id_old_to_new, namespace_id_old_to_new,
996 /*new_last_added_document_id=*/kInvalidDocumentId),
997 IsOk());
998 EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId));
999
1000 // Verify all data are discarded after Optimize().
1001 EXPECT_THAT(index, Pointee(IsEmpty()));
1002 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id),
1003 IsOkAndHolds(IsEmpty()));
1004 }
1005
TEST_F(QualifiedIdJoinIndexImplV2Test,OptimizeDeleteAllDocuments)1006 TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeDeleteAllDocuments) {
1007 // Specific test for Optimize(): delete all document ids.
1008
1009 ICING_ASSERT_OK_AND_ASSIGN(
1010 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
1011 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
1012 /*pre_mapping_fbv=*/false));
1013
1014 SchemaTypeId schema_type_id = 2;
1015 JoinablePropertyId joinable_property_id = 1;
1016
1017 NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/101);
1018 NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/102);
1019 NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/103);
1020 NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/104);
1021 NamespaceFingerprintIdentifier id5(/*namespace_id=*/1, /*fingerprint=*/105);
1022 NamespaceFingerprintIdentifier id6(/*namespace_id=*/1, /*fingerprint=*/106);
1023
1024 EXPECT_THAT(
1025 index->Put(schema_type_id, joinable_property_id, /*document_id=*/3,
1026 /*ref_namespace_fingerprint_ids=*/{id1, id2}),
1027 IsOk());
1028 EXPECT_THAT(
1029 index->Put(schema_type_id, joinable_property_id, /*document_id=*/5,
1030 /*ref_namespace_fingerprint_ids=*/{id3}),
1031 IsOk());
1032 EXPECT_THAT(
1033 index->Put(schema_type_id, joinable_property_id, /*document_id=*/8,
1034 /*ref_namespace_fingerprint_ids=*/{id4}),
1035 IsOk());
1036 EXPECT_THAT(
1037 index->Put(schema_type_id, joinable_property_id, /*document_id=*/13,
1038 /*ref_namespace_fingerprint_ids=*/{id5}),
1039 IsOk());
1040 EXPECT_THAT(
1041 index->Put(schema_type_id, joinable_property_id, /*document_id=*/21,
1042 /*ref_namespace_fingerprint_ids=*/{id6}),
1043 IsOk());
1044 index->set_last_added_document_id(21);
1045
1046 ASSERT_THAT(index, Pointee(SizeIs(6)));
1047
1048 // Delete all documents.
1049 std::vector<DocumentId> document_id_old_to_new(22, kInvalidDocumentId);
1050
1051 // No change for namespace id.
1052 std::vector<NamespaceId> namespace_id_old_to_new = {0, 1};
1053
1054 EXPECT_THAT(
1055 index->Optimize(document_id_old_to_new, namespace_id_old_to_new,
1056 /*new_last_added_document_id=*/kInvalidDocumentId),
1057 IsOk());
1058 EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId));
1059
1060 // Verify all data are discarded after Optimize().
1061 EXPECT_THAT(index, Pointee(IsEmpty()));
1062 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id),
1063 IsOkAndHolds(IsEmpty()));
1064 }
1065
TEST_F(QualifiedIdJoinIndexImplV2Test,OptimizeNamespaceIdChange)1066 TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeNamespaceIdChange) {
1067 // Specific test for Optimize(): referenced namespace id compaction.
1068
1069 ICING_ASSERT_OK_AND_ASSIGN(
1070 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
1071 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
1072 /*pre_mapping_fbv=*/false));
1073
1074 SchemaTypeId schema_type_id = 2;
1075 JoinablePropertyId joinable_property_id = 1;
1076
1077 NamespaceFingerprintIdentifier id1(/*namespace_id=*/3, /*fingerprint=*/101);
1078 NamespaceFingerprintIdentifier id2(/*namespace_id=*/5, /*fingerprint=*/102);
1079 NamespaceFingerprintIdentifier id3(/*namespace_id=*/4, /*fingerprint=*/103);
1080 NamespaceFingerprintIdentifier id4(/*namespace_id=*/0, /*fingerprint=*/104);
1081 NamespaceFingerprintIdentifier id5(/*namespace_id=*/2, /*fingerprint=*/105);
1082 NamespaceFingerprintIdentifier id6(/*namespace_id=*/1, /*fingerprint=*/106);
1083
1084 EXPECT_THAT(
1085 index->Put(schema_type_id, joinable_property_id, /*document_id=*/2,
1086 /*ref_namespace_fingerprint_ids=*/{id1}),
1087 IsOk());
1088 EXPECT_THAT(
1089 index->Put(schema_type_id, joinable_property_id, /*document_id=*/3,
1090 /*ref_namespace_fingerprint_ids=*/{id2}),
1091 IsOk());
1092 EXPECT_THAT(
1093 index->Put(schema_type_id, joinable_property_id, /*document_id=*/5,
1094 /*ref_namespace_fingerprint_ids=*/{id3}),
1095 IsOk());
1096 EXPECT_THAT(
1097 index->Put(schema_type_id, joinable_property_id, /*document_id=*/8,
1098 /*ref_namespace_fingerprint_ids=*/{id4}),
1099 IsOk());
1100 EXPECT_THAT(
1101 index->Put(schema_type_id, joinable_property_id, /*document_id=*/13,
1102 /*ref_namespace_fingerprint_ids=*/{id5}),
1103 IsOk());
1104 EXPECT_THAT(
1105 index->Put(schema_type_id, joinable_property_id, /*document_id=*/21,
1106 /*ref_namespace_fingerprint_ids=*/{id6}),
1107 IsOk());
1108 index->set_last_added_document_id(21);
1109
1110 ASSERT_THAT(index, Pointee(SizeIs(6)));
1111
1112 // No change for document id.
1113 std::vector<DocumentId> document_id_old_to_new(22);
1114 std::iota(document_id_old_to_new.begin(), document_id_old_to_new.end(), 0);
1115
1116 // Delete namespace id 2, 4. Reorder namespace id [0, 1, 3, 5] to [2, 3, 1,
1117 // 0].
1118 std::vector<NamespaceId> namespace_id_old_to_new(6, kInvalidNamespaceId);
1119 namespace_id_old_to_new[0] = 2;
1120 namespace_id_old_to_new[1] = 3;
1121 namespace_id_old_to_new[3] = 1;
1122 namespace_id_old_to_new[5] = 0;
1123
1124 DocumentId new_last_added_document_id = 21;
1125 EXPECT_THAT(index->Optimize(document_id_old_to_new, namespace_id_old_to_new,
1126 new_last_added_document_id),
1127 IsOk());
1128 EXPECT_THAT(index, Pointee(SizeIs(4)));
1129 EXPECT_THAT(index->last_added_document_id(), Eq(new_last_added_document_id));
1130
1131 // Verify GetIterator API should work normally after Optimize().
1132 // - id6 (old_namespace_id=1): new_namespace_id=3 (document_id = 21)
1133 // - id5 (old_namespace_id=2): NOT FOUND
1134 // - id4 (old_namespace_id=0): new_namespace_id=2 (document_id = 8)
1135 // - id3 (old_namespace_id=4): NOT FOUND
1136 // - id2 (old_namespace_id=5): new_namespace_id=0 (document_id = 3)
1137 // - id1 (old_namespace_id=3): new_namespace_id=1 (document_id = 2)
1138 EXPECT_THAT(
1139 GetJoinData(*index, schema_type_id, joinable_property_id),
1140 IsOkAndHolds(ElementsAre(
1141 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1142 /*document_id=*/21, /*join_info=*/NamespaceFingerprintIdentifier(
1143 /*namespace_id=*/3, /*fingerprint=*/106)),
1144 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1145 /*document_id=*/8, /*join_info=*/NamespaceFingerprintIdentifier(
1146 /*namespace_id=*/2, /*fingerprint=*/104)),
1147 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1148 /*document_id=*/3, /*join_info=*/NamespaceFingerprintIdentifier(
1149 /*namespace_id=*/0, /*fingerprint=*/102)),
1150 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1151 /*document_id=*/2, /*join_info=*/NamespaceFingerprintIdentifier(
1152 /*namespace_id=*/1, /*fingerprint=*/101)))));
1153
1154 // Verify Put API should work normally after Optimize().
1155 NamespaceFingerprintIdentifier id7(/*namespace_id=*/1, /*fingerprint=*/107);
1156 EXPECT_THAT(
1157 index->Put(schema_type_id, joinable_property_id, /*document_id=*/99,
1158 /*ref_namespace_fingerprint_ids=*/{id7}),
1159 IsOk());
1160 index->set_last_added_document_id(99);
1161
1162 EXPECT_THAT(index, Pointee(SizeIs(5)));
1163 EXPECT_THAT(index->last_added_document_id(), Eq(99));
1164 EXPECT_THAT(
1165 GetJoinData(*index, schema_type_id, joinable_property_id),
1166 IsOkAndHolds(ElementsAre(
1167 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1168 /*document_id=*/99, /*join_info=*/id7),
1169 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1170 /*document_id=*/21, /*join_info=*/NamespaceFingerprintIdentifier(
1171 /*namespace_id=*/3, /*fingerprint=*/106)),
1172 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1173 /*document_id=*/8, /*join_info=*/NamespaceFingerprintIdentifier(
1174 /*namespace_id=*/2, /*fingerprint=*/104)),
1175 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1176 /*document_id=*/3, /*join_info=*/NamespaceFingerprintIdentifier(
1177 /*namespace_id=*/0, /*fingerprint=*/102)),
1178 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1179 /*document_id=*/2, /*join_info=*/NamespaceFingerprintIdentifier(
1180 /*namespace_id=*/1, /*fingerprint=*/101)))));
1181 }
1182
TEST_F(QualifiedIdJoinIndexImplV2Test,OptimizeNamespaceIdChangeShouldReorder)1183 TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeNamespaceIdChangeShouldReorder) {
1184 // Specific test for Optimize(): referenced namespace id reorder.
1185
1186 ICING_ASSERT_OK_AND_ASSIGN(
1187 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
1188 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
1189 /*pre_mapping_fbv=*/false));
1190
1191 SchemaTypeId schema_type_id = 2;
1192 JoinablePropertyId joinable_property_id = 1;
1193
1194 NamespaceFingerprintIdentifier id1(/*namespace_id=*/0, /*fingerprint=*/101);
1195 NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/102);
1196 NamespaceFingerprintIdentifier id3(/*namespace_id=*/2, /*fingerprint=*/103);
1197 NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/104);
1198
1199 EXPECT_THAT(
1200 index->Put(schema_type_id, joinable_property_id, /*document_id=*/0,
1201 /*ref_namespace_fingerprint_ids=*/{id1, id2, id3}),
1202 IsOk());
1203 EXPECT_THAT(
1204 index->Put(schema_type_id, joinable_property_id, /*document_id=*/1,
1205 /*ref_namespace_fingerprint_ids=*/{id4}),
1206 IsOk());
1207 index->set_last_added_document_id(1);
1208
1209 ASSERT_THAT(index, Pointee(SizeIs(4)));
1210
1211 // No change for document id.
1212 std::vector<DocumentId> document_id_old_to_new = {0, 1};
1213
1214 // Reorder namespace id [0, 1, 2] to [2, 0, 1].
1215 std::vector<NamespaceId> namespace_id_old_to_new = {2, 0, 1};
1216
1217 DocumentId new_last_added_document_id = 1;
1218 EXPECT_THAT(index->Optimize(document_id_old_to_new, namespace_id_old_to_new,
1219 new_last_added_document_id),
1220 IsOk());
1221 EXPECT_THAT(index, Pointee(SizeIs(4)));
1222 EXPECT_THAT(index->last_added_document_id(), Eq(new_last_added_document_id));
1223
1224 // Verify GetIterator API should work normally after Optimize().
1225 // - id4 (old_namespace_id=1): new_namespace_id=0 (document_id = 1)
1226 // - id3 (old_namespace_id=2): new_namespace_id=1 (document_id = 0)
1227 // - id2 (old_namespace_id=1): new_namespace_id=0 (document_id = 0)
1228 // - id1 (old_namespace_id=0): new_namespace_id=2 (document_id = 0)
1229 //
1230 // Should reorder to [id4, id1, id3, id2] due to posting list restriction.
1231 EXPECT_THAT(
1232 GetJoinData(*index, schema_type_id, joinable_property_id),
1233 IsOkAndHolds(ElementsAre(
1234 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1235 /*document_id=*/1, /*join_info=*/NamespaceFingerprintIdentifier(
1236 /*namespace_id=*/0, /*fingerprint=*/104)),
1237 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1238 /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier(
1239 /*namespace_id=*/2, /*fingerprint=*/101)),
1240 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1241 /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier(
1242 /*namespace_id=*/1, /*fingerprint=*/103)),
1243 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1244 /*document_id=*/0, /*join_info=*/NamespaceFingerprintIdentifier(
1245 /*namespace_id=*/0, /*fingerprint=*/102)))));
1246 }
1247
TEST_F(QualifiedIdJoinIndexImplV2Test,OptimizeOutOfRangeNamespaceId)1248 TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeOutOfRangeNamespaceId) {
1249 // Specific test for Optimize(): out of range referenced namespace id.
1250
1251 ICING_ASSERT_OK_AND_ASSIGN(
1252 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
1253 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
1254 /*pre_mapping_fbv=*/false));
1255
1256 SchemaTypeId schema_type_id = 2;
1257 JoinablePropertyId joinable_property_id = 1;
1258 NamespaceFingerprintIdentifier id(/*namespace_id=*/99, /*fingerprint=*/101);
1259
1260 EXPECT_THAT(
1261 index->Put(schema_type_id, joinable_property_id, /*document_id=*/0,
1262 /*ref_namespace_fingerprint_ids=*/{id}),
1263 IsOk());
1264 index->set_last_added_document_id(0);
1265
1266 // Create namespace_id_old_to_new with size = 1. Optimize should handle out of
1267 // range NamespaceId properly.
1268 std::vector<DocumentId> document_id_old_to_new = {0};
1269 std::vector<NamespaceId> namespace_id_old_to_new = {kInvalidNamespaceId};
1270
1271 // There shouldn't be any error due to vector index.
1272 EXPECT_THAT(
1273 index->Optimize(document_id_old_to_new, namespace_id_old_to_new,
1274 /*new_last_added_document_id=*/kInvalidDocumentId),
1275 IsOk());
1276 EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId));
1277
1278 // Verify all data are discarded after Optimize().
1279 EXPECT_THAT(index, Pointee(IsEmpty()));
1280 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id),
1281 IsOkAndHolds(IsEmpty()));
1282 }
1283
TEST_F(QualifiedIdJoinIndexImplV2Test,OptimizeDeleteAllNamespaces)1284 TEST_F(QualifiedIdJoinIndexImplV2Test, OptimizeDeleteAllNamespaces) {
1285 // Specific test for Optimize(): delete all referenced namespace ids.
1286
1287 ICING_ASSERT_OK_AND_ASSIGN(
1288 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
1289 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
1290 /*pre_mapping_fbv=*/false));
1291
1292 SchemaTypeId schema_type_id = 2;
1293 JoinablePropertyId joinable_property_id = 1;
1294
1295 NamespaceFingerprintIdentifier id1(/*namespace_id=*/0, /*fingerprint=*/101);
1296 NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/102);
1297 NamespaceFingerprintIdentifier id3(/*namespace_id=*/2, /*fingerprint=*/103);
1298
1299 EXPECT_THAT(
1300 index->Put(schema_type_id, joinable_property_id, /*document_id=*/0,
1301 /*ref_namespace_fingerprint_ids=*/{id1}),
1302 IsOk());
1303 EXPECT_THAT(
1304 index->Put(schema_type_id, joinable_property_id, /*document_id=*/1,
1305 /*ref_namespace_fingerprint_ids=*/{id2}),
1306 IsOk());
1307 EXPECT_THAT(
1308 index->Put(schema_type_id, joinable_property_id, /*document_id=*/2,
1309 /*ref_namespace_fingerprint_ids=*/{id3}),
1310 IsOk());
1311 index->set_last_added_document_id(3);
1312
1313 ASSERT_THAT(index, Pointee(SizeIs(3)));
1314
1315 // No change for document id.
1316 std::vector<DocumentId> document_id_old_to_new = {0, 1, 2};
1317
1318 // Delete all namespaces.
1319 std::vector<NamespaceId> namespace_id_old_to_new(3, kInvalidNamespaceId);
1320
1321 EXPECT_THAT(
1322 index->Optimize(document_id_old_to_new, namespace_id_old_to_new,
1323 /*new_last_added_document_id=*/kInvalidDocumentId),
1324 IsOk());
1325 EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId));
1326
1327 // Verify all data are discarded after Optimize().
1328 EXPECT_THAT(index, Pointee(IsEmpty()));
1329 EXPECT_THAT(GetJoinData(*index, schema_type_id, joinable_property_id),
1330 IsOkAndHolds(IsEmpty()));
1331 }
1332
TEST_F(QualifiedIdJoinIndexImplV2Test,Clear)1333 TEST_F(QualifiedIdJoinIndexImplV2Test, Clear) {
1334 NamespaceFingerprintIdentifier id1(/*namespace_id=*/1, /*fingerprint=*/12);
1335 NamespaceFingerprintIdentifier id2(/*namespace_id=*/1, /*fingerprint=*/34);
1336 NamespaceFingerprintIdentifier id3(/*namespace_id=*/1, /*fingerprint=*/56);
1337 NamespaceFingerprintIdentifier id4(/*namespace_id=*/1, /*fingerprint=*/78);
1338
1339 // Create new qualified id join index
1340 ICING_ASSERT_OK_AND_ASSIGN(
1341 std::unique_ptr<QualifiedIdJoinIndexImplV2> index,
1342 QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
1343 /*pre_mapping_fbv=*/false));
1344 // Insert some data.
1345 ICING_ASSERT_OK(index->Put(
1346 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/5,
1347 /*ref_namespace_fingerprint_ids=*/{id2, id1}));
1348 ICING_ASSERT_OK(index->Put(
1349 /*schema_type_id=*/3, /*joinable_property_id=*/10, /*document_id=*/6,
1350 /*ref_namespace_fingerprint_ids=*/{id3}));
1351 ICING_ASSERT_OK(index->Put(
1352 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/12,
1353 /*ref_namespace_fingerprint_ids=*/{id4}));
1354 ASSERT_THAT(index, Pointee(SizeIs(4)));
1355 index->set_last_added_document_id(12);
1356 ASSERT_THAT(index->last_added_document_id(), Eq(12));
1357
1358 // After Clear(), last_added_document_id should be set to kInvalidDocumentId,
1359 // and the previous added data should be deleted.
1360 EXPECT_THAT(index->Clear(), IsOk());
1361 EXPECT_THAT(index, Pointee(IsEmpty()));
1362 EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId));
1363 EXPECT_THAT(
1364 GetJoinData(*index, /*schema_type_id=*/2, /*joinable_property_id=*/1),
1365 IsOkAndHolds(IsEmpty()));
1366 EXPECT_THAT(
1367 GetJoinData(*index, /*schema_type_id=*/3, /*joinable_property_id=*/10),
1368 IsOkAndHolds(IsEmpty()));
1369
1370 // Join index should be able to work normally after Clear().
1371 ICING_ASSERT_OK(index->Put(
1372 /*schema_type_id=*/2, /*joinable_property_id=*/1, /*document_id=*/20,
1373 /*ref_namespace_fingerprint_ids=*/{id4, id2, id1, id3}));
1374 index->set_last_added_document_id(20);
1375
1376 EXPECT_THAT(index, Pointee(SizeIs(4)));
1377 EXPECT_THAT(index->last_added_document_id(), Eq(20));
1378 EXPECT_THAT(
1379 GetJoinData(*index, /*schema_type_id=*/2, /*joinable_property_id=*/1),
1380 IsOkAndHolds(
1381 ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1382 /*document_id=*/20, /*join_info=*/id4),
1383 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1384 /*document_id=*/20, /*join_info=*/id3),
1385 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1386 /*document_id=*/20, /*join_info=*/id2),
1387 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1388 /*document_id=*/20, /*join_info=*/id1))));
1389
1390 ICING_ASSERT_OK(index->PersistToDisk());
1391 index.reset();
1392
1393 // Verify index after reconstructing.
1394 ICING_ASSERT_OK_AND_ASSIGN(
1395 index, QualifiedIdJoinIndexImplV2::Create(filesystem_, working_path_,
1396 /*pre_mapping_fbv=*/false));
1397 EXPECT_THAT(index->last_added_document_id(), Eq(20));
1398 EXPECT_THAT(
1399 GetJoinData(*index, /*schema_type_id=*/2, /*joinable_property_id=*/1),
1400 IsOkAndHolds(
1401 ElementsAre(DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1402 /*document_id=*/20, /*join_info=*/id4),
1403 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1404 /*document_id=*/20, /*join_info=*/id3),
1405 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1406 /*document_id=*/20, /*join_info=*/id2),
1407 DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>(
1408 /*document_id=*/20, /*join_info=*/id1))));
1409 }
1410
1411 } // namespace
1412
1413 } // namespace lib
1414 } // namespace icing
1415