1 // Copyright (C) 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include <unistd.h>
16
17 #include <cstdint>
18 #include <limits>
19 #include <memory>
20 #include <string>
21 #include <utility>
22
23 #include "icing/text_classifier/lib3/utils/base/status.h"
24 #include "gmock/gmock.h"
25 #include "gtest/gtest.h"
26 #include "icing/document-builder.h"
27 #include "icing/file/filesystem.h"
28 #include "icing/file/mock-filesystem.h"
29 #include "icing/icing-search-engine.h"
30 #include "icing/jni/jni-cache.h"
31 #include "icing/join/join-processor.h"
32 #include "icing/portable/endian.h"
33 #include "icing/portable/equals-proto.h"
34 #include "icing/portable/platform.h"
35 #include "icing/proto/debug.pb.h"
36 #include "icing/proto/document.pb.h"
37 #include "icing/proto/document_wrapper.pb.h"
38 #include "icing/proto/initialize.pb.h"
39 #include "icing/proto/logging.pb.h"
40 #include "icing/proto/optimize.pb.h"
41 #include "icing/proto/persist.pb.h"
42 #include "icing/proto/reset.pb.h"
43 #include "icing/proto/schema.pb.h"
44 #include "icing/proto/scoring.pb.h"
45 #include "icing/proto/search.pb.h"
46 #include "icing/proto/status.pb.h"
47 #include "icing/proto/storage.pb.h"
48 #include "icing/proto/term.pb.h"
49 #include "icing/proto/usage.pb.h"
50 #include "icing/query/query-features.h"
51 #include "icing/schema-builder.h"
52 #include "icing/store/document-log-creator.h"
53 #include "icing/testing/common-matchers.h"
54 #include "icing/testing/fake-clock.h"
55 #include "icing/testing/icu-data-file-helper.h"
56 #include "icing/testing/jni-test-helpers.h"
57 #include "icing/testing/test-data.h"
58 #include "icing/testing/tmp-directory.h"
59
60 namespace icing {
61 namespace lib {
62
63 namespace {
64
65 using ::icing::lib::portable_equals_proto::EqualsProto;
66 using ::testing::Eq;
67 using ::testing::Ge;
68 using ::testing::Gt;
69 using ::testing::HasSubstr;
70 using ::testing::Lt;
71 using ::testing::Return;
72
73 // For mocking purpose, we allow tests to provide a custom Filesystem.
74 class TestIcingSearchEngine : public IcingSearchEngine {
75 public:
TestIcingSearchEngine(const IcingSearchEngineOptions & options,std::unique_ptr<const Filesystem> filesystem,std::unique_ptr<const IcingFilesystem> icing_filesystem,std::unique_ptr<Clock> clock,std::unique_ptr<JniCache> jni_cache)76 TestIcingSearchEngine(const IcingSearchEngineOptions& options,
77 std::unique_ptr<const Filesystem> filesystem,
78 std::unique_ptr<const IcingFilesystem> icing_filesystem,
79 std::unique_ptr<Clock> clock,
80 std::unique_ptr<JniCache> jni_cache)
81 : IcingSearchEngine(options, std::move(filesystem),
82 std::move(icing_filesystem), std::move(clock),
83 std::move(jni_cache)) {}
84 };
85
GetTestBaseDir()86 std::string GetTestBaseDir() { return GetTestTempDir() + "/icing"; }
87
88 // This test is meant to cover all tests relating to
89 // IcingSearchEngine::Optimize.
90 class IcingSearchEngineOptimizeTest : public testing::Test {
91 protected:
SetUp()92 void SetUp() override {
93 if (!IsCfStringTokenization() && !IsReverseJniTokenization()) {
94 // If we've specified using the reverse-JNI method for segmentation (i.e.
95 // not ICU), then we won't have the ICU data file included to set up.
96 // Technically, we could choose to use reverse-JNI for segmentation AND
97 // include an ICU data file, but that seems unlikely and our current BUILD
98 // setup doesn't do this.
99 // File generated via icu_data_file rule in //icing/BUILD.
100 std::string icu_data_file_path =
101 GetTestFilePath("icing/icu.dat");
102 ICING_ASSERT_OK(
103 icu_data_file_helper::SetUpICUDataFile(icu_data_file_path));
104 }
105 filesystem_.CreateDirectoryRecursively(GetTestBaseDir().c_str());
106 }
107
TearDown()108 void TearDown() override {
109 filesystem_.DeleteDirectoryRecursively(GetTestBaseDir().c_str());
110 }
111
filesystem() const112 const Filesystem* filesystem() const { return &filesystem_; }
113
114 private:
115 Filesystem filesystem_;
116 };
117
118 // Non-zero value so we don't override it to be the current time
119 constexpr int64_t kDefaultCreationTimestampMs = 1575492852000;
120
GetDefaultIcingOptions()121 IcingSearchEngineOptions GetDefaultIcingOptions() {
122 IcingSearchEngineOptions icing_options;
123 icing_options.set_base_dir(GetTestBaseDir());
124 return icing_options;
125 }
126
GetDefaultScoringSpec()127 ScoringSpecProto GetDefaultScoringSpec() {
128 ScoringSpecProto scoring_spec;
129 scoring_spec.set_rank_by(ScoringSpecProto::RankingStrategy::DOCUMENT_SCORE);
130 return scoring_spec;
131 }
132
133 // TODO(b/272145329): create SearchSpecBuilder, JoinSpecBuilder,
134 // SearchResultProtoBuilder and ResultProtoBuilder for unit tests and build all
135 // instances by them.
136
TEST_F(IcingSearchEngineOptimizeTest,AllPageTokensShouldBeInvalidatedAfterOptimization)137 TEST_F(IcingSearchEngineOptimizeTest,
138 AllPageTokensShouldBeInvalidatedAfterOptimization) {
139 SchemaProto schema =
140 SchemaBuilder()
141 .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty(
142 PropertyConfigBuilder()
143 .SetName("body")
144 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
145 .SetCardinality(CARDINALITY_REQUIRED)))
146 .Build();
147
148 DocumentProto document1 =
149 DocumentBuilder()
150 .SetKey("namespace", "uri1")
151 .SetSchema("Message")
152 .AddStringProperty("body", "message body one")
153 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
154 .Build();
155 DocumentProto document2 =
156 DocumentBuilder()
157 .SetKey("namespace", "uri2")
158 .SetSchema("Message")
159 .AddStringProperty("body", "message body two")
160 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
161 .Build();
162
163 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
164 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
165 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
166
167 ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk());
168 ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk());
169
170 SearchSpecProto search_spec;
171 search_spec.set_term_match_type(TermMatchType::PREFIX);
172 search_spec.set_query("message");
173
174 ResultSpecProto result_spec;
175 result_spec.set_num_per_page(1);
176
177 // Searches and gets the first page, 1 result
178 SearchResultProto expected_search_result_proto;
179 expected_search_result_proto.mutable_status()->set_code(StatusProto::OK);
180 *expected_search_result_proto.mutable_results()->Add()->mutable_document() =
181 document2;
182 SearchResultProto search_result_proto =
183 icing.Search(search_spec, GetDefaultScoringSpec(), result_spec);
184 EXPECT_THAT(search_result_proto.next_page_token(), Gt(kInvalidNextPageToken));
185 uint64_t next_page_token = search_result_proto.next_page_token();
186 // Since the token is a random number, we don't need to verify
187 expected_search_result_proto.set_next_page_token(next_page_token);
188 EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores(
189 expected_search_result_proto));
190 // Now document1 is still to be fetched.
191
192 OptimizeResultProto optimize_result_proto;
193 optimize_result_proto.mutable_status()->set_code(StatusProto::OK);
194 optimize_result_proto.mutable_status()->set_message("");
195 OptimizeResultProto actual_result = icing.Optimize();
196 actual_result.clear_optimize_stats();
197 ASSERT_THAT(actual_result, EqualsProto(optimize_result_proto));
198
199 // Tries to fetch the second page, no results since all tokens have been
200 // invalidated during Optimize()
201 expected_search_result_proto.clear_results();
202 expected_search_result_proto.clear_next_page_token();
203 search_result_proto = icing.GetNextPage(next_page_token);
204 EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores(
205 expected_search_result_proto));
206 }
207
TEST_F(IcingSearchEngineOptimizeTest,OptimizationShouldRemoveDeletedDocs)208 TEST_F(IcingSearchEngineOptimizeTest, OptimizationShouldRemoveDeletedDocs) {
209 SchemaProto schema =
210 SchemaBuilder()
211 .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty(
212 PropertyConfigBuilder()
213 .SetName("body")
214 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
215 .SetCardinality(CARDINALITY_REQUIRED)))
216 .Build();
217
218 DocumentProto document1 =
219 DocumentBuilder()
220 .SetKey("namespace", "uri1")
221 .SetSchema("Message")
222 .AddStringProperty("body", "message body one")
223 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
224 .Build();
225
226 IcingSearchEngineOptions icing_options = GetDefaultIcingOptions();
227
228 GetResultProto expected_get_result_proto;
229 expected_get_result_proto.mutable_status()->set_code(StatusProto::NOT_FOUND);
230 expected_get_result_proto.mutable_status()->set_message(
231 "Document (namespace, uri1) not found.");
232 {
233 IcingSearchEngine icing(icing_options, GetTestJniCache());
234 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
235 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
236 ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk());
237
238 // Deletes document1
239 ASSERT_THAT(icing.Delete("namespace", "uri1").status(), ProtoIsOk());
240 const std::string document_log_path =
241 icing_options.base_dir() + "/document_dir/" +
242 DocumentLogCreator::GetDocumentLogFilename();
243 int64_t document_log_size_before =
244 filesystem()->GetFileSize(document_log_path.c_str());
245 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
246 int64_t document_log_size_after =
247 filesystem()->GetFileSize(document_log_path.c_str());
248
249 // Validates that document can't be found right after Optimize()
250 EXPECT_THAT(
251 icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()),
252 EqualsProto(expected_get_result_proto));
253 // Validates that document is actually removed from document log
254 EXPECT_THAT(document_log_size_after, Lt(document_log_size_before));
255 } // Destroys IcingSearchEngine to make sure nothing is cached.
256
257 IcingSearchEngine icing(icing_options, GetTestJniCache());
258 EXPECT_THAT(icing.Initialize().status(), ProtoIsOk());
259 EXPECT_THAT(
260 icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()),
261 EqualsProto(expected_get_result_proto));
262 }
263
TEST_F(IcingSearchEngineOptimizeTest,OptimizationShouldDeleteTemporaryDirectory)264 TEST_F(IcingSearchEngineOptimizeTest,
265 OptimizationShouldDeleteTemporaryDirectory) {
266 SchemaProto schema =
267 SchemaBuilder()
268 .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty(
269 PropertyConfigBuilder()
270 .SetName("body")
271 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
272 .SetCardinality(CARDINALITY_REQUIRED)))
273 .Build();
274
275 IcingSearchEngineOptions icing_options = GetDefaultIcingOptions();
276 IcingSearchEngine icing(icing_options, GetTestJniCache());
277 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
278 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
279
280 // Create a tmp dir that will be used in Optimize() to swap files,
281 // this validates that any tmp dirs will be deleted before using.
282 const std::string tmp_dir =
283 icing_options.base_dir() + "/document_dir_optimize_tmp";
284
285 const std::string tmp_file = tmp_dir + "/file";
286 ASSERT_TRUE(filesystem()->CreateDirectory(tmp_dir.c_str()));
287 ScopedFd fd(filesystem()->OpenForWrite(tmp_file.c_str()));
288 ASSERT_TRUE(fd.is_valid());
289 ASSERT_TRUE(filesystem()->Write(fd.get(), "1234", 4));
290 fd.reset();
291
292 EXPECT_THAT(icing.Optimize().status(), ProtoIsOk());
293
294 EXPECT_FALSE(filesystem()->DirectoryExists(tmp_dir.c_str()));
295 EXPECT_FALSE(filesystem()->FileExists(tmp_file.c_str()));
296 }
297
TEST_F(IcingSearchEngineOptimizeTest,GetOptimizeInfoHasCorrectStats)298 TEST_F(IcingSearchEngineOptimizeTest, GetOptimizeInfoHasCorrectStats) {
299 SchemaProto schema =
300 SchemaBuilder()
301 .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty(
302 PropertyConfigBuilder()
303 .SetName("body")
304 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
305 .SetCardinality(CARDINALITY_REQUIRED)))
306 .Build();
307
308 DocumentProto document1 =
309 DocumentBuilder()
310 .SetKey("namespace", "uri1")
311 .SetSchema("Message")
312 .AddStringProperty("body", "message body one")
313 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
314 .Build();
315 DocumentProto document2 = DocumentBuilder()
316 .SetKey("namespace", "uri2")
317 .SetSchema("Message")
318 .AddStringProperty("body", "message body two")
319 .SetCreationTimestampMs(100)
320 .SetTtlMs(500)
321 .Build();
322
323 {
324 auto fake_clock = std::make_unique<FakeClock>();
325 fake_clock->SetSystemTimeMilliseconds(1000);
326
327 TestIcingSearchEngine icing(GetDefaultIcingOptions(),
328 std::make_unique<Filesystem>(),
329 std::make_unique<IcingFilesystem>(),
330 std::move(fake_clock), GetTestJniCache());
331 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
332
333 // Just initialized, nothing is optimizable yet.
334 GetOptimizeInfoResultProto optimize_info = icing.GetOptimizeInfo();
335 EXPECT_THAT(optimize_info.status(), ProtoIsOk());
336 EXPECT_THAT(optimize_info.optimizable_docs(), Eq(0));
337 EXPECT_THAT(optimize_info.estimated_optimizable_bytes(), Eq(0));
338 EXPECT_THAT(optimize_info.time_since_last_optimize_ms(), Eq(0));
339
340 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
341 ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk());
342
343 // Only have active documents, nothing is optimizable yet.
344 optimize_info = icing.GetOptimizeInfo();
345 EXPECT_THAT(optimize_info.status(), ProtoIsOk());
346 EXPECT_THAT(optimize_info.optimizable_docs(), Eq(0));
347 EXPECT_THAT(optimize_info.estimated_optimizable_bytes(), Eq(0));
348 EXPECT_THAT(optimize_info.time_since_last_optimize_ms(), Eq(0));
349
350 // Deletes document1
351 ASSERT_THAT(icing.Delete("namespace", "uri1").status(), ProtoIsOk());
352
353 optimize_info = icing.GetOptimizeInfo();
354 EXPECT_THAT(optimize_info.status(), ProtoIsOk());
355 EXPECT_THAT(optimize_info.optimizable_docs(), Eq(1));
356 EXPECT_THAT(optimize_info.estimated_optimizable_bytes(), Gt(0));
357 EXPECT_THAT(optimize_info.time_since_last_optimize_ms(), Eq(0));
358 int64_t first_estimated_optimizable_bytes =
359 optimize_info.estimated_optimizable_bytes();
360
361 // Add a second document, but it'll be expired since the time (1000) is
362 // greater than the document's creation timestamp (100) + the document's ttl
363 // (500)
364 ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk());
365
366 optimize_info = icing.GetOptimizeInfo();
367 EXPECT_THAT(optimize_info.status(), ProtoIsOk());
368 EXPECT_THAT(optimize_info.optimizable_docs(), Eq(2));
369 EXPECT_THAT(optimize_info.estimated_optimizable_bytes(),
370 Gt(first_estimated_optimizable_bytes));
371 EXPECT_THAT(optimize_info.time_since_last_optimize_ms(), Eq(0));
372
373 // Optimize
374 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
375 }
376
377 {
378 // Recreate with new time
379 auto fake_clock = std::make_unique<FakeClock>();
380 fake_clock->SetSystemTimeMilliseconds(5000);
381
382 TestIcingSearchEngine icing(GetDefaultIcingOptions(),
383 std::make_unique<Filesystem>(),
384 std::make_unique<IcingFilesystem>(),
385 std::move(fake_clock), GetTestJniCache());
386 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
387
388 // Nothing is optimizable now that everything has been optimized away.
389 GetOptimizeInfoResultProto optimize_info = icing.GetOptimizeInfo();
390 EXPECT_THAT(optimize_info.status(), ProtoIsOk());
391 EXPECT_THAT(optimize_info.optimizable_docs(), Eq(0));
392 EXPECT_THAT(optimize_info.estimated_optimizable_bytes(), Eq(0));
393 EXPECT_THAT(optimize_info.time_since_last_optimize_ms(), Eq(4000));
394 }
395 }
396
TEST_F(IcingSearchEngineOptimizeTest,GetAndPutShouldWorkAfterOptimization)397 TEST_F(IcingSearchEngineOptimizeTest, GetAndPutShouldWorkAfterOptimization) {
398 SchemaProto schema =
399 SchemaBuilder()
400 .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty(
401 PropertyConfigBuilder()
402 .SetName("body")
403 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
404 .SetCardinality(CARDINALITY_REQUIRED)))
405 .Build();
406
407 DocumentProto document1 =
408 DocumentBuilder()
409 .SetKey("namespace", "uri1")
410 .SetSchema("Message")
411 .AddStringProperty("body", "message body one")
412 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
413 .Build();
414 DocumentProto document2 =
415 DocumentBuilder()
416 .SetKey("namespace", "uri2")
417 .SetSchema("Message")
418 .AddStringProperty("body", "message body two")
419 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
420 .Build();
421 DocumentProto document3 =
422 DocumentBuilder()
423 .SetKey("namespace", "uri3")
424 .SetSchema("Message")
425 .AddStringProperty("body", "message body three")
426 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
427 .Build();
428 DocumentProto document4 =
429 DocumentBuilder()
430 .SetKey("namespace", "uri4")
431 .SetSchema("Message")
432 .AddStringProperty("body", "message body four")
433 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
434 .Build();
435 DocumentProto document5 =
436 DocumentBuilder()
437 .SetKey("namespace", "uri5")
438 .SetSchema("Message")
439 .AddStringProperty("body", "message body five")
440 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
441 .Build();
442
443 GetResultProto expected_get_result_proto;
444 expected_get_result_proto.mutable_status()->set_code(StatusProto::OK);
445
446 {
447 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
448 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
449 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
450
451 ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk());
452 ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk());
453 ASSERT_THAT(icing.Put(document3).status(), ProtoIsOk());
454 ASSERT_THAT(icing.Delete("namespace", "uri2").status(), ProtoIsOk());
455 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
456
457 // Validates that Get() and Put() are good right after Optimize()
458 *expected_get_result_proto.mutable_document() = document1;
459 EXPECT_THAT(
460 icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()),
461 EqualsProto(expected_get_result_proto));
462 EXPECT_THAT(
463 icing.Get("namespace", "uri2", GetResultSpecProto::default_instance())
464 .status()
465 .code(),
466 Eq(StatusProto::NOT_FOUND));
467 *expected_get_result_proto.mutable_document() = document3;
468 EXPECT_THAT(
469 icing.Get("namespace", "uri3", GetResultSpecProto::default_instance()),
470 EqualsProto(expected_get_result_proto));
471 EXPECT_THAT(icing.Put(document4).status(), ProtoIsOk());
472 } // Destroys IcingSearchEngine to make sure nothing is cached.
473
474 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
475 EXPECT_THAT(icing.Initialize().status(), ProtoIsOk());
476 *expected_get_result_proto.mutable_document() = document1;
477 EXPECT_THAT(
478 icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()),
479 EqualsProto(expected_get_result_proto));
480 EXPECT_THAT(
481 icing.Get("namespace", "uri2", GetResultSpecProto::default_instance())
482 .status()
483 .code(),
484 Eq(StatusProto::NOT_FOUND));
485 *expected_get_result_proto.mutable_document() = document3;
486 EXPECT_THAT(
487 icing.Get("namespace", "uri3", GetResultSpecProto::default_instance()),
488 EqualsProto(expected_get_result_proto));
489 *expected_get_result_proto.mutable_document() = document4;
490 EXPECT_THAT(
491 icing.Get("namespace", "uri4", GetResultSpecProto::default_instance()),
492 EqualsProto(expected_get_result_proto));
493
494 EXPECT_THAT(icing.Put(document5).status(), ProtoIsOk());
495 }
496
TEST_F(IcingSearchEngineOptimizeTest,GetAndPutShouldWorkAfterOptimizationWithEmptyDocuments)497 TEST_F(IcingSearchEngineOptimizeTest,
498 GetAndPutShouldWorkAfterOptimizationWithEmptyDocuments) {
499 SchemaProto schema =
500 SchemaBuilder()
501 .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty(
502 PropertyConfigBuilder()
503 .SetName("body")
504 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
505 .SetCardinality(CARDINALITY_REQUIRED)))
506 .Build();
507
508 DocumentProto empty_document1 =
509 DocumentBuilder()
510 .SetKey("namespace", "uri1")
511 .SetSchema("Message")
512 .AddStringProperty("body", "")
513 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
514 .Build();
515 DocumentProto empty_document2 =
516 DocumentBuilder()
517 .SetKey("namespace", "uri2")
518 .SetSchema("Message")
519 .AddStringProperty("body", "")
520 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
521 .Build();
522 DocumentProto empty_document3 =
523 DocumentBuilder()
524 .SetKey("namespace", "uri3")
525 .SetSchema("Message")
526 .AddStringProperty("body", "")
527 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
528 .Build();
529 GetResultProto expected_get_result_proto;
530 expected_get_result_proto.mutable_status()->set_code(StatusProto::OK);
531
532 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
533 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
534 ASSERT_THAT(icing.SetSchema(std::move(schema)).status(), ProtoIsOk());
535
536 ASSERT_THAT(icing.Put(empty_document1).status(), ProtoIsOk());
537 ASSERT_THAT(icing.Put(empty_document2).status(), ProtoIsOk());
538 ASSERT_THAT(icing.Delete("namespace", "uri2").status(), ProtoIsOk());
539 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
540
541 // Validates that Get() and Put() are good right after Optimize()
542 *expected_get_result_proto.mutable_document() = empty_document1;
543 EXPECT_THAT(
544 icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()),
545 EqualsProto(expected_get_result_proto));
546 EXPECT_THAT(
547 icing.Get("namespace", "uri2", GetResultSpecProto::default_instance())
548 .status()
549 .code(),
550 Eq(StatusProto::NOT_FOUND));
551 EXPECT_THAT(icing.Put(empty_document3).status(), ProtoIsOk());
552 }
553
TEST_F(IcingSearchEngineOptimizeTest,DeleteShouldWorkAfterOptimization)554 TEST_F(IcingSearchEngineOptimizeTest, DeleteShouldWorkAfterOptimization) {
555 SchemaProto schema =
556 SchemaBuilder()
557 .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty(
558 PropertyConfigBuilder()
559 .SetName("body")
560 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
561 .SetCardinality(CARDINALITY_REQUIRED)))
562 .Build();
563
564 DocumentProto document1 =
565 DocumentBuilder()
566 .SetKey("namespace", "uri1")
567 .SetSchema("Message")
568 .AddStringProperty("body", "message body one")
569 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
570 .Build();
571 DocumentProto document2 =
572 DocumentBuilder()
573 .SetKey("namespace", "uri2")
574 .SetSchema("Message")
575 .AddStringProperty("body", "message body two")
576 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
577 .Build();
578
579 {
580 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
581 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
582 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
583 ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk());
584 ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk());
585 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
586
587 // Validates that Delete() works right after Optimize()
588 EXPECT_THAT(icing.Delete("namespace", "uri1").status(), ProtoIsOk());
589
590 GetResultProto expected_get_result_proto;
591 expected_get_result_proto.mutable_status()->set_code(
592 StatusProto::NOT_FOUND);
593 expected_get_result_proto.mutable_status()->set_message(
594 "Document (namespace, uri1) not found.");
595 EXPECT_THAT(
596 icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()),
597 EqualsProto(expected_get_result_proto));
598
599 expected_get_result_proto.mutable_status()->set_code(StatusProto::OK);
600 expected_get_result_proto.mutable_status()->clear_message();
601 *expected_get_result_proto.mutable_document() = document2;
602 EXPECT_THAT(
603 icing.Get("namespace", "uri2", GetResultSpecProto::default_instance()),
604 EqualsProto(expected_get_result_proto));
605 } // Destroys IcingSearchEngine to make sure nothing is cached.
606
607 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
608 EXPECT_THAT(icing.Initialize().status(), ProtoIsOk());
609 EXPECT_THAT(icing.Delete("namespace", "uri2").status(), ProtoIsOk());
610
611 GetResultProto expected_get_result_proto;
612 expected_get_result_proto.mutable_status()->set_code(StatusProto::NOT_FOUND);
613 expected_get_result_proto.mutable_status()->set_message(
614 "Document (namespace, uri1) not found.");
615 EXPECT_THAT(
616 icing.Get("namespace", "uri1", GetResultSpecProto::default_instance()),
617 EqualsProto(expected_get_result_proto));
618
619 expected_get_result_proto.mutable_status()->set_message(
620 "Document (namespace, uri2) not found.");
621 EXPECT_THAT(
622 icing.Get("namespace", "uri2", GetResultSpecProto::default_instance()),
623 EqualsProto(expected_get_result_proto));
624 }
625
TEST_F(IcingSearchEngineOptimizeTest,OptimizationFailureUninitializesIcing)626 TEST_F(IcingSearchEngineOptimizeTest, OptimizationFailureUninitializesIcing) {
627 // Setup filesystem to fail
628 auto mock_filesystem = std::make_unique<MockFilesystem>();
629 bool just_swapped_files = false;
630 auto create_dir_lambda = [this, &just_swapped_files](const char* dir_name) {
631 if (just_swapped_files) {
632 // We should fail the first call immediately after swapping files.
633 just_swapped_files = false;
634 return false;
635 }
636 return filesystem()->CreateDirectoryRecursively(dir_name);
637 };
638 ON_CALL(*mock_filesystem, CreateDirectoryRecursively)
639 .WillByDefault(create_dir_lambda);
640
641 auto swap_lambda = [&just_swapped_files](const char* first_dir,
642 const char* second_dir) {
643 just_swapped_files = true;
644 return false;
645 };
646 IcingSearchEngineOptions options = GetDefaultIcingOptions();
647 ON_CALL(*mock_filesystem, SwapFiles(HasSubstr("document_dir_optimize_tmp"),
648 HasSubstr("document_dir")))
649 .WillByDefault(swap_lambda);
650 TestIcingSearchEngine icing(options, std::move(mock_filesystem),
651 std::make_unique<IcingFilesystem>(),
652 std::make_unique<FakeClock>(), GetTestJniCache());
653 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
654
655 // The mocks should cause an unrecoverable error during Optimize - returning
656 // INTERNAL.
657 ASSERT_THAT(icing.Optimize().status(), ProtoStatusIs(StatusProto::INTERNAL));
658
659 // Ordinary operations should fail safely.
660 SchemaProto simple_schema =
661 SchemaBuilder()
662 .AddType(SchemaTypeConfigBuilder().SetType("type0").AddProperty(
663 PropertyConfigBuilder()
664 .SetName("prop0")
665 .SetDataType(TYPE_STRING)
666 .SetCardinality(CARDINALITY_OPTIONAL)))
667 .Build();
668
669 DocumentProto simple_doc = DocumentBuilder()
670 .SetKey("namespace0", "uri0")
671 .SetSchema("type0")
672 .AddStringProperty("prop0", "foo")
673 .Build();
674
675 SearchSpecProto search_spec;
676 search_spec.set_query("foo");
677 search_spec.set_term_match_type(TermMatchType::EXACT_ONLY);
678 ResultSpecProto result_spec;
679 ScoringSpecProto scoring_spec;
680 scoring_spec.set_rank_by(
681 ScoringSpecProto::RankingStrategy::CREATION_TIMESTAMP);
682
683 EXPECT_THAT(icing.SetSchema(simple_schema).status(),
684 ProtoStatusIs(StatusProto::FAILED_PRECONDITION));
685 EXPECT_THAT(icing.Put(simple_doc).status(),
686 ProtoStatusIs(StatusProto::FAILED_PRECONDITION));
687 EXPECT_THAT(icing
688 .Get(simple_doc.namespace_(), simple_doc.uri(),
689 GetResultSpecProto::default_instance())
690 .status(),
691 ProtoStatusIs(StatusProto::FAILED_PRECONDITION));
692 EXPECT_THAT(icing.Search(search_spec, scoring_spec, result_spec).status(),
693 ProtoStatusIs(StatusProto::FAILED_PRECONDITION));
694
695 // Reset should get icing back to a safe (empty) and working state.
696 EXPECT_THAT(icing.Reset().status(), ProtoIsOk());
697 EXPECT_THAT(icing.SetSchema(simple_schema).status(), ProtoIsOk());
698 EXPECT_THAT(icing.Put(simple_doc).status(), ProtoIsOk());
699 EXPECT_THAT(icing
700 .Get(simple_doc.namespace_(), simple_doc.uri(),
701 GetResultSpecProto::default_instance())
702 .status(),
703 ProtoIsOk());
704 EXPECT_THAT(icing.Search(search_spec, scoring_spec, result_spec).status(),
705 ProtoIsOk());
706 }
707
TEST_F(IcingSearchEngineOptimizeTest,SetSchemaShouldWorkAfterOptimization)708 TEST_F(IcingSearchEngineOptimizeTest, SetSchemaShouldWorkAfterOptimization) {
709 // Creates 3 test schemas
710 SchemaProto schema1 =
711 SchemaBuilder()
712 .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty(
713 PropertyConfigBuilder()
714 .SetName("body")
715 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
716 .SetCardinality(CARDINALITY_REQUIRED)))
717 .Build();
718
719 SchemaProto schema2 = SchemaProto(schema1);
720 *schema2.mutable_types(0)->add_properties() =
721 PropertyConfigBuilder()
722 .SetName("property2")
723 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
724 .SetCardinality(CARDINALITY_OPTIONAL)
725 .Build();
726
727 SchemaProto schema3 = SchemaProto(schema2);
728 *schema3.mutable_types(0)->add_properties() =
729 PropertyConfigBuilder()
730 .SetName("property3")
731 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
732 .SetCardinality(CARDINALITY_OPTIONAL)
733 .Build();
734
735 {
736 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
737 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
738 ASSERT_THAT(icing.SetSchema(schema1).status(), ProtoIsOk());
739 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
740
741 // Validates that SetSchema() works right after Optimize()
742 EXPECT_THAT(icing.SetSchema(schema2).status(), ProtoIsOk());
743 } // Destroys IcingSearchEngine to make sure nothing is cached.
744
745 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
746 EXPECT_THAT(icing.Initialize().status(), ProtoIsOk());
747 EXPECT_THAT(icing.SetSchema(schema3).status(), ProtoIsOk());
748 }
749
TEST_F(IcingSearchEngineOptimizeTest,SearchShouldWorkAfterOptimization)750 TEST_F(IcingSearchEngineOptimizeTest, SearchShouldWorkAfterOptimization) {
751 SchemaProto schema =
752 SchemaBuilder()
753 .AddType(SchemaTypeConfigBuilder()
754 .SetType("Message")
755 .AddProperty(PropertyConfigBuilder()
756 .SetName("body")
757 .SetDataTypeString(TERM_MATCH_PREFIX,
758 TOKENIZER_PLAIN)
759 .SetCardinality(CARDINALITY_REQUIRED))
760 .AddProperty(PropertyConfigBuilder()
761 .SetName("indexableInteger")
762 .SetDataTypeInt64(NUMERIC_MATCH_RANGE)
763 .SetCardinality(CARDINALITY_REQUIRED)))
764 .Build();
765
766 DocumentProto document =
767 DocumentBuilder()
768 .SetKey("namespace", "uri")
769 .SetSchema("Message")
770 .AddStringProperty("body", "message body")
771 .AddInt64Property("indexableInteger", 123)
772 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
773 .Build();
774
775 SearchSpecProto search_spec1;
776 search_spec1.set_term_match_type(TermMatchType::PREFIX);
777 search_spec1.set_query("m");
778
779 SearchSpecProto search_spec2;
780 search_spec2.set_query("indexableInteger == 123");
781 search_spec2.add_enabled_features(std::string(kNumericSearchFeature));
782
783 SearchResultProto expected_search_result_proto;
784 expected_search_result_proto.mutable_status()->set_code(StatusProto::OK);
785 *expected_search_result_proto.mutable_results()->Add()->mutable_document() =
786 document;
787
788 {
789 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
790 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
791 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
792 ASSERT_THAT(icing.Put(document).status(), ProtoIsOk());
793 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
794
795 // Validates that Search() works right after Optimize()
796 // Term search
797 SearchResultProto search_result_proto1 =
798 icing.Search(search_spec1, GetDefaultScoringSpec(),
799 ResultSpecProto::default_instance());
800 EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores(
801 expected_search_result_proto));
802
803 // Numeric (integer) search
804 SearchResultProto search_result_google::protobuf =
805 icing.Search(search_spec2, GetDefaultScoringSpec(),
806 ResultSpecProto::default_instance());
807 EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores(
808 expected_search_result_proto));
809 } // Destroys IcingSearchEngine to make sure nothing is cached.
810
811 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
812 EXPECT_THAT(icing.Initialize().status(), ProtoIsOk());
813
814 // Verify term search
815 SearchResultProto search_result_proto1 =
816 icing.Search(search_spec1, GetDefaultScoringSpec(),
817 ResultSpecProto::default_instance());
818 EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores(
819 expected_search_result_proto));
820
821 // Verify numeric (integer) search
822 SearchResultProto search_result_google::protobuf =
823 icing.Search(search_spec2, GetDefaultScoringSpec(),
824 ResultSpecProto::default_instance());
825 EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores(
826 expected_search_result_proto));
827 }
828
TEST_F(IcingSearchEngineOptimizeTest,JoinShouldWorkAfterOptimizationDeleteParent)829 TEST_F(IcingSearchEngineOptimizeTest,
830 JoinShouldWorkAfterOptimizationDeleteParent) {
831 SchemaProto schema =
832 SchemaBuilder()
833 .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty(
834 PropertyConfigBuilder()
835 .SetName("name")
836 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
837 .SetCardinality(CARDINALITY_REQUIRED)))
838 .AddType(SchemaTypeConfigBuilder()
839 .SetType("Message")
840 .AddProperty(PropertyConfigBuilder()
841 .SetName("body")
842 .SetDataTypeString(TERM_MATCH_PREFIX,
843 TOKENIZER_PLAIN)
844 .SetCardinality(CARDINALITY_REQUIRED))
845 .AddProperty(PropertyConfigBuilder()
846 .SetName("senderQualifiedId")
847 .SetDataTypeJoinableString(
848 JOINABLE_VALUE_TYPE_QUALIFIED_ID)
849 .SetCardinality(CARDINALITY_REQUIRED)))
850 .Build();
851
852 DocumentProto person1 =
853 DocumentBuilder()
854 .SetKey("namespace", "person1")
855 .SetSchema("Person")
856 .AddStringProperty("name", "person one")
857 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
858 .Build();
859 DocumentProto person2 =
860 DocumentBuilder()
861 .SetKey("namespace", "person2")
862 .SetSchema("Person")
863 .AddStringProperty("name", "person two")
864 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
865 .Build();
866
867 DocumentProto message1 =
868 DocumentBuilder()
869 .SetKey("namespace", "message1")
870 .SetSchema("Message")
871 .AddStringProperty("body", "message body one")
872 .AddStringProperty("senderQualifiedId", "namespace#person1")
873 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
874 .Build();
875 DocumentProto message2 =
876 DocumentBuilder()
877 .SetKey("namespace", "message2")
878 .SetSchema("Message")
879 .AddStringProperty("body", "message body two")
880 .AddStringProperty("senderQualifiedId", "namespace#person1")
881 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
882 .Build();
883 DocumentProto message3 =
884 DocumentBuilder()
885 .SetKey("namespace", "message3")
886 .SetSchema("Message")
887 .AddStringProperty("body", "message body three")
888 .AddStringProperty("senderQualifiedId", "namespace#person2")
889 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
890 .Build();
891
892 // Prepare join search spec to join a query for `name:person` with a child
893 // query for `body:message` based on the child's `senderQualifiedId` field.
894 SearchSpecProto search_spec;
895 search_spec.set_term_match_type(TermMatchType::EXACT_ONLY);
896 search_spec.set_query("name:person");
897 JoinSpecProto* join_spec = search_spec.mutable_join_spec();
898 join_spec->set_parent_property_expression(
899 std::string(JoinProcessor::kQualifiedIdExpr));
900 join_spec->set_child_property_expression("senderQualifiedId");
901 join_spec->set_aggregation_scoring_strategy(
902 JoinSpecProto::AggregationScoringStrategy::COUNT);
903 JoinSpecProto::NestedSpecProto* nested_spec =
904 join_spec->mutable_nested_spec();
905 SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec();
906 nested_search_spec->set_term_match_type(TermMatchType::EXACT_ONLY);
907 nested_search_spec->set_query("body:message");
908 *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec();
909 *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance();
910
911 ResultSpecProto result_spec = ResultSpecProto::default_instance();
912 result_spec.set_max_joined_children_per_parent_to_return(
913 std::numeric_limits<int32_t>::max());
914
915 // Person1 is going to be deleted below. Only person2 which is joined with
916 // message3 should match the query.
917 SearchResultProto expected_search_result_proto;
918 expected_search_result_proto.mutable_status()->set_code(StatusProto::OK);
919 SearchResultProto::ResultProto* result_proto =
920 expected_search_result_proto.mutable_results()->Add();
921 *result_proto->mutable_document() = person2;
922 *result_proto->mutable_joined_results()->Add()->mutable_document() = message3;
923
924 {
925 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
926 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
927 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
928 ASSERT_THAT(icing.Put(person1).status(), ProtoIsOk());
929 ASSERT_THAT(icing.Put(person2).status(), ProtoIsOk());
930 ASSERT_THAT(icing.Put(message1).status(), ProtoIsOk());
931 ASSERT_THAT(icing.Put(message2).status(), ProtoIsOk());
932 ASSERT_THAT(icing.Put(message3).status(), ProtoIsOk());
933 // Delete parent document: person1
934 ASSERT_THAT(icing.Delete("namespace", "person1").status(), ProtoIsOk());
935 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
936
937 // Validates that join search query works right after Optimize()
938 SearchResultProto search_result_proto =
939 icing.Search(search_spec, GetDefaultScoringSpec(), result_spec);
940 EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores(
941 expected_search_result_proto));
942 } // Destroys IcingSearchEngine to make sure nothing is cached.
943
944 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
945 EXPECT_THAT(icing.Initialize().status(), ProtoIsOk());
946
947 SearchResultProto search_result_proto =
948 icing.Search(search_spec, GetDefaultScoringSpec(), result_spec);
949 EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores(
950 expected_search_result_proto));
951 }
952
TEST_F(IcingSearchEngineOptimizeTest,JoinShouldWorkAfterOptimizationDeleteChild)953 TEST_F(IcingSearchEngineOptimizeTest,
954 JoinShouldWorkAfterOptimizationDeleteChild) {
955 SchemaProto schema =
956 SchemaBuilder()
957 .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty(
958 PropertyConfigBuilder()
959 .SetName("name")
960 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
961 .SetCardinality(CARDINALITY_REQUIRED)))
962 .AddType(SchemaTypeConfigBuilder()
963 .SetType("Message")
964 .AddProperty(PropertyConfigBuilder()
965 .SetName("body")
966 .SetDataTypeString(TERM_MATCH_PREFIX,
967 TOKENIZER_PLAIN)
968 .SetCardinality(CARDINALITY_REQUIRED))
969 .AddProperty(PropertyConfigBuilder()
970 .SetName("senderQualifiedId")
971 .SetDataTypeJoinableString(
972 JOINABLE_VALUE_TYPE_QUALIFIED_ID)
973 .SetCardinality(CARDINALITY_REQUIRED)))
974 .Build();
975
976 DocumentProto person1 =
977 DocumentBuilder()
978 .SetKey("namespace", "person1")
979 .SetSchema("Person")
980 .AddStringProperty("name", "person one")
981 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
982 .Build();
983 DocumentProto person2 =
984 DocumentBuilder()
985 .SetKey("namespace", "person2")
986 .SetSchema("Person")
987 .AddStringProperty("name", "person two")
988 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
989 .Build();
990
991 DocumentProto message1 =
992 DocumentBuilder()
993 .SetKey("namespace", "message1")
994 .SetSchema("Message")
995 .AddStringProperty("body", "message body one")
996 .AddStringProperty("senderQualifiedId", "namespace#person1")
997 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
998 .Build();
999 DocumentProto message2 =
1000 DocumentBuilder()
1001 .SetKey("namespace", "message2")
1002 .SetSchema("Message")
1003 .AddStringProperty("body", "message body two")
1004 .AddStringProperty("senderQualifiedId", "namespace#person1")
1005 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1006 .Build();
1007 DocumentProto message3 =
1008 DocumentBuilder()
1009 .SetKey("namespace", "message3")
1010 .SetSchema("Message")
1011 .AddStringProperty("body", "message body three")
1012 .AddStringProperty("senderQualifiedId", "namespace#person2")
1013 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1014 .Build();
1015
1016 // Prepare join search spec to join a query for `name:person` with a child
1017 // query for `body:message` based on the child's `senderQualifiedId` field.
1018 SearchSpecProto search_spec;
1019 search_spec.set_term_match_type(TermMatchType::EXACT_ONLY);
1020 search_spec.set_query("name:person");
1021 JoinSpecProto* join_spec = search_spec.mutable_join_spec();
1022 join_spec->set_parent_property_expression(
1023 std::string(JoinProcessor::kQualifiedIdExpr));
1024 join_spec->set_child_property_expression("senderQualifiedId");
1025 join_spec->set_aggregation_scoring_strategy(
1026 JoinSpecProto::AggregationScoringStrategy::COUNT);
1027 JoinSpecProto::NestedSpecProto* nested_spec =
1028 join_spec->mutable_nested_spec();
1029 SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec();
1030 nested_search_spec->set_term_match_type(TermMatchType::EXACT_ONLY);
1031 nested_search_spec->set_query("body:message");
1032 *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec();
1033 *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance();
1034
1035 ResultSpecProto result_spec = ResultSpecProto::default_instance();
1036 result_spec.set_max_joined_children_per_parent_to_return(
1037 std::numeric_limits<int32_t>::max());
1038
1039 // Message1 and message3 are going to be deleted below. Both person1 and
1040 // person2 should be included even though person2 has no child (since we're
1041 // doing left join).
1042 SearchResultProto expected_search_result_proto;
1043 expected_search_result_proto.mutable_status()->set_code(StatusProto::OK);
1044 SearchResultProto::ResultProto* result_proto1 =
1045 expected_search_result_proto.mutable_results()->Add();
1046 *result_proto1->mutable_document() = person1;
1047 *result_proto1->mutable_joined_results()->Add()->mutable_document() =
1048 message2;
1049 SearchResultProto::ResultProto* result_google::protobuf =
1050 expected_search_result_proto.mutable_results()->Add();
1051 *result_google::protobuf->mutable_document() = person2;
1052
1053 {
1054 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
1055 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
1056 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
1057 ASSERT_THAT(icing.Put(person1).status(), ProtoIsOk());
1058 ASSERT_THAT(icing.Put(person2).status(), ProtoIsOk());
1059 ASSERT_THAT(icing.Put(message1).status(), ProtoIsOk());
1060 ASSERT_THAT(icing.Put(message2).status(), ProtoIsOk());
1061 ASSERT_THAT(icing.Put(message3).status(), ProtoIsOk());
1062 // Delete child documents: message1 and message3
1063 ASSERT_THAT(icing.Delete("namespace", "message1").status(), ProtoIsOk());
1064 ASSERT_THAT(icing.Delete("namespace", "message3").status(), ProtoIsOk());
1065 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
1066
1067 // Validates that join search query works right after Optimize()
1068 SearchResultProto search_result_proto =
1069 icing.Search(search_spec, GetDefaultScoringSpec(), result_spec);
1070 EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores(
1071 expected_search_result_proto));
1072 } // Destroys IcingSearchEngine to make sure nothing is cached.
1073
1074 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
1075 EXPECT_THAT(icing.Initialize().status(), ProtoIsOk());
1076
1077 SearchResultProto search_result_proto =
1078 icing.Search(search_spec, GetDefaultScoringSpec(), result_spec);
1079 EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores(
1080 expected_search_result_proto));
1081 }
1082
TEST_F(IcingSearchEngineOptimizeTest,IcingShouldWorkFineIfOptimizationIsAborted)1083 TEST_F(IcingSearchEngineOptimizeTest,
1084 IcingShouldWorkFineIfOptimizationIsAborted) {
1085 SchemaProto schema =
1086 SchemaBuilder()
1087 .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty(
1088 PropertyConfigBuilder()
1089 .SetName("name")
1090 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
1091 .SetCardinality(CARDINALITY_REQUIRED)))
1092 .AddType(SchemaTypeConfigBuilder()
1093 .SetType("Message")
1094 .AddProperty(PropertyConfigBuilder()
1095 .SetName("body")
1096 .SetDataTypeString(TERM_MATCH_PREFIX,
1097 TOKENIZER_PLAIN)
1098 .SetCardinality(CARDINALITY_REQUIRED))
1099 .AddProperty(PropertyConfigBuilder()
1100 .SetName("indexableInteger")
1101 .SetDataTypeInt64(NUMERIC_MATCH_RANGE)
1102 .SetCardinality(CARDINALITY_REQUIRED))
1103 .AddProperty(PropertyConfigBuilder()
1104 .SetName("senderQualifiedId")
1105 .SetDataTypeJoinableString(
1106 JOINABLE_VALUE_TYPE_QUALIFIED_ID)
1107 .SetCardinality(CARDINALITY_REQUIRED)))
1108 .Build();
1109
1110 DocumentProto person =
1111 DocumentBuilder()
1112 .SetKey("namespace", "person")
1113 .SetSchema("Person")
1114 .AddStringProperty("name", "person")
1115 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1116 .Build();
1117
1118 DocumentProto message1 =
1119 DocumentBuilder()
1120 .SetKey("namespace", "message1")
1121 .SetSchema("Message")
1122 .AddStringProperty("body", "message body one")
1123 .AddInt64Property("indexableInteger", 123)
1124 .AddStringProperty("senderQualifiedId", "namespace#person")
1125 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1126 .Build();
1127 {
1128 // Initializes a normal icing to create files needed
1129 IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache());
1130 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
1131 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
1132 ASSERT_THAT(icing.Put(person).status(), ProtoIsOk());
1133 ASSERT_THAT(icing.Put(message1).status(), ProtoIsOk());
1134 }
1135
1136 // Creates a mock filesystem in which DeleteDirectoryRecursively() always
1137 // fails. This will fail IcingSearchEngine::OptimizeDocumentStore() and makes
1138 // it return ABORTED_ERROR.
1139 auto mock_filesystem = std::make_unique<MockFilesystem>();
1140 ON_CALL(*mock_filesystem,
1141 DeleteDirectoryRecursively(HasSubstr("_optimize_tmp")))
1142 .WillByDefault(Return(false));
1143
1144 TestIcingSearchEngine icing(GetDefaultIcingOptions(),
1145 std::move(mock_filesystem),
1146 std::make_unique<IcingFilesystem>(),
1147 std::make_unique<FakeClock>(), GetTestJniCache());
1148 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
1149 EXPECT_THAT(icing.Optimize().status(), ProtoStatusIs(StatusProto::ABORTED));
1150
1151 // Now optimization is aborted, we verify that document-related functions
1152 // still work as expected.
1153
1154 GetResultProto expected_get_result_proto;
1155 expected_get_result_proto.mutable_status()->set_code(StatusProto::OK);
1156 *expected_get_result_proto.mutable_document() = message1;
1157 EXPECT_THAT(icing.Get("namespace", "message1",
1158 GetResultSpecProto::default_instance()),
1159 EqualsProto(expected_get_result_proto));
1160
1161 DocumentProto message2 =
1162 DocumentBuilder()
1163 .SetKey("namespace", "message2")
1164 .SetSchema("Message")
1165 .AddStringProperty("body", "message body two")
1166 .AddInt64Property("indexableInteger", 123)
1167 .AddStringProperty("senderQualifiedId", "namespace#person")
1168 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1169 .Build();
1170
1171 EXPECT_THAT(icing.Put(message2).status(), ProtoIsOk());
1172
1173 SearchResultProto expected_search_result_proto;
1174 expected_search_result_proto.mutable_status()->set_code(StatusProto::OK);
1175 *expected_search_result_proto.mutable_results()->Add()->mutable_document() =
1176 message2;
1177 *expected_search_result_proto.mutable_results()->Add()->mutable_document() =
1178 message1;
1179
1180 // Verify term search
1181 SearchSpecProto search_spec1;
1182 search_spec1.set_query("body:m");
1183 search_spec1.set_term_match_type(TermMatchType::PREFIX);
1184
1185 SearchResultProto search_result_proto1 =
1186 icing.Search(search_spec1, GetDefaultScoringSpec(),
1187 ResultSpecProto::default_instance());
1188 EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores(
1189 expected_search_result_proto));
1190
1191 // Verify numeric (integer) search
1192 SearchSpecProto search_spec2;
1193 search_spec2.set_query("indexableInteger == 123");
1194 search_spec2.add_enabled_features(std::string(kNumericSearchFeature));
1195
1196 SearchResultProto search_result_google::protobuf =
1197 icing.Search(search_spec2, GetDefaultScoringSpec(),
1198 ResultSpecProto::default_instance());
1199 EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores(
1200 expected_search_result_proto));
1201
1202 // Verify join search: join a query for `name:person` with a child query for
1203 // `body:message` based on the child's `senderQualifiedId` field.
1204 SearchSpecProto search_spec3;
1205 search_spec3.set_term_match_type(TermMatchType::EXACT_ONLY);
1206 search_spec3.set_query("name:person");
1207 JoinSpecProto* join_spec = search_spec3.mutable_join_spec();
1208 join_spec->set_parent_property_expression(
1209 std::string(JoinProcessor::kQualifiedIdExpr));
1210 join_spec->set_child_property_expression("senderQualifiedId");
1211 join_spec->set_aggregation_scoring_strategy(
1212 JoinSpecProto::AggregationScoringStrategy::COUNT);
1213 JoinSpecProto::NestedSpecProto* nested_spec =
1214 join_spec->mutable_nested_spec();
1215 SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec();
1216 nested_search_spec->set_term_match_type(TermMatchType::EXACT_ONLY);
1217 nested_search_spec->set_query("body:message");
1218 *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec();
1219 *nested_spec->mutable_result_spec() = ResultSpecProto::default_instance();
1220
1221 ResultSpecProto result_spec3 = ResultSpecProto::default_instance();
1222 result_spec3.set_max_joined_children_per_parent_to_return(
1223 std::numeric_limits<int32_t>::max());
1224
1225 SearchResultProto expected_join_search_result_proto;
1226 expected_join_search_result_proto.mutable_status()->set_code(StatusProto::OK);
1227 SearchResultProto::ResultProto* result_proto =
1228 expected_join_search_result_proto.mutable_results()->Add();
1229 *result_proto->mutable_document() = person;
1230 *result_proto->mutable_joined_results()->Add()->mutable_document() = message2;
1231 *result_proto->mutable_joined_results()->Add()->mutable_document() = message1;
1232
1233 SearchResultProto search_result_proto3 =
1234 icing.Search(search_spec3, GetDefaultScoringSpec(), result_spec3);
1235 EXPECT_THAT(search_result_proto3, EqualsSearchResultIgnoreStatsAndScores(
1236 expected_join_search_result_proto));
1237 }
1238
TEST_F(IcingSearchEngineOptimizeTest,OptimizationShouldRecoverIfFileDirectoriesAreMissing)1239 TEST_F(IcingSearchEngineOptimizeTest,
1240 OptimizationShouldRecoverIfFileDirectoriesAreMissing) {
1241 SchemaProto schema =
1242 SchemaBuilder()
1243 .AddType(SchemaTypeConfigBuilder()
1244 .SetType("Message")
1245 .AddProperty(PropertyConfigBuilder()
1246 .SetName("body")
1247 .SetDataTypeString(TERM_MATCH_PREFIX,
1248 TOKENIZER_PLAIN)
1249 .SetCardinality(CARDINALITY_REQUIRED))
1250 .AddProperty(PropertyConfigBuilder()
1251 .SetName("indexableInteger")
1252 .SetDataTypeInt64(NUMERIC_MATCH_RANGE)
1253 .SetCardinality(CARDINALITY_REQUIRED)))
1254 .Build();
1255
1256 DocumentProto document =
1257 DocumentBuilder()
1258 .SetKey("namespace", "uri1")
1259 .SetSchema("Message")
1260 .AddStringProperty("body", "message body")
1261 .AddInt64Property("indexableInteger", 123)
1262 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1263 .Build();
1264
1265 // Creates a mock filesystem in which SwapFiles() always fails and deletes the
1266 // directories. This will fail IcingSearchEngine::OptimizeDocumentStore().
1267 auto mock_filesystem = std::make_unique<MockFilesystem>();
1268 ON_CALL(*mock_filesystem, SwapFiles(HasSubstr("document_dir_optimize_tmp"),
1269 HasSubstr("document_dir")))
1270 .WillByDefault([this](const char* one, const char* two) {
1271 filesystem()->DeleteDirectoryRecursively(one);
1272 filesystem()->DeleteDirectoryRecursively(two);
1273 return false;
1274 });
1275
1276 TestIcingSearchEngine icing(GetDefaultIcingOptions(),
1277 std::move(mock_filesystem),
1278 std::make_unique<IcingFilesystem>(),
1279 std::make_unique<FakeClock>(), GetTestJniCache());
1280
1281 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
1282 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
1283 ASSERT_THAT(icing.Put(document).status(), ProtoIsOk());
1284
1285 // Optimize() fails due to filesystem error
1286 OptimizeResultProto result = icing.Optimize();
1287 EXPECT_THAT(result.status(), ProtoStatusIs(StatusProto::WARNING_DATA_LOSS));
1288 // Should rebuild the index for data loss.
1289 EXPECT_THAT(result.optimize_stats().index_restoration_mode(),
1290 Eq(OptimizeStatsProto::FULL_INDEX_REBUILD));
1291
1292 // Document is not found because original file directory is missing
1293 GetResultProto expected_get_result_proto;
1294 expected_get_result_proto.mutable_status()->set_code(StatusProto::NOT_FOUND);
1295 expected_get_result_proto.mutable_status()->set_message(
1296 "Document (namespace, uri) not found.");
1297 EXPECT_THAT(
1298 icing.Get("namespace", "uri", GetResultSpecProto::default_instance()),
1299 EqualsProto(expected_get_result_proto));
1300
1301 DocumentProto new_document =
1302 DocumentBuilder()
1303 .SetKey("namespace", "uri2")
1304 .SetSchema("Message")
1305 .AddStringProperty("body", "new body")
1306 .AddInt64Property("indexableInteger", 456)
1307 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1308 .Build();
1309
1310 EXPECT_THAT(icing.Put(new_document).status(), ProtoIsOk());
1311
1312 SearchSpecProto search_spec1;
1313 search_spec1.set_query("m");
1314 search_spec1.set_term_match_type(TermMatchType::PREFIX);
1315
1316 SearchSpecProto search_spec2;
1317 search_spec2.set_query("indexableInteger == 123");
1318 search_spec2.add_enabled_features(std::string(kNumericSearchFeature));
1319
1320 SearchResultProto expected_search_result_proto;
1321 expected_search_result_proto.mutable_status()->set_code(StatusProto::OK);
1322
1323 // Searching old content returns nothing because original file directory is
1324 // missing
1325 // Term search
1326 SearchResultProto search_result_proto1 =
1327 icing.Search(search_spec1, GetDefaultScoringSpec(),
1328 ResultSpecProto::default_instance());
1329 EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores(
1330 expected_search_result_proto));
1331
1332 // Numeric (integer) search
1333 SearchResultProto search_result_google::protobuf =
1334 icing.Search(search_spec2, GetDefaultScoringSpec(),
1335 ResultSpecProto::default_instance());
1336 EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores(
1337 expected_search_result_proto));
1338
1339 // Searching new content returns the new document
1340 *expected_search_result_proto.mutable_results()->Add()->mutable_document() =
1341 new_document;
1342 // Term search
1343 search_spec1.set_query("n");
1344 search_result_proto1 = icing.Search(search_spec1, GetDefaultScoringSpec(),
1345 ResultSpecProto::default_instance());
1346 EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores(
1347 expected_search_result_proto));
1348
1349 // Numeric (integer) search
1350 search_spec2.set_query("indexableInteger == 456");
1351 search_result_google::protobuf = icing.Search(search_spec2, GetDefaultScoringSpec(),
1352 ResultSpecProto::default_instance());
1353 EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores(
1354 expected_search_result_proto));
1355 }
1356
TEST_F(IcingSearchEngineOptimizeTest,OptimizationShouldRecoverIfDataFilesAreMissing)1357 TEST_F(IcingSearchEngineOptimizeTest,
1358 OptimizationShouldRecoverIfDataFilesAreMissing) {
1359 SchemaProto schema =
1360 SchemaBuilder()
1361 .AddType(SchemaTypeConfigBuilder()
1362 .SetType("Message")
1363 .AddProperty(PropertyConfigBuilder()
1364 .SetName("body")
1365 .SetDataTypeString(TERM_MATCH_PREFIX,
1366 TOKENIZER_PLAIN)
1367 .SetCardinality(CARDINALITY_REQUIRED))
1368 .AddProperty(PropertyConfigBuilder()
1369 .SetName("indexableInteger")
1370 .SetDataTypeInt64(NUMERIC_MATCH_RANGE)
1371 .SetCardinality(CARDINALITY_REQUIRED)))
1372 .Build();
1373
1374 DocumentProto document =
1375 DocumentBuilder()
1376 .SetKey("namespace", "uri1")
1377 .SetSchema("Message")
1378 .AddStringProperty("body", "message body")
1379 .AddInt64Property("indexableInteger", 123)
1380 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1381 .Build();
1382
1383 // Creates a mock filesystem in which SwapFiles() always fails and empties the
1384 // directories. This will fail IcingSearchEngine::OptimizeDocumentStore().
1385 auto mock_filesystem = std::make_unique<MockFilesystem>();
1386 ON_CALL(*mock_filesystem, SwapFiles(HasSubstr("document_dir_optimize_tmp"),
1387 HasSubstr("document_dir")))
1388 .WillByDefault([this](const char* one, const char* two) {
1389 filesystem()->DeleteDirectoryRecursively(one);
1390 filesystem()->CreateDirectoryRecursively(one);
1391 filesystem()->DeleteDirectoryRecursively(two);
1392 filesystem()->CreateDirectoryRecursively(two);
1393 return false;
1394 });
1395
1396 TestIcingSearchEngine icing(GetDefaultIcingOptions(),
1397 std::move(mock_filesystem),
1398 std::make_unique<IcingFilesystem>(),
1399 std::make_unique<FakeClock>(), GetTestJniCache());
1400
1401 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
1402 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
1403 ASSERT_THAT(icing.Put(document).status(), ProtoIsOk());
1404
1405 // Optimize() fails due to filesystem error
1406 OptimizeResultProto result = icing.Optimize();
1407 EXPECT_THAT(result.status(), ProtoStatusIs(StatusProto::WARNING_DATA_LOSS));
1408 // Should rebuild the index for data loss.
1409 EXPECT_THAT(result.optimize_stats().index_restoration_mode(),
1410 Eq(OptimizeStatsProto::FULL_INDEX_REBUILD));
1411
1412 // Document is not found because original files are missing
1413 GetResultProto expected_get_result_proto;
1414 expected_get_result_proto.mutable_status()->set_code(StatusProto::NOT_FOUND);
1415 expected_get_result_proto.mutable_status()->set_message(
1416 "Document (namespace, uri) not found.");
1417 EXPECT_THAT(
1418 icing.Get("namespace", "uri", GetResultSpecProto::default_instance()),
1419 EqualsProto(expected_get_result_proto));
1420
1421 DocumentProto new_document =
1422 DocumentBuilder()
1423 .SetKey("namespace", "uri2")
1424 .SetSchema("Message")
1425 .AddStringProperty("body", "new body")
1426 .AddInt64Property("indexableInteger", 456)
1427 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1428 .Build();
1429
1430 EXPECT_THAT(icing.Put(new_document).status(), ProtoIsOk());
1431
1432 SearchSpecProto search_spec1;
1433 search_spec1.set_query("m");
1434 search_spec1.set_term_match_type(TermMatchType::PREFIX);
1435
1436 SearchSpecProto search_spec2;
1437 search_spec2.set_query("indexableInteger == 123");
1438 search_spec2.add_enabled_features(std::string(kNumericSearchFeature));
1439
1440 SearchResultProto expected_search_result_proto;
1441 expected_search_result_proto.mutable_status()->set_code(StatusProto::OK);
1442
1443 // Searching old content returns nothing because original files are missing
1444 // Term search
1445 SearchResultProto search_result_proto1 =
1446 icing.Search(search_spec1, GetDefaultScoringSpec(),
1447 ResultSpecProto::default_instance());
1448 EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores(
1449 expected_search_result_proto));
1450
1451 // Numeric (integer) search
1452 SearchResultProto search_result_google::protobuf =
1453 icing.Search(search_spec2, GetDefaultScoringSpec(),
1454 ResultSpecProto::default_instance());
1455 EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores(
1456 expected_search_result_proto));
1457
1458 // Searching new content returns the new document
1459 *expected_search_result_proto.mutable_results()->Add()->mutable_document() =
1460 new_document;
1461 // Term search
1462 search_spec1.set_query("n");
1463 search_result_proto1 = icing.Search(search_spec1, GetDefaultScoringSpec(),
1464 ResultSpecProto::default_instance());
1465 EXPECT_THAT(search_result_proto1, EqualsSearchResultIgnoreStatsAndScores(
1466 expected_search_result_proto));
1467
1468 // Numeric (integer) search
1469 search_spec2.set_query("indexableInteger == 456");
1470 search_result_google::protobuf = icing.Search(search_spec2, GetDefaultScoringSpec(),
1471 ResultSpecProto::default_instance());
1472 EXPECT_THAT(search_result_google::protobuf, EqualsSearchResultIgnoreStatsAndScores(
1473 expected_search_result_proto));
1474 }
1475
TEST_F(IcingSearchEngineOptimizeTest,OptimizeThresholdTest)1476 TEST_F(IcingSearchEngineOptimizeTest, OptimizeThresholdTest) {
1477 SchemaProto schema =
1478 SchemaBuilder()
1479 .AddType(SchemaTypeConfigBuilder()
1480 .SetType("Message")
1481 .AddProperty(PropertyConfigBuilder()
1482 .SetName("body")
1483 .SetDataTypeString(TERM_MATCH_PREFIX,
1484 TOKENIZER_PLAIN)
1485 .SetCardinality(CARDINALITY_REQUIRED))
1486 .AddProperty(PropertyConfigBuilder()
1487 .SetName("indexableInteger")
1488 .SetDataTypeInt64(NUMERIC_MATCH_RANGE)
1489 .SetCardinality(CARDINALITY_REQUIRED)))
1490 .Build();
1491
1492 DocumentProto document1 =
1493 DocumentBuilder()
1494 .SetKey("namespace", "uri1")
1495 .SetSchema("Message")
1496 .AddStringProperty("body", "message body one")
1497 .AddInt64Property("indexableInteger", 1)
1498 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1499 .Build();
1500 DocumentProto document2 = DocumentBuilder()
1501 .SetKey("namespace", "uri2")
1502 .SetSchema("Message")
1503 .AddStringProperty("body", "message body two")
1504 .AddInt64Property("indexableInteger", 2)
1505 .SetCreationTimestampMs(9000)
1506 .SetTtlMs(500)
1507 .Build();
1508 DocumentProto document3 =
1509 DocumentBuilder()
1510 .SetKey("namespace", "uri3")
1511 .SetSchema("Message")
1512 .AddStringProperty("body", "message body three")
1513 .AddInt64Property("indexableInteger", 3)
1514 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1515 .Build();
1516
1517 auto fake_clock = std::make_unique<FakeClock>();
1518 fake_clock->SetTimerElapsedMilliseconds(5);
1519 fake_clock->SetSystemTimeMilliseconds(10000);
1520 IcingSearchEngineOptions options = GetDefaultIcingOptions();
1521 // Set the threshold to 0.9 to test that the threshold works.
1522 options.set_optimize_rebuild_index_threshold(0.9);
1523 auto icing = std::make_unique<TestIcingSearchEngine>(
1524 options, std::make_unique<Filesystem>(),
1525 std::make_unique<IcingFilesystem>(), std::move(fake_clock),
1526 GetTestJniCache());
1527 ASSERT_THAT(icing->Initialize().status(), ProtoIsOk());
1528 ASSERT_THAT(icing->SetSchema(schema).status(), ProtoIsOk());
1529
1530 // Add three documents.
1531 ASSERT_THAT(icing->Put(document1).status(), ProtoIsOk());
1532 ASSERT_THAT(icing->Put(document2).status(), ProtoIsOk());
1533 ASSERT_THAT(icing->Put(document3).status(), ProtoIsOk());
1534
1535 // Delete the first document.
1536 ASSERT_THAT(icing->Delete(document1.namespace_(), document1.uri()).status(),
1537 ProtoIsOk());
1538 ASSERT_THAT(icing->PersistToDisk(PersistType::FULL).status(), ProtoIsOk());
1539
1540 OptimizeStatsProto expected;
1541 expected.set_latency_ms(5);
1542 expected.set_document_store_optimize_latency_ms(5);
1543 expected.set_index_restoration_latency_ms(5);
1544 expected.set_num_original_documents(3);
1545 expected.set_num_deleted_documents(1);
1546 expected.set_num_expired_documents(1);
1547 expected.set_num_original_namespaces(1);
1548 expected.set_num_deleted_namespaces(0);
1549 expected.set_index_restoration_mode(OptimizeStatsProto::INDEX_TRANSLATION);
1550
1551 // Run Optimize
1552 OptimizeResultProto result = icing->Optimize();
1553 // Depending on how many blocks the documents end up spread across, it's
1554 // possible that Optimize can remove documents without shrinking storage. The
1555 // first Optimize call will also write the OptimizeStatusProto for the first
1556 // time which will take up 1 block. So make sure that before_size is no less
1557 // than after_size - 1 block.
1558 uint32_t page_size = getpagesize();
1559 EXPECT_THAT(result.optimize_stats().storage_size_before(),
1560 Ge(result.optimize_stats().storage_size_after() - page_size));
1561 result.mutable_optimize_stats()->clear_storage_size_before();
1562 result.mutable_optimize_stats()->clear_storage_size_after();
1563 EXPECT_THAT(result.optimize_stats(), EqualsProto(expected));
1564
1565 fake_clock = std::make_unique<FakeClock>();
1566 fake_clock->SetTimerElapsedMilliseconds(5);
1567 fake_clock->SetSystemTimeMilliseconds(20000);
1568 icing = std::make_unique<TestIcingSearchEngine>(
1569 options, std::make_unique<Filesystem>(),
1570 std::make_unique<IcingFilesystem>(), std::move(fake_clock),
1571 GetTestJniCache());
1572 ASSERT_THAT(icing->Initialize().status(), ProtoIsOk());
1573
1574 expected = OptimizeStatsProto();
1575 expected.set_latency_ms(5);
1576 expected.set_document_store_optimize_latency_ms(5);
1577 expected.set_index_restoration_latency_ms(5);
1578 expected.set_num_original_documents(1);
1579 expected.set_num_deleted_documents(0);
1580 expected.set_num_expired_documents(0);
1581 expected.set_num_original_namespaces(1);
1582 expected.set_num_deleted_namespaces(0);
1583 expected.set_time_since_last_optimize_ms(10000);
1584 expected.set_index_restoration_mode(OptimizeStatsProto::INDEX_TRANSLATION);
1585
1586 // Run Optimize
1587 result = icing->Optimize();
1588 EXPECT_THAT(result.optimize_stats().storage_size_before(),
1589 Eq(result.optimize_stats().storage_size_after()));
1590 result.mutable_optimize_stats()->clear_storage_size_before();
1591 result.mutable_optimize_stats()->clear_storage_size_after();
1592 EXPECT_THAT(result.optimize_stats(), EqualsProto(expected));
1593
1594 // Delete the last document.
1595 ASSERT_THAT(icing->Delete(document3.namespace_(), document3.uri()).status(),
1596 ProtoIsOk());
1597
1598 expected = OptimizeStatsProto();
1599 expected.set_latency_ms(5);
1600 expected.set_document_store_optimize_latency_ms(5);
1601 expected.set_index_restoration_latency_ms(5);
1602 expected.set_num_original_documents(1);
1603 expected.set_num_deleted_documents(1);
1604 expected.set_num_expired_documents(0);
1605 expected.set_num_original_namespaces(1);
1606 expected.set_num_deleted_namespaces(1);
1607 expected.set_time_since_last_optimize_ms(0);
1608 // Should rebuild the index since all documents are removed.
1609 expected.set_index_restoration_mode(OptimizeStatsProto::FULL_INDEX_REBUILD);
1610
1611 // Run Optimize
1612 result = icing->Optimize();
1613 EXPECT_THAT(result.optimize_stats().storage_size_before(),
1614 Ge(result.optimize_stats().storage_size_after()));
1615 result.mutable_optimize_stats()->clear_storage_size_before();
1616 result.mutable_optimize_stats()->clear_storage_size_after();
1617 EXPECT_THAT(result.optimize_stats(), EqualsProto(expected));
1618 }
1619
TEST_F(IcingSearchEngineOptimizeTest,OptimizeStatsProtoTest)1620 TEST_F(IcingSearchEngineOptimizeTest, OptimizeStatsProtoTest) {
1621 SchemaProto schema =
1622 SchemaBuilder()
1623 .AddType(SchemaTypeConfigBuilder()
1624 .SetType("Message")
1625 .AddProperty(PropertyConfigBuilder()
1626 .SetName("body")
1627 .SetDataTypeString(TERM_MATCH_PREFIX,
1628 TOKENIZER_PLAIN)
1629 .SetCardinality(CARDINALITY_REQUIRED))
1630 .AddProperty(PropertyConfigBuilder()
1631 .SetName("indexableInteger")
1632 .SetDataTypeInt64(NUMERIC_MATCH_RANGE)
1633 .SetCardinality(CARDINALITY_REQUIRED)))
1634 .Build();
1635
1636 DocumentProto document1 =
1637 DocumentBuilder()
1638 .SetKey("namespace", "uri1")
1639 .SetSchema("Message")
1640 .AddStringProperty("body", "message body one")
1641 .AddInt64Property("indexableInteger", 1)
1642 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1643 .Build();
1644 DocumentProto document2 = DocumentBuilder()
1645 .SetKey("namespace", "uri2")
1646 .SetSchema("Message")
1647 .AddStringProperty("body", "message body two")
1648 .AddInt64Property("indexableInteger", 2)
1649 .SetCreationTimestampMs(9000)
1650 .SetTtlMs(500)
1651 .Build();
1652 DocumentProto document3 =
1653 DocumentBuilder()
1654 .SetKey("namespace", "uri3")
1655 .SetSchema("Message")
1656 .AddStringProperty("body", "message body three")
1657 .AddInt64Property("indexableInteger", 3)
1658 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1659 .Build();
1660
1661 auto fake_clock = std::make_unique<FakeClock>();
1662 fake_clock->SetTimerElapsedMilliseconds(5);
1663 fake_clock->SetSystemTimeMilliseconds(10000);
1664 // Use the default Icing options, so that a change to the default value will
1665 // require updating this test.
1666 auto icing = std::make_unique<TestIcingSearchEngine>(
1667 GetDefaultIcingOptions(), std::make_unique<Filesystem>(),
1668 std::make_unique<IcingFilesystem>(), std::move(fake_clock),
1669 GetTestJniCache());
1670 ASSERT_THAT(icing->Initialize().status(), ProtoIsOk());
1671 ASSERT_THAT(icing->SetSchema(schema).status(), ProtoIsOk());
1672
1673 // Add three documents.
1674 ASSERT_THAT(icing->Put(document1).status(), ProtoIsOk());
1675 ASSERT_THAT(icing->Put(document2).status(), ProtoIsOk());
1676 ASSERT_THAT(icing->Put(document3).status(), ProtoIsOk());
1677
1678 // Delete the first document.
1679 ASSERT_THAT(icing->Delete(document1.namespace_(), document1.uri()).status(),
1680 ProtoIsOk());
1681 ASSERT_THAT(icing->PersistToDisk(PersistType::FULL).status(), ProtoIsOk());
1682
1683 OptimizeStatsProto expected;
1684 expected.set_latency_ms(5);
1685 expected.set_document_store_optimize_latency_ms(5);
1686 expected.set_index_restoration_latency_ms(5);
1687 expected.set_num_original_documents(3);
1688 expected.set_num_deleted_documents(1);
1689 expected.set_num_expired_documents(1);
1690 expected.set_num_original_namespaces(1);
1691 expected.set_num_deleted_namespaces(0);
1692 expected.set_index_restoration_mode(OptimizeStatsProto::FULL_INDEX_REBUILD);
1693
1694 // Run Optimize
1695 OptimizeResultProto result = icing->Optimize();
1696 // Depending on how many blocks the documents end up spread across, it's
1697 // possible that Optimize can remove documents without shrinking storage. The
1698 // first Optimize call will also write the OptimizeStatusProto for the first
1699 // time which will take up 1 block. So make sure that before_size is no less
1700 // than after_size - 1 block.
1701 uint32_t page_size = getpagesize();
1702 EXPECT_THAT(result.optimize_stats().storage_size_before(),
1703 Ge(result.optimize_stats().storage_size_after() - page_size));
1704 result.mutable_optimize_stats()->clear_storage_size_before();
1705 result.mutable_optimize_stats()->clear_storage_size_after();
1706 EXPECT_THAT(result.optimize_stats(), EqualsProto(expected));
1707
1708 fake_clock = std::make_unique<FakeClock>();
1709 fake_clock->SetTimerElapsedMilliseconds(5);
1710 fake_clock->SetSystemTimeMilliseconds(20000);
1711 // Use the default Icing options, so that a change to the default value will
1712 // require updating this test.
1713 icing = std::make_unique<TestIcingSearchEngine>(
1714 GetDefaultIcingOptions(), std::make_unique<Filesystem>(),
1715 std::make_unique<IcingFilesystem>(), std::move(fake_clock),
1716 GetTestJniCache());
1717 ASSERT_THAT(icing->Initialize().status(), ProtoIsOk());
1718
1719 expected = OptimizeStatsProto();
1720 expected.set_latency_ms(5);
1721 expected.set_document_store_optimize_latency_ms(5);
1722 expected.set_index_restoration_latency_ms(5);
1723 expected.set_num_original_documents(1);
1724 expected.set_num_deleted_documents(0);
1725 expected.set_num_expired_documents(0);
1726 expected.set_num_original_namespaces(1);
1727 expected.set_num_deleted_namespaces(0);
1728 expected.set_time_since_last_optimize_ms(10000);
1729 expected.set_index_restoration_mode(OptimizeStatsProto::FULL_INDEX_REBUILD);
1730
1731 // Run Optimize
1732 result = icing->Optimize();
1733 EXPECT_THAT(result.optimize_stats().storage_size_before(),
1734 Eq(result.optimize_stats().storage_size_after()));
1735 result.mutable_optimize_stats()->clear_storage_size_before();
1736 result.mutable_optimize_stats()->clear_storage_size_after();
1737 EXPECT_THAT(result.optimize_stats(), EqualsProto(expected));
1738
1739 // Delete the last document.
1740 ASSERT_THAT(icing->Delete(document3.namespace_(), document3.uri()).status(),
1741 ProtoIsOk());
1742
1743 expected = OptimizeStatsProto();
1744 expected.set_latency_ms(5);
1745 expected.set_document_store_optimize_latency_ms(5);
1746 expected.set_index_restoration_latency_ms(5);
1747 expected.set_num_original_documents(1);
1748 expected.set_num_deleted_documents(1);
1749 expected.set_num_expired_documents(0);
1750 expected.set_num_original_namespaces(1);
1751 expected.set_num_deleted_namespaces(1);
1752 expected.set_time_since_last_optimize_ms(0);
1753 expected.set_index_restoration_mode(OptimizeStatsProto::FULL_INDEX_REBUILD);
1754
1755 // Run Optimize
1756 result = icing->Optimize();
1757 EXPECT_THAT(result.optimize_stats().storage_size_before(),
1758 Ge(result.optimize_stats().storage_size_after()));
1759 result.mutable_optimize_stats()->clear_storage_size_before();
1760 result.mutable_optimize_stats()->clear_storage_size_after();
1761 EXPECT_THAT(result.optimize_stats(), EqualsProto(expected));
1762 }
1763
TEST_F(IcingSearchEngineOptimizeTest,OptimizationRewritesDocsWithNewCompressionLevel)1764 TEST_F(IcingSearchEngineOptimizeTest,
1765 OptimizationRewritesDocsWithNewCompressionLevel) {
1766 SchemaProto schema =
1767 SchemaBuilder()
1768 .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty(
1769 PropertyConfigBuilder()
1770 .SetName("body")
1771 .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)
1772 .SetCardinality(CARDINALITY_REQUIRED)))
1773 .Build();
1774 DocumentProto document1 =
1775 DocumentBuilder()
1776 .SetKey("namespace", "uri1")
1777 .SetSchema("Message")
1778 .AddStringProperty("body", "message body one")
1779 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1780 .Build();
1781 DocumentProto document2 =
1782 DocumentBuilder()
1783 .SetKey("namespace", "uri2")
1784 .SetSchema("Message")
1785 .AddStringProperty("body", "message body two")
1786 .SetCreationTimestampMs(kDefaultCreationTimestampMs)
1787 .Build();
1788 IcingSearchEngineOptions icing_options = GetDefaultIcingOptions();
1789 icing_options.set_compression_level(3);
1790 int64_t document_log_size_compression_3;
1791 int64_t document_log_size_after_opti_no_compression;
1792 int64_t document_log_size_after_opti_compression_3;
1793 const std::string document_log_path =
1794 icing_options.base_dir() + "/document_dir/" +
1795 DocumentLogCreator::GetDocumentLogFilename();
1796 {
1797 IcingSearchEngine icing(icing_options, GetTestJniCache());
1798 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
1799 ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk());
1800 ASSERT_THAT(icing.Put(document1).status(), ProtoIsOk());
1801 ASSERT_THAT(icing.Put(document2).status(), ProtoIsOk());
1802 ASSERT_THAT(icing.PersistToDisk(PersistType::FULL).status(), ProtoIsOk());
1803 document_log_size_compression_3 =
1804 filesystem()->GetFileSize(document_log_path.c_str());
1805 } // Destroys IcingSearchEngine to make sure nothing is cached.
1806
1807 // Turn off compression
1808 icing_options.set_compression_level(0);
1809
1810 {
1811 IcingSearchEngine icing(icing_options, GetTestJniCache());
1812 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
1813 // Document log size is the same even after reopening with a different
1814 // compression level
1815 ASSERT_EQ(document_log_size_compression_3,
1816 filesystem()->GetFileSize(document_log_path.c_str()));
1817 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
1818 document_log_size_after_opti_no_compression =
1819 filesystem()->GetFileSize(document_log_path.c_str());
1820 // Document log size is larger after optimizing since optimizing rewrites
1821 // with the new compression level which is 0 or none
1822 ASSERT_GT(document_log_size_after_opti_no_compression,
1823 document_log_size_compression_3);
1824 }
1825
1826 // Restore the original compression level
1827 icing_options.set_compression_level(3);
1828
1829 {
1830 IcingSearchEngine icing(icing_options, GetTestJniCache());
1831 ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
1832 // Document log size is the same even after reopening with a different
1833 // compression level
1834 ASSERT_EQ(document_log_size_after_opti_no_compression,
1835 filesystem()->GetFileSize(document_log_path.c_str()));
1836 ASSERT_THAT(icing.Optimize().status(), ProtoIsOk());
1837 document_log_size_after_opti_compression_3 =
1838 filesystem()->GetFileSize(document_log_path.c_str());
1839 // Document log size should be the same as it was originally
1840 ASSERT_EQ(document_log_size_after_opti_compression_3,
1841 document_log_size_compression_3);
1842 }
1843 }
1844
1845 } // namespace
1846 } // namespace lib
1847 } // namespace icing
1848