1 // Copyright (C) 2023 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #ifndef ICING_JOIN_QUALIFIED_ID_JOIN_INDEX_H_ 16 #define ICING_JOIN_QUALIFIED_ID_JOIN_INDEX_H_ 17 18 #include <cstdint> 19 #include <memory> 20 #include <string> 21 #include <string_view> 22 #include <utility> 23 #include <vector> 24 25 #include "icing/text_classifier/lib3/utils/base/status.h" 26 #include "icing/text_classifier/lib3/utils/base/statusor.h" 27 #include "icing/file/filesystem.h" 28 #include "icing/file/persistent-storage.h" 29 #include "icing/join/doc-join-info.h" 30 #include "icing/join/document-id-to-join-info.h" 31 #include "icing/schema/joinable-property.h" 32 #include "icing/store/document-filter-data.h" 33 #include "icing/store/document-id.h" 34 #include "icing/store/namespace-fingerprint-identifier.h" 35 #include "icing/store/namespace-id.h" 36 #include "icing/util/crc32.h" 37 38 namespace icing { 39 namespace lib { 40 41 // QualifiedIdJoinIndex: an abstract class to maintain data for qualified id 42 // joining. 43 class QualifiedIdJoinIndex : public PersistentStorage { 44 public: 45 class JoinDataIteratorBase { 46 public: 47 virtual ~JoinDataIteratorBase() = default; 48 49 virtual libtextclassifier3::Status Advance() = 0; 50 51 virtual const DocumentIdToJoinInfo<NamespaceFingerprintIdentifier>& 52 GetCurrent() const = 0; 53 }; 54 55 static constexpr WorkingPathType kWorkingPathType = 56 WorkingPathType::kDirectory; 57 58 // Deletes QualifiedIdJoinIndex under working_path. 59 // 60 // Returns: 61 // - OK on success 62 // - INTERNAL_ERROR on I/O error Discard(const Filesystem & filesystem,const std::string & working_path)63 static libtextclassifier3::Status Discard(const Filesystem& filesystem, 64 const std::string& working_path) { 65 return PersistentStorage::Discard(filesystem, working_path, 66 kWorkingPathType); 67 } 68 69 virtual ~QualifiedIdJoinIndex() override = default; 70 71 // (v1 only) Puts a new data into index: DocJoinInfo (DocumentId, 72 // JoinablePropertyId) references to ref_qualified_id_str (the identifier of 73 // another document). 74 // 75 // REQUIRES: ref_qualified_id_str contains no '\0'. 76 // 77 // Returns: 78 // - OK on success 79 // - INVALID_ARGUMENT_ERROR if doc_join_info is invalid 80 // - Any KeyMapper errors 81 virtual libtextclassifier3::Status Put( 82 const DocJoinInfo& doc_join_info, 83 std::string_view ref_qualified_id_str) = 0; 84 85 // (v2 only) Puts a list of referenced NamespaceFingerprintIdentifier into 86 // index, given the DocumentId, SchemaTypeId and JoinablePropertyId. 87 // 88 // Returns: 89 // - OK on success 90 // - INVALID_ARGUMENT_ERROR if schema_type_id, joinable_property_id, or 91 // document_id is invalid 92 // - Any KeyMapper/FlashIndexStorage errors 93 virtual libtextclassifier3::Status Put( 94 SchemaTypeId schema_type_id, JoinablePropertyId joinable_property_id, 95 DocumentId document_id, 96 std::vector<NamespaceFingerprintIdentifier>&& 97 ref_namespace_fingerprint_ids) = 0; 98 99 // (v1 only) Gets the referenced document's qualified id string by 100 // DocJoinInfo. 101 // 102 // Returns: 103 // - A qualified id string referenced by the given DocJoinInfo (DocumentId, 104 // JoinablePropertyId) on success 105 // - INVALID_ARGUMENT_ERROR if doc_join_info is invalid 106 // - NOT_FOUND_ERROR if doc_join_info doesn't exist 107 // - Any KeyMapper errors 108 virtual libtextclassifier3::StatusOr<std::string_view> Get( 109 const DocJoinInfo& doc_join_info) const = 0; 110 111 // (v2 only) Returns a JoinDataIterator for iterating through all join data of 112 // the specified (schema_type_id, joinable_property_id). 113 // 114 // Returns: 115 // - On success: a JoinDataIterator 116 // - INVALID_ARGUMENT_ERROR if schema_type_id or joinable_property_id is 117 // invalid 118 // - Any KeyMapper/FlashIndexStorage errors 119 virtual libtextclassifier3::StatusOr<std::unique_ptr<JoinDataIteratorBase>> 120 GetIterator(SchemaTypeId schema_type_id, 121 JoinablePropertyId joinable_property_id) const = 0; 122 123 // Reduces internal file sizes by reclaiming space and ids of deleted 124 // documents. Qualified id type joinable index will convert all entries to the 125 // new document ids. 126 // 127 // - document_id_old_to_new: a map for converting old document id to new 128 // document id. 129 // - namespace_id_old_to_new: a map for converting old namespace id to new 130 // namespace id. 131 // - new_last_added_document_id: will be used to update the last added 132 // document id in the qualified id type joinable 133 // index. 134 // 135 // Returns: 136 // - OK on success 137 // - INTERNAL_ERROR on I/O error. This could potentially leave the index in 138 // an invalid state and the caller should handle it properly (e.g. discard 139 // and rebuild) 140 virtual libtextclassifier3::Status Optimize( 141 const std::vector<DocumentId>& document_id_old_to_new, 142 const std::vector<NamespaceId>& namespace_id_old_to_new, 143 DocumentId new_last_added_document_id) = 0; 144 145 // Clears all data and set last_added_document_id to kInvalidDocumentId. 146 // 147 // Returns: 148 // - OK on success 149 // - INTERNAL_ERROR on I/O error 150 virtual libtextclassifier3::Status Clear() = 0; 151 152 virtual bool is_v2() const = 0; 153 154 virtual int32_t size() const = 0; 155 156 virtual bool empty() const = 0; 157 158 virtual DocumentId last_added_document_id() const = 0; 159 160 virtual void set_last_added_document_id(DocumentId document_id) = 0; 161 162 protected: QualifiedIdJoinIndex(const Filesystem & filesystem,std::string && working_path)163 explicit QualifiedIdJoinIndex(const Filesystem& filesystem, 164 std::string&& working_path) 165 : PersistentStorage(filesystem, std::move(working_path), 166 kWorkingPathType) {} 167 168 virtual libtextclassifier3::Status PersistStoragesToDisk( 169 bool force) override = 0; 170 171 virtual libtextclassifier3::Status PersistMetadataToDisk( 172 bool force) override = 0; 173 174 virtual libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum( 175 bool force) override = 0; 176 177 virtual libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum( 178 bool force) override = 0; 179 180 virtual Crcs& crcs() override = 0; 181 virtual const Crcs& crcs() const override = 0; 182 }; 183 184 } // namespace lib 185 } // namespace icing 186 187 #endif // ICING_JOIN_QUALIFIED_ID_JOIN_INDEX_H_ 188