1 // Copyright (C) 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #ifndef ICING_SCHEMA_SECTION_H_
16 #define ICING_SCHEMA_SECTION_H_
17
18 #include <cstdint>
19 #include <string>
20 #include <string_view>
21 #include <utility>
22 #include <vector>
23
24 #include "icing/proto/document.pb.h"
25 #include "icing/proto/schema.pb.h"
26 #include "icing/proto/term.pb.h"
27
28 namespace icing {
29 namespace lib {
30
31 using SectionId = int8_t;
32 // 6 bits for 64 values.
33 inline constexpr int kSectionIdBits = 6;
34 inline constexpr SectionId kTotalNumSections = (1 << kSectionIdBits);
35 inline constexpr SectionId kInvalidSectionId = kTotalNumSections;
36 inline constexpr SectionId kMaxSectionId = kTotalNumSections - 1;
37 // Prior versions of Icing only supported 16 indexed properties.
38 inline constexpr SectionId kOldTotalNumSections = 16;
39 inline constexpr SectionId kMinSectionId = 0;
IsSectionIdValid(SectionId section_id)40 constexpr bool IsSectionIdValid(SectionId section_id) {
41 return section_id >= kMinSectionId && section_id <= kMaxSectionId;
42 }
43
44 using SectionIdMask = int64_t;
45 inline constexpr SectionIdMask kSectionIdMaskAll = ~SectionIdMask{0};
46 inline constexpr SectionIdMask kSectionIdMaskNone = SectionIdMask{0};
47
48 static_assert(kSectionIdBits < 8 * sizeof(SectionId),
49 "Cannot exhaust all bits of SectionId since it is a signed "
50 "integer and the most significant bit should be preserved.");
51
52 static_assert(
53 kMaxSectionId < 8 * sizeof(SectionIdMask),
54 "SectionIdMask is not large enough to represent all section values!");
55
56 struct SectionMetadata {
57 // Dot-joined property names, representing the location of section inside an
58 // document. E.g. "property1.property2"
59 std::string path;
60
61 // A unique id of property within a type config
62 SectionId id;
63
64 // Indexable data type of this section. E.g. STRING, INT64.
65 PropertyConfigProto::DataType::Code data_type;
66
67 // How strings should be tokenized. It is invalid for a string section
68 // (data_type == 'STRING') to have tokenizer == 'NONE'.
69 StringIndexingConfig::TokenizerType::Code tokenizer;
70
71 // How tokens in a string section should be matched.
72 //
73 // TermMatchType::UNKNOWN:
74 // Terms will not match anything
75 //
76 // TermMatchType::PREFIX:
77 // Terms will be stored as a prefix match, "fool" matches "foo" and "fool"
78 //
79 // TermMatchType::EXACT_ONLY:
80 // Terms will be only stored as an exact match, "fool" only matches "fool"
81 TermMatchType::Code term_match_type = TermMatchType::UNKNOWN;
82
83 // How tokens in a numeric section should be matched.
84 //
85 // NumericMatchType::UNKNOWN:
86 // Contents will not match anything. It is invalid for a numeric section
87 // (data_type == 'INT64') to have numeric_match_type == 'UNKNOWN'.
88 //
89 // NumericMatchType::RANGE:
90 // Contents will be matched by a range query.
91 IntegerIndexingConfig::NumericMatchType::Code numeric_match_type;
92
93 // How vectors in a vector section should be indexed.
94 //
95 // EmbeddingIndexingType::UNKNOWN:
96 // Contents will not be indexed. It is invalid for a vector section
97 // (data_type == 'VECTOR') to have embedding_indexing_type == 'UNKNOWN'.
98 //
99 // EmbeddingIndexingType::LINEAR_SEARCH:
100 // Contents will be indexed for linear search.
101 EmbeddingIndexingConfig::EmbeddingIndexingType::Code embedding_indexing_type;
102
SectionMetadataSectionMetadata103 explicit SectionMetadata(
104 SectionId id_in, PropertyConfigProto::DataType::Code data_type_in,
105 StringIndexingConfig::TokenizerType::Code tokenizer,
106 TermMatchType::Code term_match_type_in,
107 IntegerIndexingConfig::NumericMatchType::Code numeric_match_type_in,
108 EmbeddingIndexingConfig::EmbeddingIndexingType::Code
109 embedding_indexing_type_in,
110 std::string&& path_in)
111 : path(std::move(path_in)),
112 id(id_in),
113 data_type(data_type_in),
114 tokenizer(tokenizer),
115 term_match_type(term_match_type_in),
116 numeric_match_type(numeric_match_type_in),
117 embedding_indexing_type(embedding_indexing_type_in) {}
118
119 SectionMetadata(const SectionMetadata& other) = default;
120 SectionMetadata& operator=(const SectionMetadata& other) = default;
121
122 SectionMetadata(SectionMetadata&& other) = default;
123 SectionMetadata& operator=(SectionMetadata&& other) = default;
124
125 bool operator==(const SectionMetadata& rhs) const {
126 return path == rhs.path && id == rhs.id && data_type == rhs.data_type &&
127 tokenizer == rhs.tokenizer &&
128 term_match_type == rhs.term_match_type &&
129 numeric_match_type == rhs.numeric_match_type;
130 }
131 };
132
133 // Section is an icing internal concept similar to document property but with
134 // extra metadata. The content can be a value or the combination of repeated
135 // values of a property, and the type of content is specified by template.
136 //
137 // Current supported types:
138 // - std::string_view (PropertyConfigProto::DataType::STRING)
139 // - int64_t (PropertyConfigProto::DataType::INT64)
140 template <typename T>
141 struct Section {
142 SectionMetadata metadata;
143 std::vector<T> content;
144
SectionSection145 explicit Section(SectionMetadata&& metadata_in, std::vector<T>&& content_in)
146 : metadata(std::move(metadata_in)), content(std::move(content_in)) {}
147
data_typeSection148 PropertyConfigProto::DataType::Code data_type() const {
149 return metadata.data_type;
150 }
151 };
152
153 // Groups of different type sections. Callers can access sections with types
154 // they want and avoid going through non-desired ones.
155 //
156 // REQUIRES: lifecycle of the property must be longer than this object, since we
157 // use std::string_view for extracting its string_values.
158 struct SectionGroup {
159 std::vector<Section<std::string_view>> string_sections;
160 std::vector<Section<int64_t>> integer_sections;
161 std::vector<Section<PropertyProto::VectorProto>> vector_sections;
162 };
163
164 } // namespace lib
165 } // namespace icing
166
167 #endif // ICING_SCHEMA_SECTION_H_
168