1 // Copyright (C) 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "icing/util/document-validator.h"
16
17 #include <cstdint>
18 #include <string>
19 #include <string_view>
20 #include <unordered_map>
21 #include <unordered_set>
22 #include <utility>
23
24 #include "icing/text_classifier/lib3/utils/base/status.h"
25 #include "icing/text_classifier/lib3/utils/base/statusor.h"
26 #include "icing/absl_ports/canonical_errors.h"
27 #include "icing/absl_ports/str_cat.h"
28 #include "icing/legacy/core/icing-string-util.h"
29 #include "icing/proto/document.pb.h"
30 #include "icing/proto/schema.pb.h"
31 #include "icing/schema/schema-store.h"
32 #include "icing/schema/schema-util.h"
33 #include "icing/store/document-filter-data.h"
34 #include "icing/util/logging.h"
35 #include "icing/util/status-macros.h"
36
37 namespace icing {
38 namespace lib {
39
40 using PropertyConfigMap =
41 std::unordered_map<std::string_view, const PropertyConfigProto*>;
42
DocumentValidator(const SchemaStore * schema_store)43 DocumentValidator::DocumentValidator(const SchemaStore* schema_store)
44 : schema_store_(schema_store) {}
45
Validate(const DocumentProto & document,int depth)46 libtextclassifier3::Status DocumentValidator::Validate(
47 const DocumentProto& document, int depth) {
48 if (document.namespace_().empty()) {
49 return absl_ports::InvalidArgumentError("Field 'namespace' is empty.");
50 }
51
52 // Only require a non-empty uri on top-level documents.
53 if (depth == 0 && document.uri().empty()) {
54 return absl_ports::InvalidArgumentError("Field 'uri' is empty.");
55 }
56
57 if (document.schema().empty()) {
58 return absl_ports::InvalidArgumentError(
59 absl_ports::StrCat("Field 'schema' is empty for key: (",
60 document.namespace_(), ", ", document.uri(), ")."));
61 }
62
63 if (document.score() < 0) {
64 return absl_ports::InvalidArgumentError("Field 'score' is negative.");
65 }
66
67 if (document.creation_timestamp_ms() < 0) {
68 return absl_ports::InvalidArgumentError(
69 "Field 'creation_timestamp_ms' is negative.");
70 }
71
72 if (document.ttl_ms() < 0) {
73 return absl_ports::InvalidArgumentError("Field 'ttl_ms' is negative.");
74 }
75
76 // TODO(b/144458732): Implement a more robust version of
77 // ICING_ASSIGN_OR_RETURN that can support error logging.
78 auto type_config_or = schema_store_->GetSchemaTypeConfig(document.schema());
79 if (!type_config_or.ok()) {
80 ICING_LOG(ERROR) << type_config_or.status().error_message()
81 << "Error while validating document ("
82 << document.namespace_() << ", " << document.uri() << ")";
83 return type_config_or.status();
84 }
85 const SchemaTypeConfigProto* type_config =
86 std::move(type_config_or).ValueOrDie();
87
88 int32_t num_required_properties_actual = 0;
89 SchemaUtil::ParsedPropertyConfigs parsed_property_configs =
90 SchemaUtil::ParsePropertyConfigs(*type_config);
91 std::unordered_set<std::string_view> unique_properties;
92
93 for (const PropertyProto& property : document.properties()) {
94 if (property.name().empty()) {
95 return absl_ports::InvalidArgumentError(absl_ports::StrCat(
96 "Field 'name' is empty in PropertyProto for key: (",
97 document.namespace_(), ", ", document.uri(), ")."));
98 }
99
100 if (!unique_properties.insert(property.name()).second) {
101 // Failed to insert because of duplicate property name
102 return absl_ports::AlreadyExistsError(absl_ports::StrCat(
103 "Property name '", property.name(), "' already exists for key: (",
104 document.namespace_(), ", ", document.uri(), ")."));
105 }
106
107 const auto& property_iter =
108 parsed_property_configs.property_config_map.find(property.name());
109 if (property_iter == parsed_property_configs.property_config_map.end()) {
110 return absl_ports::NotFoundError(absl_ports::StrCat(
111 "Property config '", property.name(), "' not found for key: (",
112 document.namespace_(), ", ", document.uri(),
113 ") of type: ", document.schema(), "."));
114 }
115 const PropertyConfigProto& property_config = *property_iter->second;
116
117 // Get the property value size according to data type.
118 int value_size = 0;
119 if (property_config.data_type() == PropertyConfigProto::DataType::STRING) {
120 value_size = property.string_values_size();
121 } else if (property_config.data_type() ==
122 PropertyConfigProto::DataType::INT64) {
123 value_size = property.int64_values_size();
124 } else if (property_config.data_type() ==
125 PropertyConfigProto::DataType::DOUBLE) {
126 value_size = property.double_values_size();
127 } else if (property_config.data_type() ==
128 PropertyConfigProto::DataType::BOOLEAN) {
129 value_size = property.boolean_values_size();
130 } else if (property_config.data_type() ==
131 PropertyConfigProto::DataType::BYTES) {
132 value_size = property.bytes_values_size();
133 } else if (property_config.data_type() ==
134 PropertyConfigProto::DataType::DOCUMENT) {
135 value_size = property.document_values_size();
136 } else if (property_config.data_type() ==
137 PropertyConfigProto::DataType::VECTOR) {
138 value_size = property.vector_values_size();
139 for (const PropertyProto::VectorProto& vector_value :
140 property.vector_values()) {
141 if (vector_value.values_size() == 0) {
142 return absl_ports::InvalidArgumentError(IcingStringUtil::StringPrintf(
143 "Property '%s' contains empty vectors for key: (%s, %s).",
144 property.name().c_str(), document.namespace_().c_str(),
145 document.uri().c_str()));
146 }
147 }
148 }
149
150 if (property_config.cardinality() ==
151 PropertyConfigProto::Cardinality::OPTIONAL) {
152 if (value_size != 0 && value_size != 1) {
153 return absl_ports::InvalidArgumentError(IcingStringUtil::StringPrintf(
154 "Property '%s' is optional but %d elements are "
155 "found for key: (%s, %s).",
156 property.name().c_str(), value_size, document.namespace_().c_str(),
157 document.uri().c_str()));
158 }
159 } else if (property_config.cardinality() ==
160 PropertyConfigProto::Cardinality::REQUIRED) {
161 if (value_size != 1) {
162 return absl_ports::InvalidArgumentError(IcingStringUtil::StringPrintf(
163 "Property '%s' with only 1 value is required but "
164 "%d elements are found for key: (%s, %s).",
165 property.name().c_str(), value_size, document.namespace_().c_str(),
166 document.uri().c_str()));
167 }
168 num_required_properties_actual++;
169 }
170
171 // We put the validation for nested DocumentProto at last separately
172 // because it takes longer time to run. If any of the previous validations
173 // fail, we don't need to validate the extra documents.
174 if (property_config.data_type() ==
175 PropertyConfigProto::DataType::DOCUMENT) {
176 ICING_ASSIGN_OR_RETURN(
177 const std::unordered_set<SchemaTypeId>* nested_type_ids_expected,
178 schema_store_->GetSchemaTypeIdsWithChildren(
179 property_config.schema_type()));
180 for (const DocumentProto& nested_document : property.document_values()) {
181 libtextclassifier3::StatusOr<SchemaTypeId> nested_document_type_id_or =
182 schema_store_->GetSchemaTypeId(nested_document.schema());
183 if (!nested_document_type_id_or.ok() ||
184 nested_type_ids_expected->count(
185 nested_document_type_id_or.ValueOrDie()) == 0) {
186 return absl_ports::InvalidArgumentError(absl_ports::StrCat(
187 "Property '", property.name(), "' should be type or subtype of '",
188 property_config.schema_type(), "' but actual value has type '",
189 nested_document.schema(), "' for key: (", document.namespace_(),
190 ", ", document.uri(), ")."));
191 }
192 ICING_RETURN_IF_ERROR(Validate(nested_document, depth + 1));
193 }
194 }
195 }
196 if (num_required_properties_actual <
197 parsed_property_configs.required_properties.size()) {
198 return absl_ports::InvalidArgumentError(
199 absl_ports::StrCat("One or more required fields missing for key: (",
200 document.namespace_(), ", ", document.uri(), ")."));
201 }
202 return libtextclassifier3::Status::OK;
203 }
204
205 } // namespace lib
206 } // namespace icing
207