1 /*
2  * Copyright 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.appsearch.localstorage.converter;
18 
19 import androidx.annotation.OptIn;
20 import androidx.annotation.RestrictTo;
21 import androidx.appsearch.app.AppSearchBlobHandle;
22 import androidx.appsearch.app.AppSearchSchema;
23 import androidx.appsearch.app.EmbeddingVector;
24 import androidx.appsearch.app.ExperimentalAppSearchApi;
25 import androidx.appsearch.app.GenericDocument;
26 import androidx.appsearch.exceptions.AppSearchException;
27 import androidx.appsearch.flags.Flags;
28 import androidx.appsearch.localstorage.AppSearchConfig;
29 import androidx.appsearch.localstorage.SchemaCache;
30 import androidx.core.util.Preconditions;
31 
32 import com.google.android.icing.proto.DocumentProto;
33 import com.google.android.icing.proto.DocumentProtoOrBuilder;
34 import com.google.android.icing.proto.PropertyProto;
35 import com.google.android.icing.proto.SchemaTypeConfigProto;
36 import com.google.android.icing.protobuf.ByteString;
37 
38 import org.jspecify.annotations.NonNull;
39 
40 import java.util.ArrayList;
41 import java.util.Collections;
42 import java.util.List;
43 import java.util.Map;
44 
45 /**
46  * Translates a {@link GenericDocument} into a {@link DocumentProto}.
47  *
48  * @exportToFramework:hide
49  */
50 @RestrictTo(RestrictTo.Scope.LIBRARY_GROUP)
51 public final class GenericDocumentToProtoConverter {
52     private static final String[] EMPTY_STRING_ARRAY = new String[0];
53     private static final long[] EMPTY_LONG_ARRAY = new long[0];
54     private static final double[] EMPTY_DOUBLE_ARRAY = new double[0];
55     private static final boolean[] EMPTY_BOOLEAN_ARRAY = new boolean[0];
56     private static final byte[][] EMPTY_BYTES_ARRAY = new byte[0][0];
57     private static final GenericDocument[] EMPTY_DOCUMENT_ARRAY = new GenericDocument[0];
58     private static final EmbeddingVector[] EMPTY_EMBEDDING_ARRAY =
59             new EmbeddingVector[0];
60 
GenericDocumentToProtoConverter()61     private GenericDocumentToProtoConverter() {
62     }
63 
64     /**
65      * Converts a {@link GenericDocument} into a {@link DocumentProto}.
66      */
67     @SuppressWarnings("unchecked")
68     @OptIn(markerClass = ExperimentalAppSearchApi.class)
toDocumentProto(@onNull GenericDocument document)69     public static @NonNull DocumentProto toDocumentProto(@NonNull GenericDocument document) {
70         Preconditions.checkNotNull(document);
71         DocumentProto.Builder mProtoBuilder = DocumentProto.newBuilder();
72         mProtoBuilder.setUri(document.getId())
73                 .setSchema(document.getSchemaType())
74                 .setNamespace(document.getNamespace())
75                 .setScore(document.getScore())
76                 .setTtlMs(document.getTtlMillis())
77                 .setCreationTimestampMs(document.getCreationTimestampMillis());
78         ArrayList<String> keys = new ArrayList<>(document.getPropertyNames());
79         Collections.sort(keys);
80         for (int i = 0; i < keys.size(); i++) {
81             String name = keys.get(i);
82             PropertyProto.Builder propertyProto = PropertyProto.newBuilder().setName(name);
83             Object property = document.getProperty(name);
84             if (property instanceof String[]) {
85                 String[] stringValues = (String[]) property;
86                 for (int j = 0; j < stringValues.length; j++) {
87                     propertyProto.addStringValues(stringValues[j]);
88                 }
89             } else if (property instanceof long[]) {
90                 long[] longValues = (long[]) property;
91                 for (int j = 0; j < longValues.length; j++) {
92                     propertyProto.addInt64Values(longValues[j]);
93                 }
94             } else if (property instanceof double[]) {
95                 double[] doubleValues = (double[]) property;
96                 for (int j = 0; j < doubleValues.length; j++) {
97                     propertyProto.addDoubleValues(doubleValues[j]);
98                 }
99             } else if (property instanceof boolean[]) {
100                 boolean[] booleanValues = (boolean[]) property;
101                 for (int j = 0; j < booleanValues.length; j++) {
102                     propertyProto.addBooleanValues(booleanValues[j]);
103                 }
104             } else if (property instanceof byte[][]) {
105                 byte[][] bytesValues = (byte[][]) property;
106                 for (int j = 0; j < bytesValues.length; j++) {
107                     propertyProto.addBytesValues(ByteString.copyFrom(bytesValues[j]));
108                 }
109             } else if (property instanceof GenericDocument[]) {
110                 GenericDocument[] documentValues = (GenericDocument[]) property;
111                 for (int j = 0; j < documentValues.length; j++) {
112                     DocumentProto proto = toDocumentProto(documentValues[j]);
113                     propertyProto.addDocumentValues(proto);
114                 }
115             } else if (property instanceof EmbeddingVector[]) {
116                 EmbeddingVector[] embeddingValues = (EmbeddingVector[]) property;
117                 for (int j = 0; j < embeddingValues.length; j++) {
118                     propertyProto.addVectorValues(
119                             embeddingVectorToVectorProto(embeddingValues[j]));
120                 }
121             } else if (property instanceof AppSearchBlobHandle[]) {
122                 AppSearchBlobHandle[] blobHandleValues = (AppSearchBlobHandle[]) property;
123                 for (int j = 0; j < blobHandleValues.length; j++) {
124                     propertyProto.addBlobHandleValues(
125                             BlobHandleToProtoConverter.toBlobHandleProto(blobHandleValues[j]));
126                 }
127             } else if (property == null) {
128                 throw new IllegalStateException(
129                         String.format("Property \"%s\" doesn't have any value!", name));
130             } else {
131                 throw new IllegalStateException(
132                         String.format("Property \"%s\" has unsupported value type %s", name,
133                                 property.getClass().toString()));
134             }
135             mProtoBuilder.addProperties(propertyProto);
136         }
137         return mProtoBuilder.build();
138     }
139 
140     /**
141      * Converts a {@link DocumentProto} into a {@link GenericDocument}.
142      *
143      * <p>In the case that the {@link DocumentProto} object proto has no values set, the
144      * converter searches for the matching property name in the {@link SchemaTypeConfigProto}
145      * object for the document, and infers the correct default value to set for the empty
146      * property based on the data type of the property defined by the schema type.
147      *
148      * @param proto         the document to convert to a {@link GenericDocument} instance. The
149      *                      document proto should have its package + database prefix stripped
150      *                      from its fields.
151      * @param prefix        the package + database prefix used searching the {@code schemaTypeMap}.
152      * @param schemaCache   The SchemaCache instance held in AppSearch.
153      */
154     @SuppressWarnings("deprecation")
155     @OptIn(markerClass = ExperimentalAppSearchApi.class)
toGenericDocument(@onNull DocumentProtoOrBuilder proto, @NonNull String prefix, @NonNull SchemaCache schemaCache, @NonNull AppSearchConfig config)156     public static @NonNull GenericDocument toGenericDocument(@NonNull DocumentProtoOrBuilder proto,
157             @NonNull String prefix,
158             @NonNull SchemaCache schemaCache,
159             @NonNull AppSearchConfig config) throws AppSearchException {
160         Preconditions.checkNotNull(proto);
161         Preconditions.checkNotNull(prefix);
162         Preconditions.checkNotNull(schemaCache);
163         Preconditions.checkNotNull(config);
164         Map<String, SchemaTypeConfigProto> schemaTypeMap =
165                 schemaCache.getSchemaMapForPrefix(prefix);
166 
167         GenericDocument.Builder<?> documentBuilder =
168                 new GenericDocument.Builder<>(proto.getNamespace(), proto.getUri(),
169                         proto.getSchema())
170                         .setScore(proto.getScore())
171                         .setTtlMillis(proto.getTtlMs())
172                         .setCreationTimestampMillis(proto.getCreationTimestampMs());
173         String prefixedSchemaType = prefix + proto.getSchema();
174         if (config.shouldRetrieveParentInfo() && !Flags.enableSearchResultParentTypes()) {
175             List<String> parentSchemaTypes =
176                     schemaCache.getTransitiveUnprefixedParentSchemaTypes(
177                             prefix, prefixedSchemaType);
178             if (!parentSchemaTypes.isEmpty()) {
179                 if (config.shouldStoreParentInfoAsSyntheticProperty()) {
180                     documentBuilder.setPropertyString(
181                             GenericDocument.PARENT_TYPES_SYNTHETIC_PROPERTY,
182                             parentSchemaTypes.toArray(new String[0]));
183                 } else {
184                     documentBuilder.setParentTypes(parentSchemaTypes);
185                 }
186             }
187         }
188 
189         for (int i = 0; i < proto.getPropertiesCount(); i++) {
190             PropertyProto property = proto.getProperties(i);
191             String name = property.getName();
192             if (property.getStringValuesCount() > 0) {
193                 String[] values = new String[property.getStringValuesCount()];
194                 for (int j = 0; j < values.length; j++) {
195                     values[j] = property.getStringValues(j);
196                 }
197                 documentBuilder.setPropertyString(name, values);
198             } else if (property.getInt64ValuesCount() > 0) {
199                 long[] values = new long[property.getInt64ValuesCount()];
200                 for (int j = 0; j < values.length; j++) {
201                     values[j] = property.getInt64Values(j);
202                 }
203                 documentBuilder.setPropertyLong(name, values);
204             } else if (property.getDoubleValuesCount() > 0) {
205                 double[] values = new double[property.getDoubleValuesCount()];
206                 for (int j = 0; j < values.length; j++) {
207                     values[j] = property.getDoubleValues(j);
208                 }
209                 documentBuilder.setPropertyDouble(name, values);
210             } else if (property.getBooleanValuesCount() > 0) {
211                 boolean[] values = new boolean[property.getBooleanValuesCount()];
212                 for (int j = 0; j < values.length; j++) {
213                     values[j] = property.getBooleanValues(j);
214                 }
215                 documentBuilder.setPropertyBoolean(name, values);
216             } else if (property.getBytesValuesCount() > 0) {
217                 byte[][] values = new byte[property.getBytesValuesCount()][];
218                 for (int j = 0; j < values.length; j++) {
219                     values[j] = property.getBytesValues(j).toByteArray();
220                 }
221                 documentBuilder.setPropertyBytes(name, values);
222             } else if (property.getDocumentValuesCount() > 0) {
223                 GenericDocument[] values = new GenericDocument[property.getDocumentValuesCount()];
224                 for (int j = 0; j < values.length; j++) {
225                     values[j] = toGenericDocument(property.getDocumentValues(j), prefix,
226                             schemaCache, config);
227                 }
228                 documentBuilder.setPropertyDocument(name, values);
229             } else if (property.getVectorValuesCount() > 0) {
230                 EmbeddingVector[] values =
231                         new EmbeddingVector[property.getVectorValuesCount()];
232                 for (int j = 0; j < values.length; j++) {
233                     values[j] = vectorProtoToEmbeddingVector(property.getVectorValues(j));
234                 }
235                 documentBuilder.setPropertyEmbedding(name, values);
236             } else if (property.getBlobHandleValuesCount() > 0) {
237                 AppSearchBlobHandle[] values =
238                         new AppSearchBlobHandle[property.getBlobHandleValuesCount()];
239                 for (int j = 0; j < values.length; j++) {
240                     values[j] = BlobHandleToProtoConverter.toAppSearchBlobHandle(
241                             property.getBlobHandleValues(j));
242                 }
243                 documentBuilder.setPropertyBlobHandle(name, values);
244             } else {
245                 // TODO(b/184966497): Optimize by caching PropertyConfigProto
246                 SchemaTypeConfigProto schema =
247                         Preconditions.checkNotNull(schemaTypeMap.get(prefixedSchemaType));
248                 setEmptyProperty(name, documentBuilder, schema);
249             }
250         }
251         return documentBuilder.build();
252     }
253 
254     /**
255      * Converts a {@link PropertyProto.VectorProto} into an {@link EmbeddingVector}.
256      */
vectorProtoToEmbeddingVector( PropertyProto.@onNull VectorProto vectorProto)257     public static @NonNull EmbeddingVector vectorProtoToEmbeddingVector(
258             PropertyProto.@NonNull VectorProto vectorProto) {
259         Preconditions.checkNotNull(vectorProto);
260 
261         float[] values = new float[vectorProto.getValuesCount()];
262         for (int i = 0; i < vectorProto.getValuesCount(); i++) {
263             values[i] = vectorProto.getValues(i);
264         }
265         return new EmbeddingVector(values, vectorProto.getModelSignature());
266     }
267 
268     /**
269      * Converts an {@link EmbeddingVector} into a {@link PropertyProto.VectorProto}.
270      */
embeddingVectorToVectorProto( @onNull EmbeddingVector embedding)271     public static PropertyProto.@NonNull VectorProto embeddingVectorToVectorProto(
272             @NonNull EmbeddingVector embedding) {
273         Preconditions.checkNotNull(embedding);
274 
275         PropertyProto.VectorProto.Builder builder = PropertyProto.VectorProto.newBuilder();
276         for (int i = 0; i < embedding.getValues().length; i++) {
277             builder.addValues(embedding.getValues()[i]);
278         }
279         builder.setModelSignature(embedding.getModelSignature());
280         return builder.build();
281     }
282 
setEmptyProperty(@onNull String propertyName, GenericDocument.@NonNull Builder<?> documentBuilder, @NonNull SchemaTypeConfigProto schema)283     private static void setEmptyProperty(@NonNull String propertyName,
284             GenericDocument.@NonNull Builder<?> documentBuilder,
285             @NonNull SchemaTypeConfigProto schema) {
286         @AppSearchSchema.PropertyConfig.DataType int dataType = 0;
287         for (int i = 0; i < schema.getPropertiesCount(); ++i) {
288             if (propertyName.equals(schema.getProperties(i).getPropertyName())) {
289                 dataType = schema.getProperties(i).getDataType().getNumber();
290                 break;
291             }
292         }
293 
294         switch (dataType) {
295             case AppSearchSchema.PropertyConfig.DATA_TYPE_STRING:
296                 documentBuilder.setPropertyString(propertyName, EMPTY_STRING_ARRAY);
297                 break;
298             case AppSearchSchema.PropertyConfig.DATA_TYPE_LONG:
299                 documentBuilder.setPropertyLong(propertyName, EMPTY_LONG_ARRAY);
300                 break;
301             case AppSearchSchema.PropertyConfig.DATA_TYPE_DOUBLE:
302                 documentBuilder.setPropertyDouble(propertyName, EMPTY_DOUBLE_ARRAY);
303                 break;
304             case AppSearchSchema.PropertyConfig.DATA_TYPE_BOOLEAN:
305                 documentBuilder.setPropertyBoolean(propertyName, EMPTY_BOOLEAN_ARRAY);
306                 break;
307             case AppSearchSchema.PropertyConfig.DATA_TYPE_BYTES:
308                 documentBuilder.setPropertyBytes(propertyName, EMPTY_BYTES_ARRAY);
309                 break;
310             case AppSearchSchema.PropertyConfig.DATA_TYPE_DOCUMENT:
311                 documentBuilder.setPropertyDocument(propertyName, EMPTY_DOCUMENT_ARRAY);
312                 break;
313             case AppSearchSchema.PropertyConfig.DATA_TYPE_EMBEDDING:
314                 documentBuilder.setPropertyEmbedding(propertyName, EMPTY_EMBEDDING_ARRAY);
315                 break;
316             default:
317                 throw new IllegalStateException("Unknown type of value: " + propertyName);
318         }
319     }
320 }
321