1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.textclassifier; 18 19 import android.os.LocaleList; 20 import android.os.ParcelFileDescriptor; 21 import android.text.TextUtils; 22 import androidx.annotation.GuardedBy; 23 import com.android.textclassifier.common.base.TcLog; 24 import com.android.textclassifier.common.logging.ResultIdUtils.ModelInfo; 25 import com.google.common.base.Optional; 26 import com.google.common.base.Preconditions; 27 import com.google.common.base.Splitter; 28 import com.google.common.collect.ImmutableList; 29 import java.io.File; 30 import java.io.FileNotFoundException; 31 import java.io.IOException; 32 import java.util.ArrayList; 33 import java.util.Arrays; 34 import java.util.Collections; 35 import java.util.List; 36 import java.util.Locale; 37 import java.util.Objects; 38 import java.util.function.Function; 39 import java.util.function.Supplier; 40 import java.util.regex.Matcher; 41 import java.util.regex.Pattern; 42 import java.util.stream.Collectors; 43 import javax.annotation.Nullable; 44 45 /** Manages model files that are listed by the model files supplier. */ 46 final class ModelFileManager { 47 private static final String TAG = "ModelFileManager"; 48 49 private final Supplier<ImmutableList<ModelFile>> modelFileSupplier; 50 ModelFileManager(Supplier<ImmutableList<ModelFile>> modelFileSupplier)51 public ModelFileManager(Supplier<ImmutableList<ModelFile>> modelFileSupplier) { 52 this.modelFileSupplier = Preconditions.checkNotNull(modelFileSupplier); 53 } 54 55 /** Returns an immutable list of model files listed by the given model files supplier. */ listModelFiles()56 public ImmutableList<ModelFile> listModelFiles() { 57 return modelFileSupplier.get(); 58 } 59 60 /** 61 * Returns the best model file for the given localelist, {@code null} if nothing is found. 62 * 63 * @param localeList the required locales, use {@code null} if there is no preference. 64 */ findBestModelFile(@ullable LocaleList localeList)65 public ModelFile findBestModelFile(@Nullable LocaleList localeList) { 66 final String languages = 67 localeList == null || localeList.isEmpty() 68 ? LocaleList.getDefault().toLanguageTags() 69 : localeList.toLanguageTags(); 70 final List<Locale.LanguageRange> languageRangeList = Locale.LanguageRange.parse(languages); 71 72 ModelFile bestModel = null; 73 for (ModelFile model : listModelFiles()) { 74 if (model.isAnyLanguageSupported(languageRangeList)) { 75 if (model.isPreferredTo(bestModel)) { 76 bestModel = model; 77 } 78 } 79 } 80 return bestModel; 81 } 82 83 /** Default implementation of the model file supplier. */ 84 public static final class ModelFileSupplierImpl implements Supplier<ImmutableList<ModelFile>> { 85 private final File updatedModelFile; 86 private final File factoryModelDir; 87 private final Pattern modelFilenamePattern; 88 private final Function<Integer, Integer> versionSupplier; 89 private final Function<Integer, String> supportedLocalesSupplier; 90 private final Object lock = new Object(); 91 92 @GuardedBy("lock") 93 private ImmutableList<ModelFile> factoryModels; 94 ModelFileSupplierImpl( File factoryModelDir, String factoryModelFileNameRegex, File updatedModelFile, Function<Integer, Integer> versionSupplier, Function<Integer, String> supportedLocalesSupplier)95 public ModelFileSupplierImpl( 96 File factoryModelDir, 97 String factoryModelFileNameRegex, 98 File updatedModelFile, 99 Function<Integer, Integer> versionSupplier, 100 Function<Integer, String> supportedLocalesSupplier) { 101 this.updatedModelFile = Preconditions.checkNotNull(updatedModelFile); 102 this.factoryModelDir = Preconditions.checkNotNull(factoryModelDir); 103 modelFilenamePattern = Pattern.compile(Preconditions.checkNotNull(factoryModelFileNameRegex)); 104 this.versionSupplier = Preconditions.checkNotNull(versionSupplier); 105 this.supportedLocalesSupplier = Preconditions.checkNotNull(supportedLocalesSupplier); 106 } 107 108 @Override get()109 public ImmutableList<ModelFile> get() { 110 final List<ModelFile> modelFiles = new ArrayList<>(); 111 // The update model has the highest precedence. 112 if (updatedModelFile.exists()) { 113 final ModelFile updatedModel = createModelFile(updatedModelFile); 114 if (updatedModel != null) { 115 modelFiles.add(updatedModel); 116 } 117 } 118 // Factory models should never have overlapping locales, so the order doesn't matter. 119 synchronized (lock) { 120 if (factoryModels == null) { 121 factoryModels = getFactoryModels(); 122 } 123 modelFiles.addAll(factoryModels); 124 } 125 return ImmutableList.copyOf(modelFiles); 126 } 127 getFactoryModels()128 private ImmutableList<ModelFile> getFactoryModels() { 129 List<ModelFile> factoryModelFiles = new ArrayList<>(); 130 if (factoryModelDir.exists() && factoryModelDir.isDirectory()) { 131 final File[] files = factoryModelDir.listFiles(); 132 for (File file : files) { 133 final Matcher matcher = modelFilenamePattern.matcher(file.getName()); 134 if (matcher.matches() && file.isFile()) { 135 final ModelFile model = createModelFile(file); 136 if (model != null) { 137 factoryModelFiles.add(model); 138 } 139 } 140 } 141 } 142 return ImmutableList.copyOf(factoryModelFiles); 143 } 144 145 /** Returns null if the path did not point to a compatible model. */ 146 @Nullable createModelFile(File file)147 private ModelFile createModelFile(File file) { 148 if (!file.exists()) { 149 return null; 150 } 151 ParcelFileDescriptor modelFd = null; 152 try { 153 modelFd = ParcelFileDescriptor.open(file, ParcelFileDescriptor.MODE_READ_ONLY); 154 if (modelFd == null) { 155 return null; 156 } 157 final int modelFdInt = modelFd.getFd(); 158 final int version = versionSupplier.apply(modelFdInt); 159 final String supportedLocalesStr = supportedLocalesSupplier.apply(modelFdInt); 160 if (supportedLocalesStr.isEmpty()) { 161 TcLog.d(TAG, "Ignoring " + file.getAbsolutePath()); 162 return null; 163 } 164 final List<Locale> supportedLocales = new ArrayList<>(); 165 for (String langTag : Splitter.on(',').split(supportedLocalesStr)) { 166 supportedLocales.add(Locale.forLanguageTag(langTag)); 167 } 168 return new ModelFile( 169 file, 170 version, 171 supportedLocales, 172 supportedLocalesStr, 173 ModelFile.LANGUAGE_INDEPENDENT.equals(supportedLocalesStr)); 174 } catch (FileNotFoundException e) { 175 TcLog.e(TAG, "Failed to find " + file.getAbsolutePath(), e); 176 return null; 177 } finally { 178 maybeCloseAndLogError(modelFd); 179 } 180 } 181 182 /** Closes the ParcelFileDescriptor, if non-null, and logs any errors that occur. */ maybeCloseAndLogError(@ullable ParcelFileDescriptor fd)183 private static void maybeCloseAndLogError(@Nullable ParcelFileDescriptor fd) { 184 if (fd == null) { 185 return; 186 } 187 try { 188 fd.close(); 189 } catch (IOException e) { 190 TcLog.e(TAG, "Error closing file.", e); 191 } 192 } 193 } 194 195 /** Describes TextClassifier model files on disk. */ 196 public static final class ModelFile { 197 public static final String LANGUAGE_INDEPENDENT = "*"; 198 199 private final File file; 200 private final int version; 201 private final List<Locale> supportedLocales; 202 private final String supportedLocalesStr; 203 private final boolean languageIndependent; 204 ModelFile( File file, int version, List<Locale> supportedLocales, String supportedLocalesStr, boolean languageIndependent)205 public ModelFile( 206 File file, 207 int version, 208 List<Locale> supportedLocales, 209 String supportedLocalesStr, 210 boolean languageIndependent) { 211 this.file = Preconditions.checkNotNull(file); 212 this.version = version; 213 this.supportedLocales = Preconditions.checkNotNull(supportedLocales); 214 this.supportedLocalesStr = Preconditions.checkNotNull(supportedLocalesStr); 215 this.languageIndependent = languageIndependent; 216 } 217 218 /** Returns the absolute path to the model file. */ getPath()219 public String getPath() { 220 return file.getAbsolutePath(); 221 } 222 223 /** Returns a name to use for id generation, effectively the name of the model file. */ getName()224 public String getName() { 225 return file.getName(); 226 } 227 228 /** Returns the version tag in the model's metadata. */ getVersion()229 public int getVersion() { 230 return version; 231 } 232 233 /** Returns whether the language supports any language in the given ranges. */ isAnyLanguageSupported(List<Locale.LanguageRange> languageRanges)234 public boolean isAnyLanguageSupported(List<Locale.LanguageRange> languageRanges) { 235 Preconditions.checkNotNull(languageRanges); 236 return languageIndependent || Locale.lookup(languageRanges, supportedLocales) != null; 237 } 238 239 /** Returns an immutable lists of supported locales. */ getSupportedLocales()240 public List<Locale> getSupportedLocales() { 241 return Collections.unmodifiableList(supportedLocales); 242 } 243 244 /** Returns the original supported locals string read from the model file. */ getSupportedLocalesStr()245 public String getSupportedLocalesStr() { 246 return supportedLocalesStr; 247 } 248 249 /** Returns if this model file is preferred to the given one. */ isPreferredTo(@ullable ModelFile model)250 public boolean isPreferredTo(@Nullable ModelFile model) { 251 // A model is preferred to no model. 252 if (model == null) { 253 return true; 254 } 255 256 // A language-specific model is preferred to a language independent 257 // model. 258 if (!languageIndependent && model.languageIndependent) { 259 return true; 260 } 261 if (languageIndependent && !model.languageIndependent) { 262 return false; 263 } 264 265 // A higher-version model is preferred. 266 if (version > model.getVersion()) { 267 return true; 268 } 269 return false; 270 } 271 272 @Override hashCode()273 public int hashCode() { 274 return Objects.hash(getPath()); 275 } 276 277 @Override equals(Object other)278 public boolean equals(Object other) { 279 if (this == other) { 280 return true; 281 } 282 if (other instanceof ModelFile) { 283 final ModelFile otherModel = (ModelFile) other; 284 return TextUtils.equals(getPath(), otherModel.getPath()); 285 } 286 return false; 287 } 288 toModelInfo()289 public ModelInfo toModelInfo() { 290 return new ModelInfo(getVersion(), supportedLocalesStr); 291 } 292 293 @Override toString()294 public String toString() { 295 return String.format( 296 Locale.US, 297 "ModelFile { path=%s name=%s version=%d locales=%s }", 298 getPath(), 299 getName(), 300 version, 301 supportedLocalesStr); 302 } 303 toModelInfos( Optional<ModelFile>.... modelFiles)304 public static ImmutableList<Optional<ModelInfo>> toModelInfos( 305 Optional<ModelFile>... modelFiles) { 306 return Arrays.stream(modelFiles) 307 .map(modelFile -> modelFile.transform(ModelFile::toModelInfo)) 308 .collect(Collectors.collectingAndThen(Collectors.toList(), ImmutableList::copyOf)); 309 } 310 } 311 } 312