1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.config; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.config.OptionSetter.OptionFieldsForName; 20 import com.android.tradefed.config.remote.GcsRemoteFileResolver; 21 import com.android.tradefed.config.remote.IRemoteFileResolver; 22 import com.android.tradefed.log.LogUtil.CLog; 23 import com.android.tradefed.util.FileUtil; 24 import com.android.tradefed.util.MultiMap; 25 26 import java.io.File; 27 import java.lang.reflect.Field; 28 import java.util.ArrayList; 29 import java.util.Collection; 30 import java.util.HashMap; 31 import java.util.HashSet; 32 import java.util.LinkedHashMap; 33 import java.util.List; 34 import java.util.Map; 35 import java.util.Map.Entry; 36 import java.util.Set; 37 import java.util.concurrent.atomic.AtomicBoolean; 38 39 /** 40 * Class that helps resolving path to remote files. 41 * 42 * <p>For example: gs://bucket/path/file.txt will be resolved by downloading the file from the GCS 43 * bucket. 44 */ 45 public class DynamicRemoteFileResolver { 46 47 public static final String DYNAMIC_RESOLVER = "dynamic-resolver"; 48 private static final Map<String, IRemoteFileResolver> PROTOCOL_SUPPORT = new HashMap<>(); 49 50 static { PROTOCOL_SUPPORT.put(GcsRemoteFileResolver.PROTOCOL, new GcsRemoteFileResolver())51 PROTOCOL_SUPPORT.put(GcsRemoteFileResolver.PROTOCOL, new GcsRemoteFileResolver()); 52 } 53 // The configuration map being static, we only need to update it once per TF instance. 54 private static AtomicBoolean sIsUpdateDone = new AtomicBoolean(false); 55 56 private Map<String, OptionFieldsForName> mOptionMap; 57 58 /** Sets the map of options coming from {@link OptionSetter} */ setOptionMap(Map<String, OptionFieldsForName> optionMap)59 public void setOptionMap(Map<String, OptionFieldsForName> optionMap) { 60 mOptionMap = optionMap; 61 } 62 63 /** 64 * Runs through all the {@link File} option type and check if their path should be resolved. 65 * 66 * @return The list of {@link File} that was resolved that way. 67 * @throws ConfigurationException 68 */ validateRemoteFilePath()69 public final Set<File> validateRemoteFilePath() throws ConfigurationException { 70 Set<File> downloadedFiles = new HashSet<>(); 71 try { 72 for (Map.Entry<String, OptionFieldsForName> optionPair : mOptionMap.entrySet()) { 73 final OptionFieldsForName optionFields = optionPair.getValue(); 74 for (Map.Entry<Object, Field> fieldEntry : optionFields) { 75 final Object obj = fieldEntry.getKey(); 76 final Field field = fieldEntry.getValue(); 77 final Option option = field.getAnnotation(Option.class); 78 if (option == null) { 79 continue; 80 } 81 // At this point, we know this is an option field; make sure it's set 82 field.setAccessible(true); 83 final Object value; 84 try { 85 value = field.get(obj); 86 } catch (IllegalAccessException e) { 87 throw new ConfigurationException( 88 String.format("internal error: %s", e.getMessage())); 89 } 90 91 if (value == null) { 92 continue; 93 } else if (value instanceof File) { 94 File consideredFile = (File) value; 95 File downloadedFile = resolveRemoteFiles(consideredFile, option); 96 if (downloadedFile != null) { 97 downloadedFiles.add(downloadedFile); 98 // Replace the field value 99 try { 100 field.set(obj, downloadedFile); 101 } catch (IllegalAccessException e) { 102 CLog.e(e); 103 throw new ConfigurationException( 104 String.format( 105 "Failed to download %s due to '%s'", 106 consideredFile.getPath(), e.getMessage()), 107 e); 108 } 109 } 110 } else if (value instanceof Collection) { 111 Collection<Object> c = (Collection<Object>) value; 112 Collection<Object> copy = new ArrayList<>(c); 113 for (Object o : copy) { 114 if (o instanceof File) { 115 File consideredFile = (File) o; 116 File downloadedFile = resolveRemoteFiles(consideredFile, option); 117 if (downloadedFile != null) { 118 downloadedFiles.add(downloadedFile); 119 // TODO: See if order could be preserved. 120 c.remove(consideredFile); 121 c.add(downloadedFile); 122 } 123 } 124 } 125 } else if (value instanceof Map) { 126 Map<Object, Object> m = (Map<Object, Object>) value; 127 Map<Object, Object> copy = new LinkedHashMap<>(m); 128 for (Entry<Object, Object> entry : copy.entrySet()) { 129 Object key = entry.getKey(); 130 Object val = entry.getValue(); 131 132 Object finalKey = key; 133 Object finalVal = val; 134 if (key instanceof File) { 135 key = resolveRemoteFiles((File) key, option); 136 if (key != null) { 137 downloadedFiles.add((File) key); 138 finalKey = key; 139 } 140 } 141 if (val instanceof File) { 142 val = resolveRemoteFiles((File) val, option); 143 if (val != null) { 144 downloadedFiles.add((File) val); 145 finalVal = val; 146 } 147 } 148 149 m.remove(entry.getKey()); 150 m.put(finalKey, finalVal); 151 } 152 } else if (value instanceof MultiMap) { 153 MultiMap<Object, Object> m = (MultiMap<Object, Object>) value; 154 MultiMap<Object, Object> copy = new MultiMap<>(m); 155 for (Object key : copy.keySet()) { 156 List<Object> mapValues = copy.get(key); 157 158 m.remove(key); 159 Object finalKey = key; 160 if (key instanceof File) { 161 key = resolveRemoteFiles((File) key, option); 162 if (key != null) { 163 downloadedFiles.add((File) key); 164 finalKey = key; 165 } 166 } 167 for (Object mapValue : mapValues) { 168 if (mapValue instanceof File) { 169 File f = resolveRemoteFiles((File) mapValue, option); 170 if (f != null) { 171 downloadedFiles.add(f); 172 mapValue = f; 173 } 174 } 175 m.put(finalKey, mapValue); 176 } 177 } 178 } 179 } 180 } 181 } catch (ConfigurationException e) { 182 // Clean up the files before throwing 183 for (File f : downloadedFiles) { 184 FileUtil.recursiveDelete(f); 185 } 186 throw e; 187 } 188 return downloadedFiles; 189 } 190 191 @VisibleForTesting getResolver(String protocol)192 protected IRemoteFileResolver getResolver(String protocol) { 193 if (updateProtocols()) { 194 IGlobalConfiguration globalConfig = getGlobalConfig(); 195 Object o = globalConfig.getConfigurationObject(DYNAMIC_RESOLVER); 196 if (o != null) { 197 if (o instanceof IRemoteFileResolver) { 198 IRemoteFileResolver resolver = (IRemoteFileResolver) o; 199 CLog.d("Adding %s to supported remote file resolver", resolver); 200 PROTOCOL_SUPPORT.put(resolver.getSupportedProtocol(), resolver); 201 } else { 202 CLog.e("%s is not of type IRemoteFileResolver", o); 203 } 204 } 205 } 206 return PROTOCOL_SUPPORT.get(protocol); 207 } 208 209 @VisibleForTesting updateProtocols()210 protected boolean updateProtocols() { 211 return sIsUpdateDone.compareAndSet(false, true); 212 } 213 214 @VisibleForTesting getGlobalConfig()215 IGlobalConfiguration getGlobalConfig() { 216 return GlobalConfiguration.getInstance(); 217 } 218 resolveRemoteFiles(File consideredFile, Option option)219 private File resolveRemoteFiles(File consideredFile, Option option) 220 throws ConfigurationException { 221 String path = consideredFile.getPath(); 222 String protocol = getProtocol(path); 223 IRemoteFileResolver resolver = getResolver(protocol); 224 if (resolver != null) { 225 return resolver.resolveRemoteFiles(consideredFile, option); 226 } 227 // Not a remote file 228 return null; 229 } 230 231 /** 232 * Java URL doesn't recognize 'gs' as a protocol and throws an exception so we do the protocol 233 * extraction ourselves. 234 */ getProtocol(String path)235 private String getProtocol(String path) { 236 int index = path.indexOf(":/"); 237 if (index == -1) { 238 return ""; 239 } 240 return path.substring(0, index); 241 } 242 } 243