1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.util; 18 19 import com.android.tradefed.build.BuildRetrievalError; 20 import com.android.tradefed.build.IFileDownloader; 21 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 22 import com.android.tradefed.log.LogUtil.CLog; 23 import com.android.tradefed.result.error.InfraErrorIdentifier; 24 import com.android.tradefed.util.gcs.GCSFileDownloaderBase; 25 26 import com.google.api.services.storage.Storage; 27 import com.google.api.services.storage.model.StorageObject; 28 import com.google.common.annotations.VisibleForTesting; 29 import com.google.common.cache.CacheBuilder; 30 import com.google.common.cache.CacheLoader; 31 import com.google.common.cache.LoadingCache; 32 33 import java.io.File; 34 import java.io.IOException; 35 import java.nio.file.Paths; 36 import java.util.ArrayList; 37 import java.util.Arrays; 38 import java.util.Collection; 39 import java.util.HashSet; 40 import java.util.List; 41 import java.util.Set; 42 import java.util.concurrent.TimeUnit; 43 44 /** File downloader to download file from google cloud storage (GCS). */ 45 public class GCSFileDownloader extends GCSFileDownloaderBase implements IFileDownloader { 46 47 // Cache the freshness 48 private final LoadingCache<String, Boolean> mFreshnessCache; 49 GCSFileDownloader(Boolean createEmptyFile)50 public GCSFileDownloader(Boolean createEmptyFile) { 51 super(createEmptyFile); 52 mFreshnessCache = 53 CacheBuilder.newBuilder() 54 .maximumSize(50) 55 .expireAfterAccess(60, TimeUnit.MINUTES) 56 .build( 57 new CacheLoader<String, Boolean>() { 58 @Override 59 public Boolean load(String key) throws BuildRetrievalError { 60 return true; 61 } 62 }); 63 } 64 GCSFileDownloader()65 public GCSFileDownloader() { 66 this(false); 67 } 68 GCSFileDownloader(File jsonKeyFile)69 public GCSFileDownloader(File jsonKeyFile) { 70 this(jsonKeyFile, false); 71 } 72 GCSFileDownloader(File jsonKeyFile, Boolean createEmptyFile)73 public GCSFileDownloader(File jsonKeyFile, Boolean createEmptyFile) { 74 this(createEmptyFile); 75 mJsonKeyFile = jsonKeyFile; 76 } 77 78 /** 79 * Override the implementation in base to support credential based on TF options. 80 * 81 * @param scopes specific scopes to request credential for. 82 * @return {@link Storage} object of the GCS bucket 83 * @throws IOException 84 */ 85 @Override getStorage(Collection<String> scopes)86 protected Storage getStorage(Collection<String> scopes) throws IOException { 87 return GCSHelper.getStorage(scopes, mJsonKeyFile); 88 } 89 clearCache()90 protected void clearCache() { 91 mFreshnessCache.invalidateAll(); 92 } 93 94 /** 95 * Download file from GCS. 96 * 97 * <p>Right now only support GCS path. 98 * 99 * @param remoteFilePath gs://bucket/file/path format GCS path. 100 * @return local file 101 * @throws BuildRetrievalError 102 */ 103 @Override downloadFile(String remoteFilePath)104 public File downloadFile(String remoteFilePath) throws BuildRetrievalError { 105 File destFile = createTempFileForRemote(remoteFilePath, null); 106 try { 107 downloadFile(remoteFilePath, destFile); 108 return destFile; 109 } catch (BuildRetrievalError e) { 110 FileUtil.recursiveDelete(destFile); 111 throw e; 112 } 113 } 114 115 @Override downloadFile(String remotePath, File destFile)116 public void downloadFile(String remotePath, File destFile) throws BuildRetrievalError { 117 String[] pathParts = parseGcsPath(remotePath); 118 downloadFile(pathParts[0], pathParts[1], destFile); 119 } 120 121 @VisibleForTesting 122 @Override downloadFile(String bucketName, String remoteFilename, File localFile)123 protected void downloadFile(String bucketName, String remoteFilename, File localFile) 124 throws BuildRetrievalError { 125 try { 126 super.downloadFile(bucketName, remoteFilename, localFile); 127 } catch (Exception e) { 128 throw new BuildRetrievalError(e.getMessage(), e, InfraErrorIdentifier.GCS_ERROR); 129 } 130 } 131 isFileFresh(File localFile, StorageObject remoteFile)132 private boolean isFileFresh(File localFile, StorageObject remoteFile) { 133 if (localFile == null && remoteFile == null) { 134 return true; 135 } 136 if (localFile == null || remoteFile == null) { 137 return false; 138 } 139 if (!localFile.exists()) { 140 return false; 141 } 142 return remoteFile.getMd5Hash().equals(FileUtil.calculateBase64Md5(localFile)); 143 } 144 145 @Override isFresh(File localFile, String remotePath)146 public boolean isFresh(File localFile, String remotePath) throws BuildRetrievalError { 147 String[] pathParts = parseGcsPath(remotePath); 148 String bucketName = pathParts[0]; 149 String remoteFilename = pathParts[1]; 150 151 if (localFile != null && localFile.exists()) { 152 Boolean cache = mFreshnessCache.getIfPresent(remotePath); 153 if (cache != null && Boolean.TRUE.equals(cache)) { 154 return true; 155 } 156 } 157 158 try (CloseableTraceScope ignored = new CloseableTraceScope("gcs_is_fresh " + remotePath)) { 159 StorageObject remoteFileMeta = getRemoteFileMetaData(bucketName, remoteFilename); 160 if (localFile == null || !localFile.exists()) { 161 if (!isRemoteFolder(bucketName, remoteFilename) && remoteFileMeta == null) { 162 // The local doesn't exist and the remote filename is not a folder or a file. 163 return true; 164 } 165 return false; 166 } 167 if (!localFile.isDirectory()) { 168 return isFileFresh(localFile, remoteFileMeta); 169 } 170 remoteFilename = sanitizeDirectoryName(remoteFilename); 171 boolean fresh = recursiveCheckFolderFreshness(bucketName, remoteFilename, localFile); 172 mFreshnessCache.put(remotePath, fresh); 173 return fresh; 174 } catch (IOException e) { 175 mFreshnessCache.invalidate(remotePath); 176 throw new BuildRetrievalError(e.getMessage(), e, InfraErrorIdentifier.GCS_ERROR); 177 } 178 } 179 180 /** 181 * Check if remote folder is the same as local folder, recursively. The remoteFolderName must 182 * end with "/". 183 * 184 * @param bucketName is the gcs bucket name. 185 * @param remoteFolderName is the relative path to the bucket. 186 * @param localFolder is the local folder 187 * @return true if local file is the same as remote file, otherwise false. 188 * @throws IOException 189 */ recursiveCheckFolderFreshness( String bucketName, String remoteFolderName, File localFolder)190 private boolean recursiveCheckFolderFreshness( 191 String bucketName, String remoteFolderName, File localFolder) throws IOException { 192 Set<String> subFilenames = new HashSet<>(Arrays.asList(localFolder.list())); 193 List<String> subRemoteFolders = new ArrayList<>(); 194 List<StorageObject> subRemoteFiles = new ArrayList<>(); 195 listRemoteFilesUnderFolder(bucketName, remoteFolderName, subRemoteFiles, subRemoteFolders); 196 for (StorageObject subRemoteFile : subRemoteFiles) { 197 String subFilename = Paths.get(subRemoteFile.getName()).getFileName().toString(); 198 if (!isFileFresh(new File(localFolder, subFilename), subRemoteFile)) { 199 return false; 200 } 201 subFilenames.remove(subFilename); 202 } 203 for (String subRemoteFolder : subRemoteFolders) { 204 String subFolderName = Paths.get(subRemoteFolder).getFileName().toString(); 205 File subFolder = new File(localFolder, subFolderName); 206 if (!subFolder.exists()) { 207 return false; 208 } 209 if (!subFolder.isDirectory()) { 210 CLog.w("%s exists as a non-directory.", subFolder); 211 subFolder = new File(localFolder, subFolderName + "_folder"); 212 } 213 if (!recursiveCheckFolderFreshness(bucketName, subRemoteFolder, subFolder)) { 214 return false; 215 } 216 subFilenames.remove(subFolder.getName()); 217 } 218 return subFilenames.isEmpty(); 219 } 220 221 @Override parseGcsPath(String remotePath)222 protected String[] parseGcsPath(String remotePath) throws BuildRetrievalError { 223 try { 224 return super.parseGcsPath(remotePath); 225 } catch (Exception e) { 226 throw new BuildRetrievalError( 227 e.getMessage(), InfraErrorIdentifier.ARTIFACT_UNSUPPORTED_PATH); 228 } 229 } 230 createTempFileForRemote(String remoteFilePath, File rootDir)231 public static File createTempFileForRemote(String remoteFilePath, File rootDir) 232 throws BuildRetrievalError { 233 try { 234 return GCSFileDownloaderBase.createTempFileForRemote(remoteFilePath, rootDir); 235 } catch (Exception e) { 236 throw new BuildRetrievalError(e.getMessage(), e); 237 } 238 } 239 } 240