1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.util; 18 19 import com.android.tradefed.build.BuildRetrievalError; 20 21 import com.google.api.client.googleapis.batch.BatchCallback; 22 import com.google.api.client.googleapis.batch.BatchRequest; 23 import com.google.api.client.http.HttpHeaders; 24 import com.google.api.client.http.InputStreamContent; 25 import com.google.api.services.storage.Storage; 26 import com.google.api.services.storage.Storage.Objects.List; 27 import com.google.api.services.storage.model.Objects; 28 import com.google.api.services.storage.model.StorageObject; 29 30 import org.junit.After; 31 import org.junit.Assert; 32 import org.junit.Before; 33 import org.junit.Test; 34 import org.junit.runner.RunWith; 35 import org.junit.runners.JUnit4; 36 37 import java.io.ByteArrayInputStream; 38 import java.io.File; 39 import java.io.IOException; 40 import java.io.InputStream; 41 import java.nio.file.Paths; 42 import java.util.Collections; 43 44 /** {@link GCSFileDownloader} functional test. */ 45 @RunWith(JUnit4.class) 46 public class GCSFileDownloaderFuncTest { 47 48 private static final String BUCKET_NAME = "tradefed_function_test"; 49 private static final String FILE_NAME1 = "a_host_config.xml"; 50 private static final String FILE_NAME2 = "file2.txt"; 51 private static final String FILE_NAME3 = "file3.txt"; 52 private static final String FILE_NAME4 = "file4.txt"; 53 private static final String FOLDER_NAME1 = "folder1"; 54 private static final String FOLDER_NAME2 = "folder2"; 55 private static final String FILE_CONTENT = "Hello World!"; 56 57 private GCSFileDownloader mDownloader; 58 private String mRemoteRoot; 59 private File mLocalRoot; 60 private Storage mStorage; 61 createFile( Storage storage, String content, String bucketName, String... pathSegs)62 private static void createFile( 63 Storage storage, String content, String bucketName, String... pathSegs) 64 throws IOException { 65 String path = String.join("/", pathSegs); 66 StorageObject object = new StorageObject(); 67 object.setName(path); 68 storage.objects() 69 .insert( 70 bucketName, 71 object, 72 new InputStreamContent(null, new ByteArrayInputStream(content.getBytes()))) 73 .execute(); 74 } 75 76 @Before setUp()77 public void setUp() throws IOException { 78 File tempFile = 79 FileUtil.createTempFile(GCSFileDownloaderFuncTest.class.getSimpleName(), ""); 80 mRemoteRoot = tempFile.getName(); 81 FileUtil.deleteFile(tempFile); 82 mDownloader = 83 new GCSFileDownloader() { 84 85 @Override 86 File createTempFile(String remoteFilePath, File rootDir) 87 throws BuildRetrievalError { 88 try { 89 File tmpFile = 90 FileUtil.createTempFileForRemote(remoteFilePath, mLocalRoot); 91 tmpFile.delete(); 92 return tmpFile; 93 } catch (IOException e) { 94 throw new BuildRetrievalError(e.getMessage(), e); 95 } 96 } 97 }; 98 mStorage = 99 mDownloader.getStorage( 100 Collections.singleton( 101 "https://www.googleapis.com/auth/devstorage.read_write")); 102 createFile(mStorage, FILE_CONTENT, BUCKET_NAME, mRemoteRoot, FILE_NAME1); 103 createFile(mStorage, FILE_NAME2, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME2); 104 createFile(mStorage, FILE_NAME3, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME3); 105 createFile( 106 mStorage, 107 FILE_NAME4, 108 BUCKET_NAME, 109 mRemoteRoot, 110 FOLDER_NAME1, 111 FOLDER_NAME2, 112 FILE_NAME4); 113 mLocalRoot = FileUtil.createTempDir(GCSFileDownloaderFuncTest.class.getSimpleName()); 114 } 115 116 @After tearDown()117 public void tearDown() throws IOException { 118 FileUtil.recursiveDelete(mLocalRoot); 119 String pageToken = null; 120 BatchRequest batchRequest = mStorage.batch(); 121 122 while (true) { 123 List listOperation = mStorage.objects().list(BUCKET_NAME).setPrefix(mRemoteRoot); 124 if (pageToken == null) { 125 listOperation.setPageToken(pageToken); 126 } 127 Objects objects = listOperation.execute(); 128 for (StorageObject object : objects.getItems()) { 129 batchRequest.queue( 130 mStorage.objects().delete(BUCKET_NAME, object.getName()).buildHttpRequest(), 131 Void.class, 132 IOException.class, 133 new BatchCallback<Void, IOException>() { 134 @Override 135 public void onSuccess(Void arg0, HttpHeaders arg1) throws IOException {} 136 137 @Override 138 public void onFailure(IOException e, HttpHeaders arg1) 139 throws IOException { 140 throw e; 141 } 142 }); 143 } 144 pageToken = objects.getNextPageToken(); 145 if (pageToken == null) { 146 batchRequest.execute(); 147 return; 148 } 149 } 150 } 151 152 @Test testDownloadFile_streamOutput()153 public void testDownloadFile_streamOutput() throws Exception { 154 InputStream inputStream = 155 mDownloader.downloadFile(BUCKET_NAME, mRemoteRoot + "/" + FILE_NAME1); 156 String content = StreamUtil.getStringFromStream(inputStream); 157 Assert.assertEquals(FILE_CONTENT, content); 158 inputStream.reset(); 159 } 160 161 @Test testDownloadFile_streamOutput_notExist()162 public void testDownloadFile_streamOutput_notExist() throws Exception { 163 try { 164 mDownloader.downloadFile(BUCKET_NAME, mRemoteRoot + "/" + "non_exist_file"); 165 Assert.fail("Should throw IOException."); 166 } catch (IOException e) { 167 // Expect IOException 168 } 169 } 170 171 @Test testGetRemoteFileMetaData()172 public void testGetRemoteFileMetaData() throws Exception { 173 String filename = mRemoteRoot + "/" + FILE_NAME1; 174 StorageObject object = mDownloader.getRemoteFileMetaData(BUCKET_NAME, filename); 175 Assert.assertEquals(filename, object.getName()); 176 } 177 178 @Test testGetRemoteFileMetaData_notExist()179 public void testGetRemoteFileMetaData_notExist() throws Exception { 180 String filename = mRemoteRoot + "/" + "not_exist"; 181 StorageObject object = mDownloader.getRemoteFileMetaData(BUCKET_NAME, filename); 182 Assert.assertNull(object); 183 } 184 185 @Test testIsRemoteFolder()186 public void testIsRemoteFolder() throws Exception { 187 Assert.assertFalse( 188 mDownloader.isRemoteFolder( 189 BUCKET_NAME, Paths.get(mRemoteRoot, FILE_NAME1).toString())); 190 Assert.assertTrue( 191 mDownloader.isRemoteFolder( 192 BUCKET_NAME, Paths.get(mRemoteRoot, FOLDER_NAME1).toString())); 193 } 194 195 @Test testDownloadFile()196 public void testDownloadFile() throws Exception { 197 File localFile = 198 mDownloader.downloadFile( 199 String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1)); 200 String content = FileUtil.readStringFromFile(localFile); 201 Assert.assertEquals(FILE_CONTENT, content); 202 } 203 204 @Test testDownloadFile_nonExist()205 public void testDownloadFile_nonExist() throws Exception { 206 try { 207 mDownloader.downloadFile( 208 String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, "non_exist_file")); 209 Assert.fail("Should throw BuildRetrievalError."); 210 } catch (BuildRetrievalError e) { 211 // Expect BuildRetrievalError 212 } 213 } 214 215 @Test testDownloadFile_folder()216 public void testDownloadFile_folder() throws Exception { 217 File localFile = 218 mDownloader.downloadFile( 219 String.format("gs://%s/%s/%s/", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1)); 220 checkDownloadedFolder(localFile); 221 } 222 223 @Test testDownloadFile_folderNotsanitize()224 public void testDownloadFile_folderNotsanitize() throws Exception { 225 File localFile = 226 mDownloader.downloadFile( 227 String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1)); 228 checkDownloadedFolder(localFile); 229 } 230 checkDownloadedFolder(File localFile)231 private void checkDownloadedFolder(File localFile) throws Exception { 232 Assert.assertTrue(localFile.isDirectory()); 233 Assert.assertEquals(3, localFile.list().length); 234 for (String filename : localFile.list()) { 235 if (filename.equals(FILE_NAME2)) { 236 Assert.assertEquals( 237 FILE_NAME2, 238 FileUtil.readStringFromFile( 239 new File(localFile.getAbsolutePath(), filename))); 240 } else if (filename.equals(FILE_NAME3)) { 241 Assert.assertEquals( 242 FILE_NAME3, 243 FileUtil.readStringFromFile( 244 new File(localFile.getAbsolutePath(), filename))); 245 } else if (filename.equals(FOLDER_NAME2)) { 246 File subFolder = new File(localFile.getAbsolutePath(), filename); 247 Assert.assertTrue(subFolder.isDirectory()); 248 Assert.assertEquals(1, subFolder.list().length); 249 Assert.assertEquals( 250 FILE_NAME4, 251 FileUtil.readStringFromFile( 252 new File(subFolder.getAbsolutePath(), subFolder.list()[0]))); 253 } else { 254 Assert.assertTrue(String.format("Unknonwn file %s", filename), false); 255 } 256 } 257 } 258 259 @Test testDownloadFile_folder_nonExist()260 public void testDownloadFile_folder_nonExist() throws Exception { 261 try { 262 mDownloader.downloadFile( 263 String.format("gs://%s/%s/%s/", BUCKET_NAME, "mRemoteRoot", "nonExistFolder")); 264 Assert.fail("Should throw BuildRetrievalError."); 265 } catch (BuildRetrievalError e) { 266 // Expect BuildRetrievalError 267 } 268 } 269 270 @Test testCheckFreshness()271 public void testCheckFreshness() throws Exception { 272 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1); 273 File localFile = mDownloader.downloadFile(remotePath); 274 Assert.assertTrue(mDownloader.isFresh(localFile, remotePath)); 275 } 276 277 @Test testCheckFreshness_notExist()278 public void testCheckFreshness_notExist() throws Exception { 279 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1); 280 Assert.assertFalse(mDownloader.isFresh(new File("/not/exist"), remotePath)); 281 } 282 283 @Test testCheckFreshness_folderNotExist()284 public void testCheckFreshness_folderNotExist() throws Exception { 285 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 286 Assert.assertFalse(mDownloader.isFresh(new File("/not/exist"), remotePath)); 287 } 288 289 @Test testCheckFreshness_remoteNotExist()290 public void testCheckFreshness_remoteNotExist() throws Exception { 291 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1); 292 String remoteNotExistPath = String.format("gs://%s/%s/no_exist", BUCKET_NAME, mRemoteRoot); 293 File localFile = mDownloader.downloadFile(remotePath); 294 Assert.assertFalse(mDownloader.isFresh(localFile, remoteNotExistPath)); 295 } 296 297 @Test testCheckFreshness_remoteFolderNotExist()298 public void testCheckFreshness_remoteFolderNotExist() throws Exception { 299 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 300 String remoteNotExistPath = String.format("gs://%s/%s/no_exist/", BUCKET_NAME, mRemoteRoot); 301 File localFolder = mDownloader.downloadFile(remotePath); 302 Assert.assertFalse(mDownloader.isFresh(localFolder, remoteNotExistPath)); 303 } 304 305 @Test testCheckFreshness_notFresh()306 public void testCheckFreshness_notFresh() throws Exception { 307 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1); 308 File localFile = mDownloader.downloadFile(remotePath); 309 // Change the remote file. 310 createFile(mStorage, "New content.", BUCKET_NAME, mRemoteRoot, FILE_NAME1); 311 Assert.assertFalse(mDownloader.isFresh(localFile, remotePath)); 312 } 313 314 @Test testCheckFreshness_folder()315 public void testCheckFreshness_folder() throws Exception { 316 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 317 File localFolder = mDownloader.downloadFile(remotePath); 318 Assert.assertTrue(mDownloader.isFresh(localFolder, remotePath)); 319 } 320 321 @Test testCheckFreshness_folder_addFile()322 public void testCheckFreshness_folder_addFile() throws Exception { 323 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 324 File localFolder = mDownloader.downloadFile(remotePath); 325 createFile( 326 mStorage, 327 "A new file", 328 BUCKET_NAME, 329 mRemoteRoot, 330 FOLDER_NAME1, 331 FOLDER_NAME2, 332 "new_file.txt"); 333 Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath)); 334 } 335 336 @Test testCheckFreshness_folder_removeFile()337 public void testCheckFreshness_folder_removeFile() throws Exception { 338 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 339 File localFolder = mDownloader.downloadFile(remotePath); 340 mStorage.objects() 341 .delete(BUCKET_NAME, Paths.get(mRemoteRoot, FOLDER_NAME1, FILE_NAME3).toString()) 342 .execute(); 343 Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath)); 344 } 345 346 @Test testCheckFreshness_folder_changeFile()347 public void testCheckFreshness_folder_changeFile() throws Exception { 348 String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1); 349 File localFolder = mDownloader.downloadFile(remotePath); 350 createFile(mStorage, "New content", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME3); 351 Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath)); 352 } 353 } 354