1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.cluster; 17 18 import com.android.tradefed.log.LogUtil.CLog; 19 import com.android.tradefed.util.CommandResult; 20 import com.android.tradefed.util.CommandStatus; 21 import com.android.tradefed.util.FileUtil; 22 import com.android.tradefed.util.IRunUtil; 23 import com.android.tradefed.util.RunUtil; 24 25 import com.google.common.annotations.VisibleForTesting; 26 27 import com.google.common.net.UrlEscapers; 28 import java.io.File; 29 import java.io.IOException; 30 import java.net.MalformedURLException; 31 import java.net.URL; 32 import java.util.Objects; 33 import java.util.stream.Collectors; 34 import java.util.stream.Stream; 35 36 /** Uploads test output files to local file system, GCS, or an HTTP(S) endpoint. */ 37 public class TestOutputUploader { 38 39 static final long UPLOAD_TIMEOUT_MS = 30 * 60 * 1000; 40 static final long RETRY_INTERVAL_MS = 10 * 1000; 41 static final int MAX_RETRY_COUNT = 2; 42 static final String FILE_PROTOCOL = "file"; 43 static final String GCS_PROTOCOL = "gs"; 44 static final String HTTP_PROTOCOL = "http"; 45 static final String HTTPS_PROTOCOL = "https"; 46 47 private String mUploadUrl = null; 48 private String mProtocol = null; 49 private IRunUtil mRunUtil = null; 50 setUploadUrl(final String url)51 public void setUploadUrl(final String url) throws MalformedURLException { 52 mUploadUrl = url; 53 URL urlObj = new URL(url); 54 mProtocol = urlObj.getProtocol(); 55 } 56 57 /** 58 * Upload a file to the specified destination path (relative to the upload URL). 59 * 60 * @param file file to upload 61 * @param destPath relative destination path 62 * @return uploaded file URL 63 */ uploadFile(File file, String destPath)64 public String uploadFile(File file, String destPath) throws IOException { 65 if (mUploadUrl == null) { 66 throw new IllegalStateException("Upload URL is not set"); 67 } 68 String uploadUrl = joinSegments(mUploadUrl, destPath); 69 CLog.i("Uploading %s to %s", file.getAbsolutePath(), uploadUrl); 70 71 switch (mProtocol) { 72 case FILE_PROTOCOL: 73 File destDir = new File(new URL(uploadUrl).getPath()); 74 destDir.mkdirs(); 75 File destFile = new File(destDir, file.getName()); 76 FileUtil.copyFile(file, destFile); 77 return joinSegments(uploadUrl, file.getName()); 78 case GCS_PROTOCOL: 79 executeUploadCommand(file, "gsutil", "cp", file.getAbsolutePath(), uploadUrl); 80 return joinSegments(uploadUrl, file.getName()); 81 case HTTP_PROTOCOL: 82 case HTTPS_PROTOCOL: 83 // Upload URL for HTTP(S) protocols should include the filename, and special 84 // characters in the paths should be escaped. 85 String fullPath = joinSegments(destPath, file.getName()); 86 String encodedPath = UrlEscapers.urlFragmentEscaper().escape(fullPath); 87 uploadUrl = joinSegments(mUploadUrl, encodedPath); 88 executeUploadCommand( 89 file, 90 "curl", 91 "--request", 92 "POST", 93 "--form", 94 "file=@" + file.getAbsolutePath(), 95 "--fail", // Return non-zero status code on error. 96 "--location", // Handle redirects. 97 uploadUrl); 98 return uploadUrl; 99 default: 100 throw new IllegalArgumentException( 101 String.format("Protocol '%s' is not supported", mProtocol)); 102 } 103 } 104 105 /** Executes an upload command and handle the result. */ executeUploadCommand(File file, String... cmdArgs)106 private void executeUploadCommand(File file, String... cmdArgs) { 107 CommandResult result = 108 getRunUtil() 109 .runTimedCmdRetry( 110 UPLOAD_TIMEOUT_MS, RETRY_INTERVAL_MS, MAX_RETRY_COUNT, cmdArgs); 111 if (!CommandStatus.SUCCESS.equals(result.getStatus())) { 112 String error = 113 String.format( 114 "Failed to upload %s, status = %s, stdout = [%s], stderr = [%s].", 115 file.getAbsolutePath(), 116 result.getStatus(), 117 result.getStdout(), 118 result.getStderr()); 119 CLog.e(error); 120 throw new RuntimeException(error); 121 } 122 } 123 124 /** Joins path segments with slashes. */ joinSegments(String... segments)125 private String joinSegments(String... segments) { 126 return Stream.of(segments) 127 .filter(Objects::nonNull) // Ignore null segments. 128 .map(s -> s.replaceAll("^/|/$", "")) // Remove leading/trailing slashes. 129 .collect(Collectors.joining("/")); 130 } 131 132 @VisibleForTesting getRunUtil()133 IRunUtil getRunUtil() { 134 if (mRunUtil == null) { 135 mRunUtil = RunUtil.getDefault(); 136 } 137 return mRunUtil; 138 } 139 } 140