• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.cluster;
17 
18 import com.android.tradefed.log.LogUtil.CLog;
19 import com.android.tradefed.util.CommandResult;
20 import com.android.tradefed.util.CommandStatus;
21 import com.android.tradefed.util.FileUtil;
22 import com.android.tradefed.util.IRunUtil;
23 import com.android.tradefed.util.RunUtil;
24 
25 import com.google.common.annotations.VisibleForTesting;
26 
27 import com.google.common.net.UrlEscapers;
28 import java.io.File;
29 import java.io.IOException;
30 import java.net.MalformedURLException;
31 import java.net.URL;
32 import java.util.Objects;
33 import java.util.stream.Collectors;
34 import java.util.stream.Stream;
35 
36 /** Uploads test output files to local file system, GCS, or an HTTP(S) endpoint. */
37 public class TestOutputUploader {
38 
39     static final long UPLOAD_TIMEOUT_MS = 30 * 60 * 1000;
40     static final long RETRY_INTERVAL_MS = 10 * 1000;
41     static final int MAX_RETRY_COUNT = 2;
42     static final String FILE_PROTOCOL = "file";
43     static final String GCS_PROTOCOL = "gs";
44     static final String HTTP_PROTOCOL = "http";
45     static final String HTTPS_PROTOCOL = "https";
46 
47     private String mUploadUrl = null;
48     private String mProtocol = null;
49     private IRunUtil mRunUtil = null;
50 
setUploadUrl(final String url)51     public void setUploadUrl(final String url) throws MalformedURLException {
52         mUploadUrl = url;
53         URL urlObj = new URL(url);
54         mProtocol = urlObj.getProtocol();
55     }
56 
57     /**
58      * Upload a file to the specified destination path (relative to the upload URL).
59      *
60      * @param file file to upload
61      * @param destPath relative destination path
62      * @return uploaded file URL
63      */
uploadFile(File file, String destPath)64     public String uploadFile(File file, String destPath) throws IOException {
65         if (mUploadUrl == null) {
66             throw new IllegalStateException("Upload URL is not set");
67         }
68         String uploadUrl = joinSegments(mUploadUrl, destPath);
69         CLog.i("Uploading %s to %s", file.getAbsolutePath(), uploadUrl);
70 
71         switch (mProtocol) {
72             case FILE_PROTOCOL:
73                 File destDir = new File(new URL(uploadUrl).getPath());
74                 destDir.mkdirs();
75                 File destFile = new File(destDir, file.getName());
76                 FileUtil.copyFile(file, destFile);
77                 return joinSegments(uploadUrl, file.getName());
78             case GCS_PROTOCOL:
79                 executeUploadCommand(file, "gsutil", "cp", file.getAbsolutePath(), uploadUrl);
80                 return joinSegments(uploadUrl, file.getName());
81             case HTTP_PROTOCOL:
82             case HTTPS_PROTOCOL:
83                 // Upload URL for HTTP(S) protocols should include the filename, and special
84                 // characters in the paths should be escaped.
85                 String fullPath = joinSegments(destPath, file.getName());
86                 String encodedPath = UrlEscapers.urlFragmentEscaper().escape(fullPath);
87                 uploadUrl = joinSegments(mUploadUrl, encodedPath);
88                 executeUploadCommand(
89                         file,
90                         "curl",
91                         "--request",
92                         "POST",
93                         "--form",
94                         "file=@" + file.getAbsolutePath(),
95                         "--fail", // Return non-zero status code on error.
96                         "--location", // Handle redirects.
97                         uploadUrl);
98                 return uploadUrl;
99             default:
100                 throw new IllegalArgumentException(
101                         String.format("Protocol '%s' is not supported", mProtocol));
102         }
103     }
104 
105     /** Executes an upload command and handle the result. */
executeUploadCommand(File file, String... cmdArgs)106     private void executeUploadCommand(File file, String... cmdArgs) {
107         CommandResult result =
108                 getRunUtil()
109                         .runTimedCmdRetry(
110                                 UPLOAD_TIMEOUT_MS, RETRY_INTERVAL_MS, MAX_RETRY_COUNT, cmdArgs);
111         if (!CommandStatus.SUCCESS.equals(result.getStatus())) {
112             String error =
113                     String.format(
114                             "Failed to upload %s, status = %s, stdout = [%s], stderr = [%s].",
115                             file.getAbsolutePath(),
116                             result.getStatus(),
117                             result.getStdout(),
118                             result.getStderr());
119             CLog.e(error);
120             throw new RuntimeException(error);
121         }
122     }
123 
124     /** Joins path segments with slashes. */
joinSegments(String... segments)125     private String joinSegments(String... segments) {
126         return Stream.of(segments)
127                 .filter(Objects::nonNull) // Ignore null segments.
128                 .map(s -> s.replaceAll("^/|/$", "")) // Remove leading/trailing slashes.
129                 .collect(Collectors.joining("/"));
130     }
131 
132     @VisibleForTesting
getRunUtil()133     IRunUtil getRunUtil() {
134         if (mRunUtil == null) {
135             mRunUtil = RunUtil.getDefault();
136         }
137         return mRunUtil;
138     }
139 }
140