1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.device.metric; 17 18 import com.android.tradefed.config.Option; 19 import com.android.tradefed.device.DeviceNotAvailableException; 20 import com.android.tradefed.device.ITestDevice; 21 import com.android.tradefed.device.TestDeviceState; 22 import com.android.tradefed.invoker.logger.CurrentInvocation; 23 import com.android.tradefed.log.LogUtil.CLog; 24 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 25 import com.android.tradefed.util.FileUtil; 26 import com.android.tradefed.util.ZipUtil; 27 import com.android.tradefed.util.proto.TfMetricProtoUtil; 28 29 import java.io.File; 30 import java.io.IOException; 31 import java.util.AbstractMap.SimpleEntry; 32 import java.util.Arrays; 33 import java.util.HashMap; 34 import java.util.LinkedHashMap; 35 import java.util.LinkedHashSet; 36 import java.util.Map; 37 import java.util.Map.Entry; 38 import java.util.Set; 39 import java.util.regex.Pattern; 40 41 /** 42 * A {@link BaseDeviceMetricCollector} that listen for metrics key coming from the device and pull 43 * them as a file from the device. Can be extended for extra-processing of the file. 44 */ 45 public abstract class FilePullerDeviceMetricCollector extends BaseDeviceMetricCollector { 46 47 @Option( 48 name = "pull-pattern-keys", 49 description = 50 "The pattern key name to be pull from the device as a file. Can be repeated.") 51 private Set<String> mKeys = new LinkedHashSet<>(); 52 53 @Option( 54 name = "directory-keys", 55 description = "Path to the directory on the device that contains the metrics.") 56 protected Set<String> mDirectoryKeys = new LinkedHashSet<>(); 57 58 @Option(name = "compress-directories", 59 description = "Compress multiple files in the matching directory into zip file") 60 private boolean mCompressDirectory = false; 61 62 @Option( 63 name = "clean-up", 64 description = "Whether to delete the file from the device after pulling it or not." 65 ) 66 private boolean mCleanUp = true; 67 68 @Option( 69 name = "collect-on-run-ended-only", 70 description = 71 "Attempt to collect the files on test run end only instead of on both test cases " 72 + "and test run ended. This is safer since test case level collection isn't" 73 + " synchronous." 74 ) 75 private boolean mCollectOnRunEndedOnly = true; 76 77 public Map<String, String> mTestCaseMetrics = new LinkedHashMap<String, String>(); 78 79 @Override onTestEnd(DeviceMetricData testData, Map<String, Metric> currentTestCaseMetrics)80 public void onTestEnd(DeviceMetricData testData, Map<String, Metric> currentTestCaseMetrics) 81 throws DeviceNotAvailableException { 82 if (mCollectOnRunEndedOnly) { 83 // Track test cases metrics in case we don't process here. 84 mTestCaseMetrics.putAll(TfMetricProtoUtil.compatibleConvert(currentTestCaseMetrics)); 85 return; 86 } 87 processMetricRequest(testData, currentTestCaseMetrics); 88 } 89 90 @Override onTestRunEnd(DeviceMetricData runData, final Map<String, Metric> currentRunMetrics)91 public void onTestRunEnd(DeviceMetricData runData, final Map<String, Metric> currentRunMetrics) 92 throws DeviceNotAvailableException { 93 processMetricRequest(runData, currentRunMetrics); 94 mTestCaseMetrics = new HashMap<>(); 95 } 96 97 /** Adds additional pattern keys to the pull from the device. */ addKeys(String... keys)98 protected void addKeys(String... keys) { 99 mKeys.addAll(Arrays.asList(keys)); 100 } 101 102 /** 103 * Implementation of the method should allow to log the file, parse it for metrics to be put in 104 * {@link DeviceMetricData}. 105 * 106 * @param key the option key associated to the file that was pulled. 107 * @param metricFile the {@link File} pulled from the device matching the option key. 108 * @param data the {@link DeviceMetricData} where metrics can be stored. 109 */ processMetricFile(String key, File metricFile, DeviceMetricData data)110 public abstract void processMetricFile(String key, File metricFile, DeviceMetricData data); 111 112 /** 113 * Implementation of the method should allow to log the directory, parse it for metrics to be 114 * put in {@link DeviceMetricData}. 115 * 116 * @param key the option key associated to the directory that was pulled. 117 * @param metricDirectory the {@link File} pulled from the device matching the option key. 118 * @param data the {@link DeviceMetricData} where metrics can be stored. 119 */ processMetricDirectory( String key, File metricDirectory, DeviceMetricData data)120 public abstract void processMetricDirectory( 121 String key, File metricDirectory, DeviceMetricData data); 122 123 /** 124 * Process the file associated with the matching key or directory name and update the data with 125 * any additional metrics. 126 * 127 * @param data where the final metrics will be stored. 128 * @param metrics where the key or directory name will be matched to the keys. 129 */ processMetricRequest(DeviceMetricData data, Map<String, Metric> metrics)130 private void processMetricRequest(DeviceMetricData data, Map<String, Metric> metrics) 131 throws DeviceNotAvailableException { 132 Map<String, String> currentMetrics = TfMetricProtoUtil 133 .compatibleConvert(metrics); 134 currentMetrics.putAll(mTestCaseMetrics); 135 if (mKeys.isEmpty() && mDirectoryKeys.isEmpty()) { 136 return; 137 } 138 Map<ITestDevice, Integer> deviceUsers = new HashMap<>(); 139 if (!mKeys.isEmpty()) { 140 for (ITestDevice device : getRealDevices()) { 141 if (!TestDeviceState.ONLINE.equals(device.getDeviceState())) { 142 CLog.d( 143 "Device '%s' is in state '%s' skipping file puller", 144 device.getSerialNumber(), device.getDeviceState()); 145 return; 146 } 147 deviceUsers.put(device, device.getCurrentUser()); 148 } 149 } 150 for (String key : mKeys) { 151 Map<String, File> pulledMetrics = pullMetricFile(key, currentMetrics, deviceUsers); 152 153 // Process all the metric files that matched the key pattern. 154 for (Map.Entry<String, File> entry : pulledMetrics.entrySet()) { 155 processMetricFile(entry.getKey(), entry.getValue(), data); 156 } 157 } 158 159 for (String key : mDirectoryKeys) { 160 Entry<String, File> pulledMetrics = pullMetricDirectory(key); 161 if (pulledMetrics != null) { 162 if (mCompressDirectory) { 163 File pulledDirectory = pulledMetrics.getValue(); 164 if (pulledDirectory.isDirectory()) { 165 try { 166 File compressedFile = ZipUtil.createZip(pulledDirectory, 167 getFileName(key)); 168 processMetricFile(key, compressedFile, data); 169 } catch (IOException e) { 170 CLog.e("Unable to compress the directory."); 171 } 172 FileUtil.recursiveDelete(pulledDirectory); 173 } 174 continue; 175 } 176 processMetricDirectory(pulledMetrics.getKey(), pulledMetrics.getValue(), data); 177 } 178 } 179 } 180 181 /** 182 * Return the last folder name from the path the in the device where the 183 * directory is pulled. 184 */ getFileName(String key)185 private String getFileName(String key) { 186 return key.substring(key.lastIndexOf("/")+1); 187 } 188 pullMetricFile( String pattern, final Map<String, String> currentMetrics, Map<ITestDevice, Integer> deviceUsers)189 private Map<String, File> pullMetricFile( 190 String pattern, 191 final Map<String, String> currentMetrics, 192 Map<ITestDevice, Integer> deviceUsers) 193 throws DeviceNotAvailableException { 194 Map<String, File> matchedFiles = new HashMap<>(); 195 Pattern p = Pattern.compile(pattern); 196 197 for (Entry<String, String> entry : currentMetrics.entrySet()) { 198 if (p.matcher(entry.getKey()).find()) { 199 for (ITestDevice device : getRealDevices()) { 200 if (!shouldCollect(device)) { 201 continue; 202 } 203 try { 204 File attemptPull = 205 retrieveFile(device, entry.getValue(), deviceUsers.get(device)); 206 if (attemptPull != null) { 207 if (mCleanUp) { 208 device.deleteFile(entry.getValue()); 209 } 210 // Store all the keys that matches the pattern and the corresponding 211 // files pulled from the device. 212 matchedFiles.put(entry.getKey(), attemptPull); 213 } 214 } catch (RuntimeException e) { 215 CLog.e( 216 "Exception when pulling metric file '%s' from %s", 217 entry.getValue(), device.getSerialNumber()); 218 CLog.e(e); 219 } 220 } 221 } 222 } 223 224 if (matchedFiles.isEmpty()) { 225 // Not a hard failure, just nice to know 226 CLog.d("Could not find a device file associated to pattern '%s'.", pattern); 227 228 } 229 return matchedFiles; 230 } 231 232 /** 233 * Pull the file from the specified path in the device. 234 * 235 * @param device which has the file. 236 * @param remoteFilePath location in the device. 237 * @param userId the user id to pull from 238 * @return File retrieved from the given path in the device. 239 * @throws DeviceNotAvailableException 240 */ retrieveFile(ITestDevice device, String remoteFilePath, int userId)241 protected File retrieveFile(ITestDevice device, String remoteFilePath, int userId) 242 throws DeviceNotAvailableException { 243 return device.pullFile(remoteFilePath, userId); 244 } 245 246 /** 247 * Pulls the directory and all its content from the device and save it in the host under the 248 * metric_tmp folder. 249 * 250 * @param keyDirectory path to the source directory in the device. 251 * @return Key,value pair of the directory name and path to the directory in the local host. 252 */ pullMetricDirectory(String keyDirectory)253 private Entry<String, File> pullMetricDirectory(String keyDirectory) 254 throws DeviceNotAvailableException { 255 try { 256 File tmpDestDir = 257 FileUtil.createTempDir("metric_tmp", CurrentInvocation.getWorkFolder()); 258 for (ITestDevice device : getRealDevices()) { 259 if (!shouldCollect(device)) { 260 continue; 261 } 262 try { 263 if (device.pullDir(keyDirectory, tmpDestDir)) { 264 if (mCleanUp) { 265 device.deleteFile(keyDirectory); 266 } 267 return new SimpleEntry<String, File>(keyDirectory, tmpDestDir); 268 } 269 } catch (RuntimeException e) { 270 CLog.e( 271 "Exception when pulling directory '%s' from %s", 272 keyDirectory, device.getSerialNumber()); 273 CLog.e(e); 274 } 275 } 276 } catch (IOException ioe) { 277 CLog.e("Exception while creating the local directory"); 278 CLog.e(ioe); 279 } 280 CLog.e("Could not find a device directory associated to path '%s'.", keyDirectory); 281 return null; 282 } 283 shouldCollect(ITestDevice device)284 private boolean shouldCollect(ITestDevice device) { 285 TestDeviceState state = device.getDeviceState(); 286 if (!TestDeviceState.ONLINE.equals(state)) { 287 CLog.d("Skip %s device is in state '%s'", this, state); 288 return false; 289 } 290 return true; 291 } 292 } 293