1 /* 2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.util.avd; 18 19 import com.android.tradefed.log.LogUtil.CLog; 20 import com.android.tradefed.util.FileUtil; 21 import com.android.tradefed.util.gcs.GCSFileDownloaderBase; 22 23 import java.io.File; 24 import java.io.FileInputStream; 25 import java.nio.file.Files; 26 import java.util.AbstractMap; 27 import java.util.ArrayList; 28 import java.util.Collections; 29 import java.util.List; 30 import java.util.Map; 31 import java.util.Scanner; 32 import java.util.Set; 33 import java.util.regex.Matcher; 34 import java.util.regex.Pattern; 35 import java.util.stream.Collectors; 36 import java.util.stream.Stream; 37 38 /** A utility for collecting logs from AVD and host VM */ 39 public class LogCollector { 40 // Maximum size of tailing part of a file to search for error signature. 41 private static final long MAX_FILE_SIZE_FOR_ERROR = 10 * 1024 * 1024; 42 43 // A map of log file name pattern to log content that the log must have. 44 private static final Map<Pattern, AbstractMap.SimpleEntry<String, String>> 45 REMOTE_LOG_NAME_PATTERN_TO_LOG_MUST_HAVE_SIGNATURE_MAP = 46 Stream.of( 47 new AbstractMap.SimpleEntry<>( 48 Pattern.compile(".*fetch.*"), 49 new AbstractMap.SimpleEntry<>( 50 "Completed all fetches", 51 "fetch_cvd_failure_general"))) 52 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); 53 private static final Map<Pattern, AbstractMap.SimpleEntry<String, String>> 54 REMOTE_LOG_NAME_PATTERN_TO_ERROR_SIGNATURE_MAP = 55 Stream.of( 56 new AbstractMap.SimpleEntry<>( 57 Pattern.compile(".*launcher.*"), 58 new AbstractMap.SimpleEntry<>( 59 "Address already in use", 60 "launch_cvd_port_collision")), 61 new AbstractMap.SimpleEntry<>( 62 Pattern.compile(".*launcher.*"), 63 new AbstractMap.SimpleEntry<>( 64 "vcpu hw run failure: 0x7", 65 "crosvm_vcpu_hw_run_failure_7")), 66 new AbstractMap.SimpleEntry<>( 67 Pattern.compile(".*(launcher|vdl_stdout).*"), 68 new AbstractMap.SimpleEntry<>( 69 "failed to initialize fetch system images", 70 "fetch_cvd_failure")), 71 new AbstractMap.SimpleEntry<>( 72 Pattern.compile(".*(vdl_stdout|fetch).*"), 73 new AbstractMap.SimpleEntry<>( 74 "Could not resolve host: ", 75 "fetch_cvd_failure_resolve_host")), 76 new AbstractMap.SimpleEntry<>( 77 Pattern.compile(".*vdl_stdout.*"), 78 new AbstractMap.SimpleEntry<>( 79 "Could not connect to server", 80 "fetch_cvd_failure_connect_server")), 81 new AbstractMap.SimpleEntry<>( 82 Pattern.compile(".*vdl_stdout.*"), 83 new AbstractMap.SimpleEntry<>( 84 // TODO(b/395472945): remove by 4/1/2025 85 "Unable to download", 86 "fetch_cvd_failure_artifact_not_found")), 87 new AbstractMap.SimpleEntry<>( 88 Pattern.compile(".*vdl_stdout.*"), 89 new AbstractMap.SimpleEntry<>( 90 "Failed to download file: File Not Found", 91 "fetch_cvd_failure_artifact_not_found")), 92 new AbstractMap.SimpleEntry<>( 93 Pattern.compile(".*launcher.*"), 94 new AbstractMap.SimpleEntry<>( 95 "failed to read from socket, retry", 96 "rootcanal_socket_error")), 97 new AbstractMap.SimpleEntry<>( 98 Pattern.compile(".*launcher.*"), 99 new AbstractMap.SimpleEntry<>( 100 "VIRTUAL_DEVICE_BOOT_PENDING: Bluetooth", 101 "bluetooth_pending")), 102 new AbstractMap.SimpleEntry<>( 103 Pattern.compile(".*launcher.*"), 104 new AbstractMap.SimpleEntry<>( 105 "another cuttlefish device already running", 106 "another_device_running")), 107 new AbstractMap.SimpleEntry<>( 108 Pattern.compile(".*launcher.*"), 109 new AbstractMap.SimpleEntry<>( 110 "Setup failed for cuttlefish::ConfigServer", 111 "config_server_failed")), 112 new AbstractMap.SimpleEntry<>( 113 Pattern.compile(".*(launcher|kernel|logcat).*"), 114 new AbstractMap.SimpleEntry<>( 115 "VIRTUAL_DEVICE_BOOT_FAILED: Dependencies not" 116 + " ready after 10 checks: Bluetooth", 117 "bluetooth_failed")), 118 new AbstractMap.SimpleEntry<>( 119 Pattern.compile("^logcat.*"), 120 new AbstractMap.SimpleEntry<>( 121 "System zygote died with fatal exception", 122 "zygote_fatal_exception")), 123 new AbstractMap.SimpleEntry<>( 124 Pattern.compile("^logcat.*"), 125 new AbstractMap.SimpleEntry<>( 126 "mkdir failed: errno 117 (Structure needs" 127 + " cleaning)", 128 "filesystem_corrupt")), 129 new AbstractMap.SimpleEntry<>( 130 Pattern.compile(".*kernel.*"), 131 new AbstractMap.SimpleEntry<>( 132 "Kernel panic - not syncing: VFS: Unable to" 133 + " mount root fs on unknown-block", 134 "cf_ramdisk_mount_failure")), 135 new AbstractMap.SimpleEntry<>( 136 Pattern.compile(".*launcher.*"), 137 new AbstractMap.SimpleEntry<>( 138 "BluetoothShellCommand:" 139 + " wait-for-state:STATE_OFF: Failed with" 140 + " status=-1", 141 "bluetooth_failed_to_stop")), 142 new AbstractMap.SimpleEntry<>( 143 Pattern.compile(".*launcher.*"), 144 new AbstractMap.SimpleEntry<>( 145 "Assertion `mutex->__data.__owner == 0' failed", 146 "cf_webrtc_crash"))) 147 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); 148 149 /** 150 * Download error logs from GCS when Oxygen failed to launch a virtual device. 151 * 152 * @param errorMessage error message raised when leasing device through Oxygen service. 153 * @param downloader: {@link GCSFileDownloaderBase} used to download files from GCS. If set to 154 * null, the default downloader will be used, which only supports default credential (e.g., 155 * service account used by the node). * 156 * @return The {@link File} object of directory storing the logs. 157 */ downloadLaunchFailureLogs( String errorMessage, GCSFileDownloaderBase downloader)158 public static File downloadLaunchFailureLogs( 159 String errorMessage, GCSFileDownloaderBase downloader) { 160 CLog.d("Downloading device launch failure logs based on error message: %s", errorMessage); 161 Pattern pattern = Pattern.compile(".*/storage/browser/(.*)\\?&project=.*", Pattern.DOTALL); 162 Matcher matcher = pattern.matcher(errorMessage); 163 if (!matcher.find()) { 164 CLog.d("Error message doesn't contain expected GCS link."); 165 return null; 166 } 167 String remoteFilePath = "gs://" + matcher.group(1); 168 File localDir; 169 try { 170 if (downloader == null) { 171 downloader = new GCSFileDownloaderBase(); 172 } 173 return downloader.downloadFile(remoteFilePath); 174 } catch (Exception e) { 175 CLog.e("Failed to download Oxygen log from %s", remoteFilePath); 176 CLog.e(e); 177 return null; 178 } 179 } 180 181 /** 182 * Collect oxygen version info from oxygeen_version.txt. 183 * 184 * @param logDir directory of logs pulled from remote host. 185 * @return a string of Oxygen version 186 */ collectOxygenVersion(File logDir)187 public static String collectOxygenVersion(File logDir) { 188 CLog.d("Collect Oxygen version from logs under: %s.", logDir); 189 try { 190 Set<String> files = FileUtil.findFiles(logDir, "^oxygen_version\\.txt.*"); 191 if (files.size() == 0) { 192 CLog.d("There is no oxygen_version.txt found."); 193 return null; 194 } 195 // Trim the tailing spaces and line breakers at the end of the string. 196 return FileUtil.readStringFromFile(new File(files.iterator().next())) 197 .replaceAll("(?s)\\n+$", "") 198 .trim(); 199 } catch (Exception e) { 200 CLog.e("Failed to read oxygen_version.txt ."); 201 CLog.e(e); 202 return null; 203 } 204 } 205 206 /** 207 * Collect error signatures from logs. 208 * 209 * @param logPath directory of logs pulled from remote host, or a single file to search for 210 * error signatures. 211 * @return a list of error signatures. 212 */ collectErrorSignatures(File logPath)213 public static List<String> collectErrorSignatures(File logPath) { 214 CLog.d("Collect error signature from logs under: %s.", logPath); 215 List<String> signatures = new ArrayList<>(); 216 try { 217 Set<String> files; 218 if (logPath.isDirectory()) { 219 files = FileUtil.findFiles(logPath, ".*"); 220 } else { 221 files = Set.of(logPath.getAbsolutePath()); 222 } 223 for (String f : files) { 224 File file = new File(f); 225 if (file.isDirectory()) { 226 continue; 227 } 228 String fileName = file.getName(); 229 List<AbstractMap.SimpleEntry<String, String>> pairs = new ArrayList<>(); 230 for (Map.Entry<Pattern, AbstractMap.SimpleEntry<String, String>> entry : 231 REMOTE_LOG_NAME_PATTERN_TO_ERROR_SIGNATURE_MAP.entrySet()) { 232 Matcher matcher = entry.getKey().matcher(fileName); 233 if (matcher.find()) { 234 pairs.add(entry.getValue()); 235 } 236 } 237 List<AbstractMap.SimpleEntry<String, String>> pairsMustHave = new ArrayList<>(); 238 for (Map.Entry<Pattern, AbstractMap.SimpleEntry<String, String>> entry : 239 REMOTE_LOG_NAME_PATTERN_TO_LOG_MUST_HAVE_SIGNATURE_MAP.entrySet()) { 240 Matcher matcher = entry.getKey().matcher(fileName); 241 if (matcher.find()) { 242 pairsMustHave.add(entry.getValue()); 243 } 244 } 245 if (pairs.size() == 0 && pairsMustHave.size() == 0) { 246 continue; 247 } 248 try (FileInputStream stream = new FileInputStream(file)) { 249 long skipSize = Files.size(file.toPath()) - MAX_FILE_SIZE_FOR_ERROR; 250 if (skipSize > 0) { 251 stream.skip(skipSize); 252 } 253 try (Scanner scanner = new Scanner(stream)) { 254 List<AbstractMap.SimpleEntry<String, String>> pairsToRemove = 255 new ArrayList<>(); 256 List<AbstractMap.SimpleEntry<String, String>> pairsMustHaveToRemove = 257 new ArrayList<>(); 258 while (scanner.hasNextLine()) { 259 String line = scanner.nextLine(); 260 for (AbstractMap.SimpleEntry<String, String> pair : pairs) { 261 if (line.indexOf(pair.getKey()) != -1) { 262 pairsToRemove.add(pair); 263 signatures.add(pair.getValue()); 264 } 265 } 266 for (AbstractMap.SimpleEntry<String, String> pair : pairsMustHave) { 267 if (line.indexOf(pair.getKey()) != -1) { 268 pairsMustHaveToRemove.add(pair); 269 } 270 } 271 if (pairsToRemove.size() > 0) { 272 pairs.removeAll(pairsToRemove); 273 } 274 if (pairsMustHaveToRemove.size() > 0) { 275 pairsMustHave.removeAll(pairsMustHaveToRemove); 276 } 277 if (pairs.size() == 0 && pairsMustHave.size() == 0) { 278 break; 279 } 280 } 281 if (pairsMustHave.size() > 0) { 282 for (AbstractMap.SimpleEntry<String, String> pair : pairsMustHave) { 283 signatures.add(pair.getValue()); 284 } 285 } 286 } 287 } 288 } 289 } catch (Exception e) { 290 CLog.e("Failed to collect error signature."); 291 CLog.e(e); 292 } 293 Collections.sort(signatures); 294 return signatures; 295 } 296 297 /** 298 * Collect device launcher metrics from vdl_stdout. 299 * 300 * @param logDir directory of logs pulled from remote host. 301 * @return a list of launch metrics: [fetch_time, launch_time] 302 */ collectDeviceLaunchMetrics(File logDir)303 public static long[] collectDeviceLaunchMetrics(File logDir) { 304 CLog.d("Collect device launcher metrics from logs under: %s.", logDir); 305 long[] metrics = {-1, -1}; 306 try { 307 Set<String> files = FileUtil.findFiles(logDir, "^vdl_stdout\\.txt.*"); 308 if (files.size() == 0) { 309 CLog.d("There is no vdl_stdout.txt found."); 310 return metrics; 311 } 312 File vdlStdout = new File(files.iterator().next()); 313 // Keep collecting cuttlefish-common for legacy 314 double cuttlefishCommon = 0; 315 // cuttlefish-host-resources and cuttlefish-operator replaces cuttlefish-common 316 // in recent versions of cuttlefish debian packages. 317 double cuttlefishHostResources = 0; 318 double cuttlefishOperator = 0; 319 double launchDevice = 0; 320 double mainstart = 0; 321 Pattern cuttlefishCommonPatteren = 322 Pattern.compile(".*\\|\\s*(\\d+\\.\\d+)\\s*\\|\\sCuttlefishCommon"); 323 Pattern cuttlefishHostResourcesPatteren = 324 Pattern.compile(".*\\|\\s*(\\d+\\.\\d+)\\s*\\|\\sCuttlefishHostResources"); 325 Pattern cuttlefishOperatorPatteren = 326 Pattern.compile(".*\\|\\s*(\\d+\\.\\d+)\\s*\\|\\sCuttlefishOperator"); 327 Pattern launchDevicePatteren = 328 Pattern.compile(".*\\|\\s*(\\d+\\.\\d+)\\s*\\|\\sLaunchDevice"); 329 Pattern mainstartPatteren = 330 Pattern.compile(".*\\|\\s*(\\d+\\.\\d+)\\s*\\|\\sCuttlefishLauncherMainstart"); 331 try (Scanner scanner = new Scanner(vdlStdout)) { 332 boolean metricsPending = false; 333 while (scanner.hasNextLine()) { 334 String line = scanner.nextLine(); 335 if (!metricsPending) { 336 if (line.indexOf("launch_cvd exited") != -1) { 337 metricsPending = true; 338 } else { 339 continue; 340 } 341 } 342 Matcher matcher; 343 if (cuttlefishCommon == 0) { 344 matcher = cuttlefishCommonPatteren.matcher(line); 345 if (matcher.find()) { 346 cuttlefishCommon = Double.parseDouble(matcher.group(1)); 347 } 348 } 349 if (cuttlefishHostResources == 0) { 350 matcher = cuttlefishHostResourcesPatteren.matcher(line); 351 if (matcher.find()) { 352 cuttlefishHostResources = Double.parseDouble(matcher.group(1)); 353 } 354 } 355 if (cuttlefishOperator == 0) { 356 matcher = cuttlefishOperatorPatteren.matcher(line); 357 if (matcher.find()) { 358 cuttlefishOperator = Double.parseDouble(matcher.group(1)); 359 } 360 } 361 if (launchDevice == 0) { 362 matcher = launchDevicePatteren.matcher(line); 363 if (matcher.find()) { 364 launchDevice = Double.parseDouble(matcher.group(1)); 365 } 366 } 367 if (mainstart == 0) { 368 matcher = mainstartPatteren.matcher(line); 369 if (matcher.find()) { 370 mainstart = Double.parseDouble(matcher.group(1)); 371 } 372 } 373 } 374 } 375 if (mainstart > 0) { 376 metrics[0] = 377 (long) 378 ((mainstart 379 - launchDevice 380 - cuttlefishCommon 381 - cuttlefishHostResources 382 - cuttlefishOperator) 383 * 1000); 384 metrics[1] = (long) (launchDevice * 1000); 385 } 386 } catch (Exception e) { 387 CLog.e("Failed to parse device launch time from vdl_stdout.txt."); 388 CLog.e(e); 389 } 390 return metrics; 391 } 392 } 393