1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.invoker.logger; 17 18 import com.android.tradefed.log.LogUtil.CLog; 19 20 import java.util.Collections; 21 import java.util.HashMap; 22 import java.util.Map; 23 24 /** A utility class for an invocation to log some metrics. */ 25 public class InvocationMetricLogger { 26 27 /** Some special named key that we will always populate for the invocation. */ 28 public enum InvocationMetricKey { 29 WIFI_AP_NAME("wifi_ap_name", false), 30 WIFI_CONNECT_TIME("wifi_connect_time", true), 31 WIFI_CONNECT_COUNT("wifi_connect_count", true), 32 WIFI_CONNECT_RETRY_COUNT("wifi_connect_retry_count", true), 33 WIFI_HELPER_V2("wifi_helper_v2", false), 34 // Bugreport time and count 35 BUGREPORT_TIME("bugreport_time", true), 36 BUGREPORT_COUNT("bugreport_count", true), 37 ANR_TIME("anr_time", true), 38 ANR_COUNT("anr_count", true), 39 // Logcat dump time and count 40 LOGCAT_DUMP_TIME("logcat_dump_time", true), 41 LOGCAT_DUMP_COUNT("logcat_dump_count", true), 42 CLEARED_RUN_ERROR("cleared_run_error", true), 43 FETCH_BUILD("fetch_build_time_ms", true), 44 SETUP("setup_time_ms", true), 45 SHARDING_DEVICE_SETUP_TIME("remote_device_sharding_setup_ms", true), 46 AUTO_RETRY_TIME("auto_retry_time_ms", true), 47 BACKFILL_BUILD_INFO("backfill_build_info", false), 48 STAGE_TESTS_TIME("stage_tests_time_ms", true), 49 STAGE_REMOTE_TIME("stage_remote_time_ms", true), 50 STAGE_TESTS_BYTES("stage_tests_bytes", true), 51 STAGE_TESTS_INDIVIDUAL_DOWNLOADS("stage_tests_individual_downloads", true), 52 STAGE_UNDEFINED_DEPENDENCY("stage_undefined_dependency", true), 53 SERVER_REFERENCE("server_reference", false), 54 INSTRUMENTATION_RERUN_FROM_FILE("instrumentation_rerun_from_file", true), 55 INSTRUMENTATION_RERUN_SERIAL("instrumentation_rerun_serial", true), 56 DOWNLOAD_RETRY_COUNT("download_retry_count", true), 57 METADATA_RETRY_COUNT("metadata_retry_count", true), 58 XTS_STAGE_TESTS_TIME("xts_stage_tests_time_ms", true), 59 XTS_STAGE_TESTS_BYTES("xts_stage_tests_bytes", true), 60 XTS_PARTIAL_DOWNLOAD_SUCCESS_COUNT("xts_partial_download_success_count", true), 61 XTS_PARTIAL_DOWNLOAD_UNSUPPORTED_FILTER_FALLBACK_COUNT( 62 "xts_partial_download_unsupported_filter_fallback_count", true), 63 XTS_PARTIAL_DOWNLOAD_FALLBACK_COUNT("xts_partial_download_fallback_count", true), 64 XTS_PARTIAL_DOWNLOAD_UNFOUND_MODULES("xts_partial_download_unfound_modules", true), 65 XTS_PARTIAL_DOWNLOAD_TOTAL_COUNT("xts_partial_download_total_count", true), 66 SANDBOX_JAR_STAGING_PARTIAL_DOWNLOAD_FEATURE_COUNT( 67 "sandbox_jar_staging_partial_download_FEATURE_count", true), 68 SANDBOX_JAR_STAGING_PARTIAL_DOWNLOAD_SUCCESS_COUNT( 69 "sandbox_jar_staging_partial_download_SUCCESS_count", true), 70 TEST_DISCOVERY_MODULE_COUNT("test_discovery_module_count", true), 71 // -- Disk memory usage -- 72 // Approximate peak disk space usage of the invocation 73 // Represent files that would usually live for the full invocation (min usage) 74 TEAR_DOWN_DISK_USAGE("teardown_disk_usage_bytes", false), 75 // Recovery Mode 76 AUTO_RECOVERY_MODE_COUNT("recovery_mode_count", true), 77 ATTEMPT_RECOVERY_LOG_COUNT("attempt_pull_recovery_log", true), 78 // Represents the time we spend attempting to recover a device. 79 RECOVERY_TIME("recovery_time", true), 80 // Represents how often we enter the recover device routine. 81 RECOVERY_ROUTINE_COUNT("recovery_routine_count", true), 82 // Represents the time we spend attempting to "adb root" a device. 83 ADB_ROOT_TIME("adb_root_time", true), 84 // Represents how often we enter the "adb root" device routine. 85 ADB_ROOT_ROUTINE_COUNT("adb_root_routine_count", true), 86 // Represents the time we spend attempting to reboot a device. 87 ADB_REBOOT_TIME("adb_reboot_time", true), 88 // Represents how often we attempt to reboot the device. 89 ADB_REBOOT_ROUTINE_COUNT("adb_reboot_routine_count", true), 90 // Represents the time attempting to reboot a device into bootloader 91 BOOTLOADER_REBOOT_TIME("bootloader_reboot_time", true), 92 // Represents how often we attempt to reboot the device into bootloader 93 BOOTLOADER_REBOOT_COUNT("bootloader_reboot_count", true), 94 // Represents the time attempting to reboot a device into fastbootd 95 FASTBOOTD_REBOOT_TIME("fastbootd_reboot_time", true), 96 // Represents how often we attempt to reboot the device into fastbootd 97 FASTBOOTD_REBOOT_COUNT("fastbootd_reboot_count", true), 98 // Represents how often we reboot a device already in bootloader 99 BOOTLOADER_SAME_STATE_REBOOT("bootloader_same_state_reboot", true), 100 // Represents the time we spend during postboot setup 101 POSTBOOT_SETUP_TIME("postboot_setup_time", true), 102 // Represents how often we go through postboot setup 103 POSTBOOT_SETUP_COUNT("postboot_setup_count", true), 104 // Represents the time we spend during postboot wifi setup 105 POSTBOOT_WIFI_SETUP_TIME("postboot_wifi_setup_time", true), 106 // Represents how often we go through postboot wifi setup 107 POSTBOOT_WIFI_SETUP_COUNT("postboot_wifi_setup_count", true), 108 // Represents the time we spend during md5 calculation 109 MD5_CALCULATION_TIME("md5_calculation_time", true), 110 // Represents how often we go through md5 calculation 111 MD5_CALCULATION_COUNT("md5_calculation_count", true), 112 113 // Represents the time we spend pulling file from device. 114 PULL_FILE_TIME("pull_file_time_ms", true), 115 // Represents how many times we pulled file from the device. 116 PULL_FILE_COUNT("pull_file_count", true), 117 // Represents the time we spend pulling dir from device. 118 PULL_DIR_TIME("pull_dir_time_ms", true), 119 // Represents how many times we pulled dir from the device. 120 PULL_DIR_COUNT("pull_dir_count", true), 121 // Represents the time we spend pushing file from device. 122 PUSH_FILE_TIME("push_file_time_ms", true), 123 // Represents how many times we pushed file from the device. 124 PUSH_FILE_COUNT("push_file_count", true), 125 // Represents the time we spend pushing dir from device. 126 PUSH_DIR_TIME("push_dir_time_ms", true), 127 // Represents how many times we pushing dir from the device. 128 PUSH_DIR_COUNT("push_dir_count", true), 129 // Represents the time we spent deleting file on device 130 DELETE_DEVICE_FILE_TIME("delete_device_file_time_ms", true), 131 // Represents how many times we call the delete file method 132 DELETE_DEVICE_FILE_COUNT("delete_device_file_count", true), 133 DOES_FILE_EXISTS_TIME("does_file_exists_time_ms", true), 134 DOES_FILE_EXISTS_COUNT("does_file_exists_count", true), 135 // Represents the time and count for installing packages 136 PACKAGE_INSTALL_TIME("package_install_time_ms", true), 137 PACKAGE_INSTALL_COUNT("package_install_count", true), 138 // Capture the time spent isolating a retry with reset 139 RESET_RETRY_ISOLATION_PAIR("reset_isolation_timestamp_pair", true), 140 // Capture the time spent isolating a retry with reboot 141 REBOOT_RETRY_ISOLATION_PAIR("reboot_isolation_timestamp_pair", true), 142 // Track metrics for skipped retries 143 RETRY_MODULE_SKIPPED_COUNT("retry_module_skipped_count", true), 144 RETRY_TEST_SKIPPED_COUNT("retry_test_skipped_count", true), 145 RETRY_SKIPPED_ALL_FILTERED_COUNT("retry_skipped_all_filtered_count", true), 146 147 // Track dynamic sharding total request latency 148 DYNAMIC_SHARDING_REQUEST_LATENCY("dynamic-sharding-request-latency", true), 149 // Track dynamic sharding total request count 150 DYNAMIC_SHARDING_REQUEST_COUNT("dynamic-sharding-request-count", true), 151 152 // The time spent inside metric collectors 153 COLLECTOR_TIME("collector_time_ms", true), 154 // Track if soft restart is occurring after test module 155 SOFT_RESTART_AFTER_MODULE("soft_restart_after_module", true), 156 CLOUD_DEVICE_PROJECT("cloud_device_project", false), 157 CLOUD_DEVICE_MACHINE_TYPE("cloud_device_machine_type", false), 158 CLOUD_DEVICE_ZONE("cloud_device_zone", false), 159 CLOUD_DEVICE_STABLE_HOST_IMAGE("stable_host_image_name", false), 160 CLOUD_DEVICE_STABLE_HOST_IMAGE_PROJECT("stable_host_image_project", false), 161 162 SHUTDOWN_BEFORE_TEST("shutdown_before_test", false), 163 SHUTDOWN_AFTER_TEST("shutdown_after_test", false), 164 SHUTDOWN_LATENCY("shutdown_latency_ms", false), 165 SHUTDOWN_HARD_LATENCY("shutdown_hard_latency_ms", false), 166 DEVICE_COUNT("device_count", false), 167 DEVICE_DONE_TIMESTAMP("device_done_timestamp", false), 168 DEVICE_RELEASE_STATE("device_release_state", false), 169 DEVICE_LOST_DETECTED("device_lost_detected", false), 170 VIRTUAL_DEVICE_LOST_DETECTED("virtual_device_lost_detected", false), 171 // Count the number of time device recovery like usb reset are successful. 172 DEVICE_RECOVERY("device_recovery", true), 173 DEVICE_RECOVERY_FROM_RECOVERY("device_recovery_from_recovery", true), 174 DEVICE_RECOVERED_FROM_SSH_TUNNEL("device_recovered_from_ssh_tunnel", true), 175 DEVICE_RECOVERED_FROM_DEVICE_RESET("device_recovered_from_device_reset", true), 176 DEVICE_RECOVERY_FAIL("device_recovery_fail", true), 177 SANDBOX_EXIT_CODE("sandbox_exit_code", false), 178 CF_FETCH_ARTIFACT_TIME("cf_fetch_artifact_time_ms", false), 179 CF_GCE_CREATE_TIME("cf_gce_create_time_ms", false), 180 CF_LAUNCH_CVD_TIME("cf_launch_cvd_time_ms", false), 181 CF_INSTANCE_COUNT("cf_instance_count", false), 182 CF_LOG_SIZE("cf_log_size_bytes", true), 183 CF_OXYGEN_SERVER_URL("cf_oxygen_server_url", false), 184 CF_OXYGEN_SESSION_ID("cf_oxygen_session_id", false), 185 CF_OXYGEN_VERSION("cf_oxygen_version", false), 186 CF_OXYGENATION_VERSION("cf_oxygenation_version", false), 187 CRASH_FAILURES("crash_failures", true), 188 UNCAUGHT_CRASH_FAILURES("uncaught_crash_failures", true), 189 TEST_CRASH_FAILURES("test_crash_failures", true), 190 UNCAUGHT_TEST_CRASH_FAILURES("uncaught_test_crash_failures", true), 191 DEVICE_RESET_COUNT("device_reset_count", true), 192 DEVICE_RESET_MODULES("device_reset_modules", true), 193 DEVICE_POWERWASH_DURATIONS("device_powerwash_durations", true), 194 DEVICE_RESET_MODULES_FOR_TARGET_PREPARER("device_reset_modules_for_target_preparer", true), 195 DEVICE_SNAPSHOT_SUCCESS_COUNT("device_snapshot_success_count", true), 196 DEVICE_SNAPSHOT_FAILURE_COUNT("device_snapshot_failure_count", true), 197 DEVICE_SNAPSHOT_DURATIONS("device_snapshot_durations", true), 198 DEVICE_SNAPSHOT_RESTORE_SUCCESS_COUNT("device_snapshot_restore_success_count", true), 199 DEVICE_SNAPSHOT_RESTORE_FAILURE_COUNT("device_snapshot_restore_failure_count", true), 200 DEVICE_SNAPSHOT_RESTORE_DURATIONS("device_snapshot_restore_durations", true), 201 DEVICE_STOP_SUCCESS_COUNT("device_stop_success_count", true), 202 DEVICE_STOP_FAILURE_COUNT("device_stop_failure_count", true), 203 DEVICE_STOP_DURATIONS("device_stop_durations", true), 204 NONPERSISTENT_DEVICE_PROPERTIES("nonpersistent_device_properties", true), 205 PERSISTENT_DEVICE_PROPERTIES("persistent_device_properties", true), 206 INVOCATION_START("tf_invocation_start_timestamp", false), 207 LOAD_TEST_CONFIGS_TIME("load_test_configs_time_ms", true), 208 OXYGEN_DEVICE_DIRECT_LEASE_COUNT("oxygen_device_direct_lease_count", true), 209 OXYGEN_DEVICE_DIRECT_RELEASE_COUNT("oxygen_device_direct_release_count", true), 210 OXYGEN_DEVICE_RELEASE_FAILURE_COUNT("oxygen_device_release_failure_count", true), 211 OXYGEN_DEVICE_RELEASE_FAILURE_MESSAGE("oxygen_device_release_failure_message", true), 212 ACLOUD_DEVICE_RELEASE_FAILURE_COUNT("acloud_device_release_failure_count", true), 213 214 DYNAMIC_FILE_RESOLVER_PAIR("tf_dynamic_resolver_pair_timestamp", true), 215 ARTIFACTS_DOWNLOAD_SIZE("tf_artifacts_download_size_bytes", true), 216 ARTIFACTS_UPLOAD_SIZE("tf_artifacts_upload_size_bytes", true), 217 LOG_SAVING_TIME("log_saving_time", true), 218 LOG_SAVING_COUNT("log_saving_count", true), 219 // TODO: Delete start/end timestamp in favor of pair. 220 FETCH_BUILD_START("tf_fetch_build_start_timestamp", false), 221 FETCH_BUILD_END("tf_fetch_build_end_timestamp", false), 222 FETCH_BUILD_PAIR("tf_fetch_build_pair_timestamp", true), 223 // TODO: Delete start/end timestamp in favor of pair. 224 SETUP_START("tf_setup_start_timestamp", false), 225 SETUP_END("tf_setup_end_timestamp", false), 226 SETUP_PAIR("tf_setup_pair_timestamp", true), 227 TEST_SETUP_PAIR("tf_test_setup_pair_timestamp", true), 228 FLASHING_FROM_FASTBOOTD("flashing_from_fastbootd", true), 229 FLASHING_TIME("flashing_time_ms", true), 230 FLASHING_PERMIT_LATENCY("flashing_permit_latency_ms", true), 231 FLASHING_METHOD("flashing_method", false), 232 FLASHSTATION_DOWNLOAD_SIZE("flashstation_download_size_bytes", true), 233 DOWNLOAD_PERMIT_LATENCY("download_permit_latency_ms", true), 234 // Unzipping metrics 235 UNZIP_TESTS_DIR_TIME("unzip_tests_dir_time_ms", true), 236 UNZIP_TESTS_DIR_COUNT("unzip_tests_dir_count", true), 237 // Don't aggregate test pair, latest report wins because it's the closest to 238 // the execution like in a subprocess. 239 TEST_PAIR("tf_test_pair_timestamp", false), 240 // TODO: Delete start/end timestamp in favor of pair. 241 TEARDOWN_START("tf_teardown_start_timestamp", false), 242 TEARDOWN_END("tf_teardown_end_timestamp", false), 243 TEARDOWN_PAIR("tf_teardown_pair_timestamp", true), 244 TEST_TEARDOWN_PAIR("tf_test_teardown_pair_timestamp", true), 245 246 INVOCATION_END("tf_invocation_end_timestamp", false), 247 248 MODULE_SETUP_PAIR("tf_module_setup_pair_timestamp", true), 249 MODULE_TEARDOWN_PAIR("tf_module_teardown_pair_timestamp", true), 250 STATUS_CHECKER_PAIR("status_checker_pair", true), 251 252 LAB_PREPARER_NOT_ILAB("lab_preparer_not_ilab", true), 253 TARGET_PREPARER_IS_ILAB("target_preparer_is_ilab", true), 254 255 ART_RUN_TEST_CHECKER_COMMAND_TIME_MS("art_run_test_checker_command_time_ms", true), 256 257 // CAS downloader metrics 258 CAS_VERSION("cas_version", false), 259 CAS_DOWNLOAD_ERRORS("cas_download_errors", true), 260 CAS_DOWNLOAD_ERROR_FILES("cas_download_error_files", true), 261 CAS_DOWNLOAD_ERROR_BUILD_ID("cas_download_error_build_id", true), 262 // Name of files downloaded by CAS downloader. 263 CAS_DOWNLOAD_FILES("cas_download_files", true), 264 CAS_DOWNLOAD_FILE_SUCCESS_COUNT("cas_download_file_success_count", true), 265 CAS_DOWNLOAD_FILE_FAIL_COUNT("cas_download_file_fail_count", true), 266 CAS_DOWNLOAD_TIME("cas_download_time_ms", true), 267 CAS_DOWNLOAD_START_TIMESTAMP("cas_download_start_timestamp", true), 268 // Records the wait time caused by CAS downloader concurrency limitation. 269 CAS_DOWNLOAD_WAIT_TIME("cas_download_wait_time_ms", true), 270 CAS_LOCK_TIMEOUTS("cas_lock_timeout", true), 271 CAS_CACHE_FALLBACK_COUNT("cas_cache_fallback_count", true), 272 CAS_TIMEOUT_COUNT("cas_timeout_count", true), 273 CAS_RETRY_BUDGET_EXHAUSTED_COUNT("cas_retry_budget_exhausted_count", true), 274 // Records cache hit metrics 275 CAS_DOWNLOAD_HOT_BYTES("cas_download_hot_bytes", true), 276 CAS_DOWNLOAD_COLD_BYTES("cas_download_cold_bytes", true), 277 CAS_DOWNLOAD_HOT_FILES_COUNT("cas_download_hot_files_count", true), 278 CAS_DOWNLOAD_COLD_FILES_COUNT("cas_download_cold_files_count", true), 279 // Records local cache metrics 280 // CAS downloader local cache type. Can be either NFS or local disk. 281 CAS_DOWNLOAD_LOCAL_CACHE_TYPE("cas_download_local_cache_type", false), 282 // Number of ATE instances that sharing a same NFS backed local cache 283 CAS_DOWNLOAD_NFS_LOCAL_CACHE_CONCURRENCY("cas_download_nfs_local_cache_concurrency", false), 284 CAS_DOWNLOAD_ACQUIRE_LOCAL_CACHE_LOCK_TIME( 285 "cas_download_acquire_local_cache_lock_time_ms", true), 286 CAS_DOWNLOAD_DIR_RETRIEVE_TIME("cas_download_dir_retrieve_time_ms", true), 287 CAS_DOWNLOAD_DIR_PREPARE_TIME("cas_download_dir_prepare_time_ms", true), 288 CAS_DOWNLOAD_FILE_DOWNLOAD_TIME("cas_download_file_download_time_ms", true), 289 CAS_DOWNLOAD_CHUNK_RESTORE_TIME("cas_download_chunk_restore_time_ms", true), 290 291 // Download Cache 292 CACHE_HIT_COUNT("cache_hit_count", true), 293 CACHE_WAIT_FOR_LOCK("cache_wait_for_lock", true), 294 295 // CF Cache metrics 296 CF_CACHE_WAIT_TIME("cf_cache_wait_time_sec", false), 297 CF_ARTIFACTS_FETCH_SOURCE("cf_artifacts_fetch_source", false), 298 299 // Ab downloader metrics 300 AB_BUILD_GET_API_TIME("ab_build_get_api_time", true), 301 AB_DOWNLOAD_SIZE_ELAPSED_TIME("ab_download_size_elapsed_time", true), 302 ZIP_PARTIAL_DOWNLOAD_CACHE_HIT("zip_partial_download_cache_hit", true), 303 DOWNLOAD_BOTH_ZIPS_AND_TS("download_both_zips_and_ts", true), 304 AB_LIST_API_TIME_PAIR("ab_list_api_time_pair", true), 305 AB_TEST_ZIP_NAME("ab_test_zip_name", true), 306 AB_MODULE_IN_ZIP("ab_module_in_zip", true), 307 308 // Ab log saver metrics 309 AB_LOG_SAVER_STAGING_TIME("ab_log_saver_staging_time", true), 310 AB_LOG_SAVER_UPLOAD_TIME("ab_log_saver_upload_time", true), 311 // Ants metrics 312 ANTS_INVOCATION_START_TIME("ants_invocation_start_time", true), 313 ANTS_INVOCATION_END_TIME("ants_invocation_end_time", true), 314 ANTS_MODULE_START_TIME("ants_module_start_time", true), 315 ANTS_MODULE_END_TIME("ants_module_end_time", true), 316 ANTS_RUN_START_TIME("ants_run_start_time", true), 317 ANTS_RUN_END_TIME("ants_run_end_time", true), 318 // Count of modules with missing testIdentifier 319 ANTS_KNOWN_FAILURE_MODULE_ERROR_COUNT("ants_known_failure_module_error_count", true), 320 321 DUPLICATE_MAPPING_DIFFERENT_OPTIONS("duplicate_mapping_different_options", true), 322 323 HAS_ANY_RUN_FAILURES("has_any_run_failures", false), 324 TOTAL_TEST_COUNT("total_test_count", true), 325 326 // Metrics to store Device failure signatures 327 DEVICE_ERROR_SIGNATURES("device_failure_signatures", false), 328 329 DEVICE_IMAGE_NOT_CHANGED("device_image_not_changed", false), 330 IMAGE_CHANGES_IN_KEY_FILE("image_changes_in_key_file", true), 331 DEVICE_IMAGE_FILE_CHANGES("device_image_file_changes", true), 332 DEVICE_IMAGE_USED_HEURISTIC("device_image_used_heuristic", true), 333 TEST_ARTIFACT_NOT_CHANGED("test_artifact_not_changed", true), 334 PURE_DEVICE_IMAGE_UNCHANGED("pure_device_image_unchanged", true), 335 TEST_ARTIFACT_CHANGE_ONLY("test_artifact_change_only", true), 336 PARTIAL_SKIP_MODULE_UNCHANGED_COUNT("partial_skip_module_unchanged_count", true), 337 WORKDIR_DIFFS_IN_COMMON("workdir_diffs_in_common", true), 338 WOKRDIR_MODULE_WITH_DIFFS("workdir_module_with_diffs", true), 339 WORKDIR_UNCHANGED_MODULES("workdir_unchanged_modules", true), 340 ABORT_CONTENT_ANALYSIS("abort_content_analysis", true), 341 ABORT_CONTENT_ANALYSIS_REASON("abort_content_analysis_reason", true), 342 XTS_DIFFS_IN_COMMON("xts_diffs_in_common", true), 343 XTS_MODULE_WITH_DIFFS("xts_module_with_diffs", true), 344 XTS_UNCHANGED_MODULES("xts_unchanged_modules", true), 345 BUILD_KEY_WITH_DIFFS("build_key_with_diffs", true), 346 FILE_WITH_DIFFS("file_with_diffs", true), 347 UNCHANGED_FILE("unchanged_file", true), 348 MULTI_DEVICES_CONTENT_ANALYSIS("multi_devices_content_analysis", true), 349 DEVICELESS_CONTENT_ANALYSIS("deviceless_content_analysis", true), 350 351 POWERWASH_TIME("powerwash_time_ms", true), 352 POWERWASH_SUCCESS_COUNT("powerwash_success_count", true), 353 POWERWASH_FAILURE_COUNT("powerwash_failure_count", true), 354 LEASE_RETRY_COUNT_SUCCESS("lease_retry_count_success", true), 355 LEASE_RETRY_COUNT_FAILURE("lease_retry_count_failure", true), 356 357 TRACE_INTERNAL_ERROR("trace_internal_error", true), 358 359 INCREMENTAL_FLASHING_TIME("incremental_flashing_time", true), 360 INCREMENTAL_FLASHING_WAIT_PARALLEL_SETUP("incremental_flashing_wait_parallel_setup", true), 361 INCREMENTAL_FLASHING_ATTEMPT_COUNT("incremental_flashing_attempt_count", true), 362 INCREMENTAL_ACROSS_RELEASE_COUNT("incremental_across_release_count", true), 363 INCREMENTAL_FLASHING_TEARDOWN_FAILURE("incremental_flashing_teardown_failure", true), 364 INCREMENTAL_FLASHING_UPDATE_FAILURE("incremental_flashing_update_failure", true), 365 INCREMENTAL_SNAPUSERD_WRITE_TIME("incremental_snapuserd_write_time", true), 366 INCREMENTAL_SNAPUSERD_WRITE_BLOCKING_TIME( 367 "incremental_snapuserd_write_blocking_time", true), 368 INCREMENTAL_FALLBACK_REASON("incremental_fallback_reason", true), 369 INCREMENTAL_RECOVERY_FALLBACK("incremental_recovery_fallback", true), 370 INCREMENTAL_FIRST_BOOTLOADER_REBOOT_FAIL("incremental_first_bootloader_reboot_fail", true), 371 INCREMENTAL_NEW_FLOW("incremental_new_flow", true), 372 DEVICE_IMAGE_CACHE_MISMATCH("device_image_cache_mismatch", true), 373 DEVICE_IMAGE_CACHE_ORIGIN("device_image_cache_origin", true), 374 375 CONTENT_BASED_ANALYSIS_ATTEMPT("content_based_analysis_attempt", true), 376 SKIP_NO_TESTS_DISCOVERED("skip_no_tests_discovered", true), 377 SKIP_NO_CHANGES("skip_no_changes", true), 378 NO_CHANGES_POSTSUBMIT("no_changes_postsubmit", true), 379 SILENT_INVOCATION_SKIP_COUNT("silent_invocation_skip_count", true), 380 ERROR_INVOCATION_SKIP("error_invocation_skip", true), 381 DEMOTION_FILTERS_RECEIVED_COUNT("demotion_filters_received_count", true), 382 DEMOTION_ERROR_RESPONSE("demotion_error_response", true), 383 SEARCH_ARTIFACT_FAILURE_COUNT("search_artifact_failure_count", true), 384 RESOLVE_SYMLINK_COUNT("resolve_symlink_count", true), 385 386 // Following are trace events also reporting as metrics 387 invocation_warm_up("invocation_warm_up", true), 388 dynamic_download("dynamic_download", true), 389 fetch_artifact("fetch_artifact", true), 390 start_logcat("start_logcat", true), 391 pre_sharding_required_setup("pre_sharding_required_setup", true), 392 sharding("sharding", true), 393 pre_multi_preparer("pre_multi_preparer", true), 394 lab_setup("lab_setup", true), 395 test_setup("test_setup", true), 396 test_execution("test_execution", true), 397 check_device_availability("check_device_availability", true), 398 bugreport("bugreport", true), 399 host_sleep("host_sleep", true), 400 test_teardown("test_teardown", true), 401 test_cleanup("test_cleanup", true), 402 log_and_release_device("log_and_release_device", true), 403 invocation_events_processing("invocation_events_processing", true), 404 stage_suite_test_artifacts("stage_suite_test_artifacts", true), 405 wait_for_results_update("wait_for_results_update", true), 406 instru_collect_tests("instru_collect_tests", true), 407 TestContentAnalyzer("TestContentAnalyzer", true), 408 screen_on_setup("screen_on_setup", true), 409 410 // Test caching metrics 411 CACHED_MODULE_RESULTS_COUNT("cached_module_results_count", true), 412 DEVICE_IMAGE_HASH("device_image_hash", false), 413 414 // Module level caching 415 MODULE_CACHE_UPLOAD_ERROR("module_cache_upload_error", true), 416 MODULE_CACHE_UPLOAD_TIME("module_cache_upload_time", true), 417 MODULE_CACHE_DOWNLOAD_ERROR("module_cache_download_error", true), 418 MODULE_CACHE_DOWNLOAD_TIME("module_cache_download_time", true), 419 MODULE_RESULTS_CHECKING_CACHE("module_results_checking_cache", true), 420 MODULE_RESULTS_CACHE_HIT("module_results_cache_hit", true), 421 MODULE_CACHE_HIT_ID("module_cache_hit_id", true), 422 MODULE_CACHE_MISS_ID("module_cache_miss_id", true), 423 MODULE_CACHE_NO_DIR("module_cache_no_dir", true), 424 MODULE_RESULTS_CACHE_DEVICE_MISMATCH("module_results_cache_device_mismatch", true), 425 // Invocation level caching 426 INVOCATION_RESULTS_CHECKING_CACHE("invocation_results_checking_cache", true), 427 INVOCATION_CACHE_HIT("invocation_cache_hit", true), 428 429 // Oxygenation metrics 430 PORTFORWARD_LHP_SUCCESS_COUNT("portfoward_lhp_success_count", true), 431 PORTFORWARD_LHP_FAIL_COUNT("portfoward_lhp_fail_count", true), 432 433 // Host orchestrator metrics 434 CVD_LONG_OPERATION_TIMEOUT_API("cvd_long_operation_timeout_api", true), 435 UNSUPPORTED_HOST_ORCHESTRATOR_API("unsupported_host_orchestrator_api", true), 436 ; 437 438 private final String mKeyName; 439 // Whether or not to add the value when the key is added again. 440 private final boolean mAdditive; 441 InvocationMetricKey(String key, boolean additive)442 private InvocationMetricKey(String key, boolean additive) { 443 mKeyName = key; 444 mAdditive = additive; 445 } 446 447 @Override toString()448 public String toString() { 449 return mKeyName; 450 } 451 shouldAdd()452 public boolean shouldAdd() { 453 return mAdditive; 454 } 455 } 456 457 /** Grouping allows to log several groups under a same key. */ 458 public enum InvocationGroupMetricKey { 459 TEST_TYPE_COUNT("test-type-count", true), 460 TARGET_PREPARER_SETUP_LATENCY("target-preparer-setup-latency", true), 461 TARGET_PREPARER_TEARDOWN_LATENCY("target-preparer-teardown-latency", true), 462 LAB_PREPARER_SETUP_LATENCY("lab-preparer-setup-latency", true), 463 LAB_PREPARER_TEARDOWN_LATENCY("lab-preparer-teardown-latency", true), 464 MULTI_TARGET_PREPARER_TEARDOWN_LATENCY("multi-target-preparer-teardown-latency", true), 465 466 INCREMENTAL_FLASHING_PATCHES_SIZE("incremental-flashing-patches-size", true), 467 INCREMENTAL_FLASHING_TARGET_SIZE("incremental-flashing-target-size", true); 468 469 private final String mGroupName; 470 // Whether or not to add the value when the key is added again. 471 private final boolean mAdditive; 472 InvocationGroupMetricKey(String groupName, boolean additive)473 private InvocationGroupMetricKey(String groupName, boolean additive) { 474 mGroupName = groupName; 475 mAdditive = additive; 476 } 477 478 @Override toString()479 public String toString() { 480 return mGroupName; 481 } 482 shouldAdd()483 public boolean shouldAdd() { 484 return mAdditive; 485 } 486 } 487 InvocationMetricLogger()488 private InvocationMetricLogger() {} 489 490 private static ThreadLocal<ThreadGroup> sLocal = new ThreadLocal<>(); 491 492 /** Tracks a localized context when using the properties inside the gRPC server */ setLocalGroup(ThreadGroup tg)493 public static void setLocalGroup(ThreadGroup tg) { 494 sLocal.set(tg); 495 } 496 497 /** Resets the localized context. */ resetLocalGroup()498 public static void resetLocalGroup() { 499 sLocal.remove(); 500 } 501 502 /** 503 * Track metrics per ThreadGroup as a proxy to invocation since an invocation run within one 504 * threadgroup. 505 */ 506 private static final Map<ThreadGroup, Map<String, String>> mPerGroupMetrics = 507 Collections.synchronizedMap(new HashMap<ThreadGroup, Map<String, String>>()); 508 509 /** 510 * Add one key-value to be tracked at the invocation level. 511 * 512 * @param key The key under which the invocation metric will be tracked. 513 * @param value The value of the invocation metric. 514 */ addInvocationMetrics(InvocationMetricKey key, long value)515 public static void addInvocationMetrics(InvocationMetricKey key, long value) { 516 if (key.shouldAdd()) { 517 String existingVal = getInvocationMetrics().get(key.toString()); 518 long existingLong = 0L; 519 if (existingVal != null) { 520 try { 521 existingLong = Long.parseLong(existingVal); 522 } catch (NumberFormatException e) { 523 CLog.e( 524 "%s is expected to contain a number, instead found: %s", 525 key.toString(), existingVal); 526 } 527 } 528 value += existingLong; 529 } 530 addInvocationMetrics(key.toString(), Long.toString(value)); 531 } 532 533 /** 534 * Add one key-value for a given group 535 * 536 * @param groupKey The key of the group 537 * @param group The group name associated with the key 538 * @param value The value for the group 539 */ addInvocationMetrics( InvocationGroupMetricKey groupKey, String group, String value)540 public static void addInvocationMetrics( 541 InvocationGroupMetricKey groupKey, String group, String value) { 542 String key = groupKey.toString() + ":" + group; 543 if (groupKey.shouldAdd()) { 544 String existingVal = getInvocationMetrics().get(key.toString()); 545 if (existingVal != null) { 546 value = String.format("%s,%s", existingVal, value); 547 } 548 } 549 addInvocationMetrics(key, value); 550 } 551 552 /** 553 * Add one key-value to be tracked at the invocation level. Don't expose the String key yet to 554 * avoid abuse, stick to the official {@link InvocationMetricKey} to start with. 555 * 556 * @param key The key under which the invocation metric will be tracked. 557 * @param value The value of the invocation metric. 558 */ addInvocationMetrics(String key, String value)559 private static void addInvocationMetrics(String key, String value) { 560 ThreadGroup group = Thread.currentThread().getThreadGroup(); 561 synchronized (mPerGroupMetrics) { 562 if (sLocal.get() != null) { 563 group = sLocal.get(); 564 } 565 if (mPerGroupMetrics.get(group) == null) { 566 mPerGroupMetrics.put(group, new HashMap<>()); 567 } 568 mPerGroupMetrics.get(group).put(key, value); 569 } 570 } 571 572 /** 573 * Add one key-value to be tracked at the invocation level for a given group. 574 * 575 * @param groupKey The key of the group 576 * @param group The group name associated with the key 577 * @param value The value for the group 578 */ addInvocationMetrics( InvocationGroupMetricKey groupKey, String group, long value)579 public static void addInvocationMetrics( 580 InvocationGroupMetricKey groupKey, String group, long value) { 581 String key = groupKey.toString() + ":" + group; 582 if (groupKey.shouldAdd()) { 583 String existingVal = getInvocationMetrics().get(key); 584 long existingLong = 0L; 585 if (existingVal != null) { 586 try { 587 existingLong = Long.parseLong(existingVal); 588 } catch (NumberFormatException e) { 589 CLog.e( 590 "%s is expected to contain a number, instead found: %s", 591 key.toString(), existingVal); 592 } 593 } 594 value += existingLong; 595 } 596 addInvocationMetrics(key, Long.toString(value)); 597 } 598 599 /** 600 * Add one key-value to be tracked at the invocation level. 601 * 602 * @param key The key under which the invocation metric will be tracked. 603 * @param value The value of the invocation metric. 604 */ addInvocationMetrics(InvocationMetricKey key, String value)605 public static void addInvocationMetrics(InvocationMetricKey key, String value) { 606 if (key.shouldAdd()) { 607 String existingVal = getInvocationMetrics().get(key.toString()); 608 if (existingVal != null) { 609 value = String.format("%s,%s", existingVal, value); 610 } 611 } 612 addInvocationMetrics(key.toString(), value); 613 } 614 615 /** 616 * Add a pair of value associated with the same key. Usually used for timestamp start and end. 617 * 618 * @param key The key under which the invocation metric will be tracked. 619 * @param start The start value of the invocation metric. 620 * @param end The end value of the invocation metric. 621 */ addInvocationPairMetrics(InvocationMetricKey key, long start, long end)622 public static void addInvocationPairMetrics(InvocationMetricKey key, long start, long end) { 623 String value = start + ":" + end; 624 if (key.shouldAdd()) { 625 String existingVal = getInvocationMetrics().get(key.toString()); 626 if (existingVal != null) { 627 value = String.format("%s,%s", existingVal, value); 628 } 629 } 630 addInvocationMetrics(key.toString(), value); 631 } 632 633 /** Returns the Map of invocation metrics for the invocation in progress. */ getInvocationMetrics()634 public static Map<String, String> getInvocationMetrics() { 635 ThreadGroup group = Thread.currentThread().getThreadGroup(); 636 synchronized (mPerGroupMetrics) { 637 if (sLocal.get() != null) { 638 group = sLocal.get(); 639 } 640 if (mPerGroupMetrics.get(group) == null) { 641 mPerGroupMetrics.put(group, new HashMap<>()); 642 } 643 return new HashMap<>(mPerGroupMetrics.get(group)); 644 } 645 } 646 647 /** Clear the invocation metrics for an invocation. */ clearInvocationMetrics()648 public static void clearInvocationMetrics() { 649 ThreadGroup group = Thread.currentThread().getThreadGroup(); 650 synchronized (mPerGroupMetrics) { 651 mPerGroupMetrics.remove(group); 652 } 653 } 654 } 655