xref: /external/toolchain-utils/crosperf/settings_factory.py

# -*- coding: utf-8 -*-
# Copyright 2013 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Setting files for global, benchmark and labels."""


from field import BooleanField
from field import EnumField
from field import FloatField
from field import IntegerField
from field import ListField
from field import TextField
from settings import Settings


class BenchmarkSettings(Settings):
    """Settings used to configure individual benchmarks."""

    def __init__(self, name):
        super(BenchmarkSettings, self).__init__(name, "benchmark")
        self.AddField(
            TextField(
                "test_name",
                description="The name of the test to run. "
                "Defaults to the name of the benchmark.",
            )
        )
        self.AddField(
            TextField(
                "test_args",
                description="Arguments to be passed to the " "test.",
            )
        )
        self.AddField(
            IntegerField(
                "iterations",
                required=False,
                default=0,
                description="Number of iterations to run the test. "
                "If not set, will run each benchmark test the optimum number of "
                "times to get a stable result.",
            )
        )
        self.AddField(
            TextField(
                "suite",
                default="test_that",
                description="The type of the benchmark.",
            )
        )
        self.AddField(
            IntegerField(
                "retries",
                default=0,
                description="Number of times to retry a " "benchmark run.",
            )
        )
        self.AddField(
            BooleanField(
                "run_local",
                description="Run benchmark harness on the DUT. "
                "Currently only compatible with the suite: "
                "telemetry_Crosperf.",
                required=False,
                default=True,
            )
        )
        self.AddField(
            FloatField(
                "weight",
                default=0.0,
                description="Weight of the benchmark for CWP approximation",
            )
        )


class LabelSettings(Settings):
    """Settings for each label."""

    def __init__(self, name):
        super(LabelSettings, self).__init__(name, "label")
        self.AddField(
            TextField(
                "chromeos_image",
                required=False,
                description="The path to the image to run tests "
                "on, for local/custom-built images. See the "
                "'build' option for official or trybot images.",
            )
        )
        self.AddField(
            TextField(
                "autotest_path",
                required=False,
                description="Autotest directory path relative to chroot which "
                "has autotest files for the image to run tests requiring autotest "
                "files.",
            )
        )
        self.AddField(
            TextField(
                "debug_path",
                required=False,
                description="Debug info directory relative to chroot which has "
                "symbols and vmlinux that can be used by perf tool.",
            )
        )
        self.AddField(
            TextField(
                "chromeos_root",
                description="The path to a chromeos checkout which "
                "contains a src/scripts directory. Defaults to "
                "the chromeos checkout which contains the "
                "chromeos_image.",
            )
        )
        self.AddField(
            ListField(
                "remote",
                description="A comma-separated list of IPs of chromeos"
                "devices to run experiments on.",
            )
        )
        self.AddField(
            TextField(
                "image_args",
                required=False,
                default="",
                description="Extra arguments to pass to " "image_chromeos.py.",
            )
        )
        self.AddField(
            TextField(
                "cache_dir",
                default="",
                description="The cache dir for this image.",
            )
        )
        self.AddField(
            TextField(
                "compiler",
                default="gcc",
                description="The compiler used to build the "
                "ChromeOS image (gcc or llvm).",
            )
        )
        self.AddField(
            TextField(
                "chrome_src",
                description="The path to the source of chrome. "
                "This is used to run telemetry benchmarks. "
                "The default one is the src inside chroot.",
                required=False,
                default="",
            )
        )
        self.AddField(
            TextField(
                "build",
                description="The xbuddy specification for an "
                "official or trybot image to use for tests. "
                "'/remote' is assumed, and the board is given "
                "elsewhere, so omit the '/remote/<board>/' xbuddy "
                "prefix.",
                required=False,
                default="",
            )
        )


class GlobalSettings(Settings):
    """Settings that apply per-experiment."""

    def __init__(self, name):
        super(GlobalSettings, self).__init__(name, "global")
        self.AddField(
            TextField(
                "name",
                description="The name of the experiment. Just an "
                "identifier.",
            )
        )
        self.AddField(
            TextField(
                "board",
                description="The target board for running "
                "experiments on, e.g. x86-alex.",
            )
        )
        self.AddField(
            BooleanField(
                "crosfleet",
                description="Whether to run experiments via crosfleet.",
                default=False,
            )
        )
        self.AddField(
            ListField(
                "remote",
                description="A comma-separated list of IPs of "
                "chromeos devices to run experiments on.",
            )
        )
        self.AddField(
            BooleanField(
                "rerun_if_failed",
                description="Whether to re-run failed test runs " "or not.",
                default=False,
            )
        )
        self.AddField(
            BooleanField(
                "rm_chroot_tmp",
                default=False,
                description="Whether to remove the test_that "
                "result in the chroot.",
            )
        )
        self.AddField(
            ListField(
                "email",
                description="Space-separated list of email "
                "addresses to send email to.",
            )
        )
        self.AddField(
            BooleanField(
                "rerun",
                description="Whether to ignore the cache and "
                "for tests to be re-run.",
                default=False,
            )
        )
        self.AddField(
            BooleanField(
                "same_specs",
                default=True,
                description="Ensure cached runs are run on the "
                "same kind of devices which are specified as a "
                "remote.",
            )
        )
        self.AddField(
            BooleanField(
                "same_machine",
                default=False,
                description="Ensure cached runs are run on the " "same remote.",
            )
        )
        self.AddField(
            BooleanField(
                "use_file_locks",
                default=False,
                description="DEPRECATED: Whether to use the file locks "
                "or AFE server lock mechanism.",
            )
        )
        self.AddField(
            IntegerField(
                "iterations",
                required=False,
                default=0,
                description="Number of iterations to run all tests. "
                "If not set, will run each benchmark test the optimum number of "
                "times to get a stable result.",
            )
        )
        self.AddField(
            TextField(
                "chromeos_root",
                description="The path to a chromeos checkout which "
                "contains a src/scripts directory. Defaults to "
                "the chromeos checkout which contains the "
                "chromeos_image.",
            )
        )
        self.AddField(
            TextField(
                "logging_level",
                default="average",
                description="The level of logging desired. "
                "Options are 'quiet', 'average', and 'verbose'.",
            )
        )
        self.AddField(
            IntegerField(
                "acquire_timeout",
                default=0,
                description="Number of seconds to wait for "
                "machine before exit if all the machines in "
                "the experiment file are busy. Default is 0.",
            )
        )
        self.AddField(
            TextField(
                "perf_args",
                default="",
                description="The optional profile command. It "
                "enables perf commands to record perforamance "
                "related counters. It must start with perf "
                "command record or stat followed by arguments.",
            )
        )
        self.AddField(
            BooleanField(
                "download_debug",
                default=True,
                description="Download compressed debug symbols alongwith "
                "image. This can provide more info matching symbols for"
                "profiles, but takes larger space. By default, download"
                "it only when perf_args is specified.",
            )
        )
        self.AddField(
            TextField(
                "cache_dir",
                default="",
                description="The abs path of cache dir. "
                "Default is /home/$(whoami)/cros_scratch.",
            )
        )
        self.AddField(
            BooleanField(
                "cache_only",
                default=False,
                description="Whether to use only cached "
                "results (do not rerun failed tests).",
            )
        )
        self.AddField(
            BooleanField(
                "no_email",
                default=False,
                description="Whether to disable the email to "
                "user after crosperf finishes.",
            )
        )
        self.AddField(
            BooleanField(
                "json_report",
                default=False,
                description="Whether to generate a json version "
                "of the report, for archiving.",
            )
        )
        self.AddField(
            BooleanField(
                "show_all_results",
                default=False,
                description="When running Telemetry tests, "
                "whether to all the results, instead of just "
                "the default (summary) results.",
            )
        )
        self.AddField(
            TextField(
                "share_cache",
                default="",
                description="Path to alternate cache whose data "
                "you want to use. It accepts multiple directories "
                'separated by a ",".',
            )
        )
        self.AddField(
            TextField("results_dir", default="", description="The results dir.")
        )
        self.AddField(
            BooleanField(
                "compress_results",
                default=True,
                description="Whether to compress all test results other than "
                "reports into a tarball to save disk space.",
            )
        )
        self.AddField(
            TextField(
                "locks_dir",
                default="",
                description="An alternate directory to use for "
                "storing/checking machine file locks for local machines. "
                "By default the file locks directory is "
                "/google/data/rw/users/mo/mobiletc-prebuild/locks.\n"
                "WARNING: If you use your own locks directory, "
                "there is no guarantee that someone else might not "
                "hold a lock on the same machine in a different "
                "locks directory.",
            )
        )
        self.AddField(
            TextField(
                "chrome_src",
                description="The path to the source of chrome. "
                "This is used to run telemetry benchmarks. "
                "The default one is the src inside chroot.",
                required=False,
                default="",
            )
        )
        self.AddField(
            IntegerField(
                "retries",
                default=0,
                description="Number of times to retry a " "benchmark run.",
            )
        )
        self.AddField(
            TextField(
                "cwp_dso",
                description="The DSO type that we want to use for "
                "CWP approximation. This is used to run telemetry "
                "benchmarks. Valid DSO types can be found from dso_list "
                "in experiment_factory.py. The default value is set to "
                "be empty.",
                required=False,
                default="",
            )
        )
        self.AddField(
            BooleanField(
                "enable_aslr",
                description="Enable ASLR on the machine to run the "
                "benchmarks. ASLR is disabled by default",
                required=False,
                default=False,
            )
        )
        self.AddField(
            BooleanField(
                "ignore_min_max",
                description="When doing math for the raw results, "
                "ignore min and max values to reduce noise.",
                required=False,
                default=False,
            )
        )
        self.AddField(
            TextField(
                "intel_pstate",
                description="Intel Pstate mode.\n"
                'Supported modes: "active", "passive", "no_hwp".\n'
                'Default is "no_hwp" which disables hardware pstates to avoid '
                "noise in benchmarks.",
                required=False,
                default="no_hwp",
            )
        )
        self.AddField(
            BooleanField(
                "turbostat",
                description="Run turbostat process in the background"
                " of a benchmark. Enabled by default.",
                required=False,
                default=True,
            )
        )
        self.AddField(
            FloatField(
                "top_interval",
                description="Run top command in the background of a benchmark with"
                " interval of sampling specified in seconds.\n"
                "Recommended values 1-5. Lower number provides more accurate"
                " data.\n"
                "With 0 - do not run top.\n"
                "NOTE: Running top with interval 1-5 sec has insignificant"
                " performance impact (performance degradation does not exceed"
                " 0.3%%, measured on x86_64, ARM32, and ARM64). "
                "The default value is 1.",
                required=False,
                default=1,
            )
        )
        self.AddField(
            IntegerField(
                "cooldown_temp",
                required=False,
                default=40,
                description="Wait until CPU temperature goes down below"
                " specified temperature in Celsius"
                " prior starting a benchmark. "
                "By default the value is set to 40 degrees.",
            )
        )
        self.AddField(
            IntegerField(
                "cooldown_time",
                required=False,
                default=10,
                description="Wait specified time in minutes allowing"
                " CPU to cool down. Zero value disables cooldown. "
                "The default value is 10 minutes.",
            )
        )
        self.AddField(
            EnumField(
                "governor",
                options=[
                    "performance",
                    "powersave",
                    "userspace",
                    "ondemand",
                    "conservative",
                    "schedutils",
                    "sched",
                    "interactive",
                ],
                default="performance",
                required=False,
                description="Setup CPU governor for all cores.\n"
                "For more details refer to:\n"
                "https://www.kernel.org/doc/Documentation/cpu-freq/governors.txt. "
                'Default is "performance" governor.',
            )
        )
        self.AddField(
            EnumField(
                "cpu_usage",
                options=[
                    "all",
                    "big_only",
                    "little_only",
                    "exclusive_cores",
                ],
                default="all",
                required=False,
                description="Restrict usage of CPUs to decrease CPU interference.\n"
                '"all" - no restrictions;\n'
                '"big-only", "little-only" - enable only big/little cores,'
                " applicable only on ARM;\n"
                '"exclusive-cores" - (for future use)'
                " isolate cores for exclusive use of benchmark processes. "
                "By default use all CPUs.",
            )
        )
        self.AddField(
            IntegerField(
                "cpu_freq_pct",
                required=False,
                default=95,
                description="Setup CPU frequency to a supported value less than"
                " or equal to a percent of max_freq. "
                "CPU frequency is reduced to 95%% by default to reduce thermal "
                "throttling.",
            )
        )
        self.AddField(
            BooleanField(
                "no_lock",
                default=False,
                description="Do not attempt to lock the DUT."
                " Useful when lock is held externally, say with crosfleet.",
            )
        )


class SettingsFactory(object):
    """Factory class for building different types of Settings objects.

    This factory is currently hardcoded to produce settings for ChromeOS
    experiment files. The idea is that in the future, other types
    of settings could be produced.
    """

    def GetSettings(self, name, settings_type):
        if settings_type == "label" or not settings_type:
            return LabelSettings(name)
        if settings_type == "global":
            return GlobalSettings(name)
        if settings_type == "benchmark":
            return BenchmarkSettings(name)

        raise TypeError("Invalid settings type: '%s'." % settings_type)