# Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import argparse def ParseArgs(): parser = argparse.ArgumentParser( description=('Script for automatically suppressing flaky/failing ' 'tests.')) parser.add_argument('--project', required=True, help=('The billing project to use for BigQuery queries. ' 'Must have access to the ResultDB BQ tables, e.g. ' '"chrome-luci-data.chromium.gpu_ci_test_results".')) parser.add_argument('--sample-period', type=int, default=7, choices=range(1, 30), help=('The number of days to sample data from.')) parser.add_argument('--no-group-by-tags', action='store_false', default=True, dest='group_by_tags', help=('Append added expectations to the end of the file ' 'instead of attempting to automatically group with ' 'similar expectations.')) parser.add_argument('--no-prompt-for-user-input', action='store_false', default=True, dest='prompt_for_user_input', help=('Generate expectations automatically based on ' 'thresholds instead of prompting the user each ' 'time. The user will still need to add associated ' 'bugs to generated expectations afterwards.')) parser.add_argument('--ignore-threshold', type=float, default=0.01, help=('The fraction of failed tests under which flakes ' 'will be ignored instead of having an expectation ' 'added when --no-prompt-for-user-input is used.')) parser.add_argument('--flaky-threshold', type=float, default=0.5, help=('The fraction of failed tests under which flakes ' 'will be marked as RetryOnFailure when ' '--no-prompt-for-user-input is used. Above this, ' 'failures will be marked as Failure.')) parser.add_argument('--include-all-tags', action='store_true', default=False, help=('Use all tags generated by a configuration when ' 'creating an expectation rather than attempting ' 'to only use the most specific one. This should ' 'only need to be passed if the tags in the ' 'expectation files are not ordered from least ' 'specific to most specific.')) parser.add_argument('--result-output-file', help=('Output file to store the generated results. If ' 'not specified, will use a temporary file.')) parser.add_argument('--bypass-up-to-date-check', action='store_true', default=False, help=('Bypasses the check that the local expectation ' 'files are up to date. Only intended for use on ' 'bots to avoid failures due to potential race ' 'conditions between updating the checkout and ' 'running the script.')) parser.add_argument( '--non-hidden-failures-only', action='store_true', default=False, help= ('Enable this option to only targeting visible failures on CI builders. ' 'The test results will fail the builder runs, flaky results will ' 'consider as pass in this option.')) parser.add_argument( '--build-fail-total-number-threshold', type=int, default=10, help=('Threshold based on failed build number when ' '--non-hidden-failures-only is used. A test will be ' 'suppressed if its failed build number is equal to or more than ' 'this threshold. All --build-fail*-thresholds must be hit in ' 'order for a test to actually be suppressed.')) parser.add_argument( '--build-fail-consecutive-days-threshold', type=int, default=2, choices=range(1, 30), help=('Threshold based on number of consecutive days that non-hidden' 'failures occur. A test will be suppressed if the number of' 'consecutive days that it has non-hidden failures is equal' 'to or more than this threshold. All --build-fail*-thresholds ' 'must be hit in order for a test to actually be suppressed.')) parser.add_argument('--build-fail-recent-days-threshold', type=int, default=2, choices=range(1, 30), help=('Suppress tests with non-hidden build failures' ' within |build_fail_recent_day_threshold| days ' 'when all other build-fail* thresholds meet.')) parser.add_argument('--builder-name', default=[], action='append', dest='builder_names', help='CI builder list to suppress tests.') args = parser.parse_args() if not args.prompt_for_user_input: if args.ignore_threshold < 0: raise ValueError('--ignore-threshold must be positive') if args.flaky_threshold < 0: raise ValueError('--flaky-threshold must be positive') if args.flaky_threshold <= args.ignore_threshold: raise ValueError( '--flaky-threshold must be greater than --ignore-threshold') if args.build_fail_total_number_threshold < 0: raise ValueError('--build-fail-total-number-threshold must be positive') return args