• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2021 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import argparse
6
7
8def ParseArgs():
9  parser = argparse.ArgumentParser(
10      description=('Script for automatically suppressing flaky/failing '
11                   'tests.'))
12  parser.add_argument('--project',
13                      required=True,
14                      help=('The billing project to use for BigQuery queries. '
15                            'Must have access to the ResultDB BQ tables, e.g. '
16                            '"chrome-luci-data.chromium.gpu_ci_test_results".'))
17  parser.add_argument('--sample-period',
18                      type=int,
19                      default=1,
20                      help=('The number of days to sample data from.'))
21  parser.add_argument('--no-group-by-tags',
22                      action='store_false',
23                      default=True,
24                      dest='group_by_tags',
25                      help=('Append added expectations to the end of the file '
26                            'instead of attempting to automatically group with '
27                            'similar expectations.'))
28  parser.add_argument('--no-prompt-for-user-input',
29                      action='store_false',
30                      default=True,
31                      dest='prompt_for_user_input',
32                      help=('Generate expectations automatically based on '
33                            'thresholds instead of prompting the user each '
34                            'time. The user will still need to add associated '
35                            'bugs to generated expectations afterwards.'))
36  parser.add_argument('--ignore-threshold',
37                      type=float,
38                      default=0.01,
39                      help=('The fraction of failed tests under which flakes '
40                            'will be ignored instead of having an expectation '
41                            'added when --no-prompt-for-user-input is used.'))
42  parser.add_argument('--flaky-threshold',
43                      type=float,
44                      default=0.5,
45                      help=('The fraction of failed tests under which flakes '
46                            'will be marked as RetryOnFailure when '
47                            '--no-prompt-for-user-input is used. Above this, '
48                            'failures will be marked as Failure.'))
49  parser.add_argument('--include-all-tags',
50                      action='store_true',
51                      default=False,
52                      help=('Use all tags generated by a configuration when '
53                            'creating an expectation rather than attempting '
54                            'to only use the most specific one. This should '
55                            'only need to be passed if the tags in the '
56                            'expectation files are not ordered from least '
57                            'specific to most specific.'))
58  parser.add_argument('--result-output-file',
59                      help=('Output file to store the generated results. If '
60                            'not specified, will use a temporary file.'))
61  parser.add_argument('--bypass-up-to-date-check',
62                      action='store_true',
63                      default=False,
64                      help=('Bypasses the check that the local expectation '
65                            'files are up to date. Only intended for use on '
66                            'bots to avoid failures due to potential race '
67                            'conditions between updating the checkout and '
68                            'running the script.'))
69  parser.add_argument(
70      '--non-hidden-failures-only',
71      action='store_true',
72      default=False,
73      help=
74      ('Enable this option to only targeting visible failures on CI builders. '
75       'The test results will fail the builder runs, flaky results will '
76       'consider as pass in this option.'))
77  parser.add_argument(
78      '--build-fail-total-number-threshold',
79      type=int,
80      default=0,
81      help=('Threshold based on failed build number when '
82            '--non-hidden-failures-only is used. A test will be '
83            'suppressed if its failed build number is equal to or more than '
84            'this threshold. All --build-fail*-thresholds must be hit in '
85            'order for a test to actually be suppressed.'))
86  parser.add_argument(
87      '--build-fail-consecutive-days-threshold',
88      type=int,
89      default=2,
90      help=('Threshold based on number of consecutive days that non-hidden'
91            'failures occur. A test will be suppressed if the number of'
92            'consecutive days that it has non-hidden failures is equal'
93            'to or more than this threshold. All --build-fail*-thresholds '
94            'must be hit in order for a test to actually be suppressed.'))
95  parser.add_argument('--builder-name',
96                      default=[],
97                      action="append",
98                      dest="builder_names",
99                      help="CI builder list to suppress tests.")
100  args = parser.parse_args()
101
102  if not args.prompt_for_user_input:
103    if args.ignore_threshold < 0:
104      raise ValueError('--ignore-threshold must be positive')
105    if args.flaky_threshold < 0:
106      raise ValueError('--flaky-threshold must be positive')
107    if args.flaky_threshold <= args.ignore_threshold:
108      raise ValueError(
109          '--flaky-threshold must be greater than --ignore-threshold')
110    if args.build_fail_total_number_threshold < 0:
111      raise ValueError('--build-fail-total-number-threshold must be positive')
112    if args.build_fail_consecutive_days_threshold < 0:
113      raise ValueError('--build-fail-consecutive-days-threshold must be '
114                       'positive')
115
116  return args
117