• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2023 The ChromiumOS Authors
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5syntax = "proto3";
6
7package chromiumos.test.api;
8
9option go_package = "go.chromium.org/chromiumos/config/go/test/api";
10
11import "chromiumos/test/api/test_suite.proto";
12
13// PreTestService acts as a pre-test landing point32 for needed services/actions.
14// For example:
15//  - filtering flaky tests
16// Could be expanded to include other services, like VM filtering, etc.
17service PreTestService {
18
19  // FilterFlakyTests will take in a policy + TestSuites
20  // and will echo out the TestSuites; with the flaky tests removed.
21  // along with a list of removed tests for logging purposes.
22  rpc FilterFlakyTests(FilterFlakyRequest)
23      returns (FilterFlakyResponse);
24}
25
26// FilterFlakyRequest request to run. Must be from the list below.
27message FilterFlakyRequest {
28  // Policy can be expanded to have new policies as needed.
29  // Note they must be implemented in the service.
30  oneof policy {
31    PassRatePolicy pass_rate_policy = 1;
32    StabilitySensorPolicy stability_sensor_policy = 2;
33  }
34
35  repeated TestSuite test_suites = 3;
36
37  // Milestone that the test filtering is for.
38  // Important as test flake can be very different per milestone based on
39  // breakages/fixes across branches.
40  string milestone = 4;
41  // Currently filtering will be done on _board_ level.
42  // Other variants (model, hwid, etc) to follow.
43  oneof variant {
44  string board = 5;
45  }
46
47  // When there is not enough signal to determine stability (based on policy); default on/off.
48  bool default_enabled = 6;
49
50  // BBID of the task
51  string bbid = 7;
52  // is_dry_run indicates if the results are to be used or not. This will only change
53  // how the system logs results.
54  bool is_dry_run = 8;
55}
56
57
58message FilterFlakyResponse {
59  repeated TestSuite test_suites = 1;
60  repeated string removed_tests = 2;
61}
62
63// StabilitySensorPolicy indicates to use the StabilitySensor for determine flake.
64message StabilitySensorPolicy {}
65
66// PassRatePolicy indicates to use the StabilitySensor for determine flake.
67message PassRatePolicy {
68  int32 pass_rate = 1;
69  int32 min_runs = 2;
70
71  // NOTE: num_of_milestones determines how many milestones back to look for results.
72  // The number here will default to 0 (meaning current only); however, if there are not enough
73  // results on the current milestone to meet `min_runs`, then will go back up to 1 milestone.
74  // Setting this value will override the default behavior, and will not have the
75  // logic to go back an additional milestone if not enough results.
76  int32 num_of_milestones = 4;
77
78  // force_enabled_tests DEPRECATED
79  repeated string force_enabled_tests = 5;
80  // force_disabled_tests DEPRECATED
81  repeated string force_disabled_tests = 6;
82  // force_enabled_boards DEPRECATED
83  repeated string force_enabled_boards = 7;
84
85  // Recent results are a method to re-enable historically flaky tests, with
86  // more recent results. Its expected that the recent result will be more stringent
87  // than the long-term results. For example, if a test has a long-term passrate of
88  // 95% with 1000 samples, but a recent passrate of 100% with 200 samples, its
89  // signal that its stable enough to be re-enabled.
90  int32 pass_rate_recent = 8;
91  int32 min_runs_recent = 9;
92  int32 recent_window = 10;
93
94  bool dryrun = 11;
95
96  repeated FilterTestConfig testconfigs = 12;
97
98  // max_filtered_percent is the max % of tests that can be filtered. Eg 20%.
99  // If the limit is hit, the filtered tests will be ordered by flake.
100  int32 max_filtered_percent = 13;
101  // max_filtered_int follows the same logic as above.
102  // NOTE: if both fields are set, we will use the more restrictive of the two.
103  // For example:
104  //   100 tests with a max_filtered_percent=50% and max_filtered_int=5
105  //   20 tests hit the filter threshold,
106  //   Only 5 would get filtered out.
107  int32 max_filtered_int = 14;
108}
109
110// FilterCfgs as a whole describes all the policies for test filtering.
111message FilterCfgs {
112  repeated FilterCfg filter_cfg = 1;
113}
114
115// FilterCfg is a policy which applies to one or more suites.
116// If there is/are no suite(s) set, it will be treated as the global policy.
117message FilterCfg {
118  // This applies to the entire suite. Any tests forced_enabled/disabled in this
119  // will apply to all boards.
120  PassRatePolicy pass_rate_policy = 1;
121
122  // Blank will be treated as the global policy. By setting this field, this policy
123  // will overwrite the global, and apply to all suites in this list.
124  repeated string test_suites = 2;
125
126  // Fully opt the suite out of filtering.
127  bool opt_out = 4;
128}
129
130// FilterTestConfig contains a test, boards, and the state of the test.
131message FilterTestConfig {
132  string test = 1;
133  repeated string board = 2;
134
135  Setting setting = 3;
136  // Force enable/disabled the test on the boards in the cfg.
137  enum Setting {
138    NOT_SET =0;
139    ENABLED = 1;
140    DISABLED = 2;
141  }
142  // List of bugs associated with the cfg
143  repeated string bugs = 4;
144
145}
146