• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env lucicfg
2#
3# Copyright (C) 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18"""LUCI project configuration for the production instance of LUCI.
19
20After modifying this file execute it ('./main.star') to regenerate the configs.
21"""
22
23lucicfg.check_version("1.30.9", "Please update depot_tools")
24
25luci.builder.defaults.experiments.set({
26    "luci.recipes.use_python3": 100,
27})
28
29# Use LUCI Scheduler BBv2 names and add Scheduler realms configs.
30lucicfg.enable_experiment("crbug.com/1182002")
31
32# Tell lucicfg what files it is allowed to touch.
33lucicfg.config(
34    config_dir = "generated",
35    fail_on_warnings = True,
36    lint_checks = ["default"],
37)
38
39# TODO: Switch to project-scoped service account.
40
41luci.project(
42    name = "art",
43    buildbucket = "cr-buildbucket.appspot.com",
44    logdog = "luci-logdog.appspot.com",
45    milo = "luci-milo.appspot.com",
46    notify = "luci-notify.appspot.com",
47    scheduler = "luci-scheduler.appspot.com",
48    swarming = "chromium-swarm.appspot.com",
49    acls = [
50        acl.entry(
51            roles = [
52                acl.BUILDBUCKET_READER,
53                acl.LOGDOG_READER,
54                acl.PROJECT_CONFIGS_READER,
55                acl.SCHEDULER_READER,
56            ],
57            groups = "googlers",
58        ),
59        acl.entry(
60            roles = [
61                acl.BUILDBUCKET_OWNER,
62                acl.SCHEDULER_OWNER,
63            ],
64            groups = "project-art-admins",
65        ),
66        acl.entry(
67            roles = acl.LOGDOG_WRITER,
68            groups = "luci-logdog-chromium-writers",
69        ),
70    ],
71    bindings = [
72        luci.binding(
73            roles = "role/swarming.poolOwner",
74            groups = "project-art-admins",
75        ),
76        luci.binding(
77            roles = "role/swarming.poolViewer",
78            groups = "googlers",
79        ),
80    ],
81)
82
83# Per-service tweaks.
84luci.logdog(gs_bucket = "chromium-luci-logdog")
85luci.milo(logo = "https://storage.googleapis.com/chrome-infra-public/logo/art-logo.png")
86
87# Allow admins to use LED and "Debug" button on every builder and bot.
88luci.binding(
89    realm = "@root",
90    roles = "role/swarming.poolUser",
91    groups = "project-art-admins",
92)
93luci.binding(
94    realm = "@root",
95    roles = "role/swarming.taskTriggerer",
96    groups = "project-art-admins",
97)
98
99# Resources shared by all subprojects.
100
101luci.realm(name = "pools/ci")
102luci.bucket(name = "ci")
103
104# Shadow bucket is needed for LED.
105luci.bucket(
106    name = "ci.shadow",
107    shadows = "ci",
108    bindings = [
109        luci.binding(
110            roles = "role/buildbucket.creator",
111            users = ["art-ci-builder@chops-service-accounts.iam.gserviceaccount.com"],
112        ),
113        luci.binding(
114            roles = "role/buildbucket.triggerer",
115            users = ["art-ci-builder@chops-service-accounts.iam.gserviceaccount.com"],
116        ),
117    ],
118    constraints = luci.bucket_constraints(
119        pools = ["luci.art.ci"],
120        service_accounts = ["art-ci-builder@chops-service-accounts.iam.gserviceaccount.com"],
121    ),
122    dynamic = True,
123)
124
125luci.notifier_template(
126    name = "default",
127    body = io.read_file("luci-notify.template"),
128)
129
130luci.console_view(
131    name = "luci",
132    repo = "https://android.googlesource.com/platform/art",
133    title = "ART LUCI Console",
134    refs = ["refs/heads/master"],
135    include_experimental_builds = True,
136)
137
138luci.notifier(
139    name = "art-team+chromium-buildbot",
140    on_new_status = [
141        "FAILURE",
142        "INFRA_FAILURE",
143    ],
144    notify_emails = [
145        "art-team+chromium-buildbot@google.com",
146    ],
147)
148
149luci.gitiles_poller(
150    name = "art",
151    bucket = "ci",
152    repo = "https://android.googlesource.com/platform/art",
153    refs = ["refs/heads/master"],
154)
155
156luci.gitiles_poller(
157    name = "libcore",
158    bucket = "ci",
159    repo = "https://android.googlesource.com/platform/libcore",
160    refs = ["refs/heads/master"],
161)
162
163luci.gitiles_poller(
164    name = "vogar",
165    bucket = "ci",
166    repo = "https://android.googlesource.com/platform/external/vogar",
167    refs = ["refs/heads/master"],
168)
169
170luci.gitiles_poller(
171    name = "manifest",
172    bucket = "ci",
173    repo = "https://android.googlesource.com/platform/manifest",
174    refs = ["refs/heads/master-art"],
175)
176
177def ci_builder(name, category, short_name, dimensions, properties={},
178               experiments={}, hidden=False):
179    luci.builder(
180        name = name,
181        bucket = "ci",
182        executable = luci.recipe(
183            cipd_package = "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
184            cipd_version = "refs/heads/main",
185            name = "art",
186        ),
187        dimensions = dimensions | {
188            "pool": "luci.art.ci",
189        },
190        service_account = "art-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
191
192        # Maximum delay between scheduling a build and the build actually starting.
193        # In a healthy state (enough free/idle devices), the delay is fairly small,
194        # but if enough devices are offline, this timeout will cause INFRA_FAILURE.
195        # Set the value reasonably high to prefer delayed builds over failing ones.
196        # NB: LUCI also enforces (expiration_timeout + execution_timeout <= 47).
197        expiration_timeout = 17 * time.hour,
198        execution_timeout = 30 * time.hour,
199        build_numbers = True,
200        properties = properties,
201        caches = [
202            # Directory called "art" that persists from build to build (one per bot).
203            # We can checkout and build in this directory to get fast incremental builds.
204            swarming.cache("art", name = "art"),
205        ],
206        notifies = ["art-team+chromium-buildbot"] if not hidden else [],
207        triggered_by = [
208            "art",
209            "libcore",
210            "manifest",
211            "vogar",
212        ],
213        experiments = experiments,
214    )
215    if not hidden:
216        luci.console_view_entry(
217            console_view = "luci",
218            builder = name,
219            category = category,
220            short_name = short_name,
221        )
222
223def add_builder(mode,
224                arch,
225                bitness,
226                ndebug=False,
227                ngen=False,
228                cmc=False,
229                gcstress=False,
230                poison=False,
231                hidden=False):
232    def check_arg(value, valid_values):
233      if value not in valid_values:
234        fail("Argument '{}' was expected to be on of {}".format(value, valid_values))
235    check_arg(mode, ["target", "host", "qemu"])
236    check_arg(arch, ["arm", "x86", "riscv"])
237    check_arg(bitness, [32, 64])
238
239    # Create builder name based on the configuaration parameters.
240    name = mode + '.' + arch
241    name += '.gcstress' if gcstress else ''
242    name += '.poison' if poison else ''
243    name += '.ngen' if ngen else ''
244    name += '.cmc' if cmc else ''
245    name += '.ndebug' if ndebug else ''
246    name += '.' + str(bitness)
247    name = name.replace("ngen.cmc", "ngen-cmc")
248
249    # Define the LUCI console category (the tree layout).
250    # The "|" splits the tree node into sub-categories.
251    # Merge some name parts to reduce the tree depth.
252    category = name.replace(".", "|")
253    category = category.replace("host|", "host.")
254    category = category.replace("target|", "target.")
255    category = category.replace("gcstress|cmc", "gcstress-cmc")
256
257    product = None
258    if arch == "arm":
259      product = "armv8" if bitness == 64 else "arm_krait"
260    if arch == "riscv":
261      product = "riscv64"
262
263    dimensions = {"os": "Android" if mode == "target" else "Ubuntu"}
264    if mode == "target":
265      if cmc:
266        # Request devices running at least Android 24Q3 (`AP1A` builds) for
267        # (`userfaultfd`-based) Concurrent Mark-Compact GC configurations.
268        # Currently (as of 2024-08-22), the only devices within the device pool
269        # allocated to ART that are running `AP1A` builds are Pixel 6 devices
270        # (all other device types are running older Android versions), which are
271        # also the only device model supporting `userfaultfd` among that pool.
272        dimensions |= {"device_os": "A|B"}
273      else:
274        # Run all other configurations on Android S since it is the oldest we support.
275        # Other than the `AP1A` builds above, all other devices are flashed to `SP2A`.
276        # This avoids allocating `userfaultfd` devices for tests that don't need it.
277        dimensions |= {"device_os": "S"}
278    elif mode == "host":
279      dimensions |= {"cores": "8"}
280    elif mode == "qemu":
281      dimensions |= {"cores": "16"}
282
283    testrunner_args = ['--verbose', '--host'] if mode == 'host' else ['--target', '--verbose']
284    testrunner_args += ['--ndebug'] if ndebug else ['--debug']
285    testrunner_args += ['--gcstress'] if gcstress else []
286
287    properties = {
288        "builder_group": "client.art",
289        "bitness": bitness,
290        "build_only": ("build_only" in name),
291        "debug": not ndebug,
292        "device": None if mode == "host" else "-".join(name.split("-")[:2]),
293        "on_virtual_machine": mode == "qemu",
294        "product": product,
295        "concurrent_collector": not cmc,
296        "generational_cc": not ngen,
297        "gcstress": gcstress,
298        "heap_poisoning": poison,
299        "testrunner_args": testrunner_args,
300    }
301
302    experiments = {"art.superproject": 100} if mode == "qemu" else {}
303
304    ci_builder(name=name,
305               category="|".join(category.split("|")[:-1]),
306               short_name=category.split("|")[-1],
307               dimensions=dimensions,
308               properties={k:v for k, v in properties.items() if v},
309               experiments=experiments,
310               hidden=hidden)
311
312def add_builders():
313  for mode, arch in [("target", "arm"), ("host", "x86")]:
314    for bitness in [32, 64]:
315      # Add first to keep these builders together and left-aligned in the console.
316      add_builder(mode, arch, bitness)
317    for bitness in [32, 64]:
318      add_builder(mode, arch, bitness, ndebug=True)
319      if mode == "host":
320        add_builder(mode, arch, bitness, ngen=True, cmc=True)
321      add_builder(mode, arch, bitness, cmc=True)
322      add_builder(mode, arch, bitness, poison=True)
323      add_builder(mode, arch, bitness, gcstress=True)
324      add_builder(mode, arch, bitness, cmc=True, gcstress=True)
325  add_builder('qemu', 'arm', bitness=64)
326  add_builder('qemu', 'riscv', bitness=64)
327
328add_builders()