• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env lucicfg
2#
3# Copyright 2021 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7# main.star: lucicfg configuration for ANGLE's standalone builders.
8
9lucicfg.check_version(min = "1.31.3", message = "Update depot_tools")
10
11# Use LUCI Scheduler BBv2 names and add Scheduler realms configs.
12lucicfg.enable_experiment("crbug.com/1182002")
13
14# Fail build when merge script fails.
15build_experiments = {"chromium_swarming.expose_merge_script_failures": 100}
16
17lucicfg.config(
18    fail_on_warnings = True,
19    lint_checks = [
20        "default",
21        "-module-docstring",
22        "-function-docstring",
23    ],
24)
25
26luci.project(
27    name = "angle",
28    buildbucket = "cr-buildbucket.appspot.com",
29    logdog = "luci-logdog.appspot.com",
30    milo = "luci-milo.appspot.com",
31    notify = "luci-notify.appspot.com",
32    scheduler = "luci-scheduler.appspot.com",
33    swarming = "chromium-swarm.appspot.com",
34    acls = [
35        acl.entry(
36            roles = [
37                acl.PROJECT_CONFIGS_READER,
38                acl.LOGDOG_READER,
39                acl.BUILDBUCKET_READER,
40                acl.SCHEDULER_READER,
41            ],
42            groups = "all",
43        ),
44        acl.entry(
45            roles = [
46                acl.SCHEDULER_OWNER,
47            ],
48            groups = "project-angle-admins",
49        ),
50        acl.entry(
51            roles = [
52                acl.LOGDOG_WRITER,
53            ],
54            groups = "luci-logdog-angle-writers",
55        ),
56    ],
57    bindings = [
58        luci.binding(
59            roles = "role/configs.validator",
60            users = "angle-try-builder@chops-service-accounts.iam.gserviceaccount.com",
61        ),
62        luci.binding(
63            roles = "role/swarming.poolOwner",
64            groups = ["project-angle-owners", "mdb/chrome-troopers"],
65        ),
66        luci.binding(
67            roles = "role/swarming.poolViewer",
68            groups = "all",
69        ),
70        # Allow any Angle build to trigger a test ran under testing accounts
71        # used on shared chromium tester pools.
72        luci.binding(
73            roles = "role/swarming.taskServiceAccount",
74            users = [
75                "chromium-tester@chops-service-accounts.iam.gserviceaccount.com",
76                "chrome-gpu-gold@chops-service-accounts.iam.gserviceaccount.com",
77            ],
78        ),
79    ],
80)
81
82# Swarming permissions
83luci.realm(name = "pools/ci")
84luci.realm(name = "pools/try")
85
86# Allow Angle owners and Chrome troopers to run tasks directly for testing and
87# development on all Angle bots. E.g. via `led` tool or "Debug" button in Swarming Web UI.
88luci.binding(
89    realm = "@root",
90    roles = "role/swarming.poolUser",
91    groups = ["project-angle-owners", "mdb/chrome-troopers"],
92)
93luci.binding(
94    realm = "@root",
95    roles = "role/swarming.taskTriggerer",
96    groups = ["project-angle-owners", "mdb/chrome-troopers"],
97)
98
99def _generate_project_pyl(ctx):
100    ctx.output["project.pyl"] = "\n".join([
101        "# This is a non-LUCI generated file",
102        "# This is consumed by presubmit checks that need to validate the config",
103        repr(dict(
104            # We don't validate matching source-side configs for simplicity.
105            validate_source_side_specs_have_builder = False,
106        )),
107        "",
108    ])
109
110lucicfg.generator(_generate_project_pyl)
111
112luci.milo(
113    logo = "https://storage.googleapis.com/chrome-infra/OpenGL%20ES_RGB_June16.svg",
114    bug_url_template = "https://bugs.chromium.org/p/angleproject/issues/entry?components=Infra",
115)
116
117luci.logdog(gs_bucket = "chromium-luci-logdog")
118
119# The category for an os: a more generic grouping than specific OS versions that
120# can be used for computing defaults
121os_category = struct(
122    ANDROID = "Android",
123    LINUX = "Linux",
124    MAC = "Mac",
125    WINDOWS = "Windows",
126)
127
128def os_enum(dimension, category, console_name):
129    return struct(dimension = dimension, category = category, console_name = console_name)
130
131os = struct(
132    ANDROID = os_enum("Ubuntu", os_category.ANDROID, "android"),
133    LINUX = os_enum("Ubuntu", os_category.LINUX, "linux"),
134    MAC = os_enum("Mac", os_category.MAC, "mac"),
135    WINDOWS = os_enum("Windows", os_category.WINDOWS, "win"),
136)
137
138# Recipes
139
140_RECIPE_NAME_PREFIX = "recipe:"
141_DEFAULT_BUILDERLESS_OS_CATEGORIES = [os_category.LINUX, os_category.WINDOWS]
142
143def _recipe_for_package(cipd_package):
144    def recipe(*, name, cipd_version = None, recipe = None, use_python3 = False):
145        # Force the caller to put the recipe prefix rather than adding it
146        # programatically to make the string greppable
147        if not name.startswith(_RECIPE_NAME_PREFIX):
148            fail("Recipe name {!r} does not start with {!r}"
149                .format(name, _RECIPE_NAME_PREFIX))
150        if recipe == None:
151            recipe = name[len(_RECIPE_NAME_PREFIX):]
152        return luci.recipe(
153            name = name,
154            cipd_package = cipd_package,
155            cipd_version = cipd_version,
156            recipe = recipe,
157            use_bbagent = True,
158            use_python3 = use_python3,
159        )
160
161    return recipe
162
163build_recipe = _recipe_for_package(
164    "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
165)
166
167build_recipe(
168    name = "recipe:angle",
169    use_python3 = True,
170)
171
172build_recipe(
173    name = "recipe:run_presubmit",
174    use_python3 = True,
175)
176
177def get_os_from_name(name):
178    if name.startswith("android"):
179        return os.ANDROID
180    if name.startswith("linux"):
181        return os.LINUX
182    if name.startswith("win"):
183        return os.WINDOWS
184    if name.startswith("mac"):
185        return os.MAC
186    return os.MAC
187
188def get_gpu_type_from_builder_name(name):
189    return name.split("-")[1]
190
191# Adds both the CI and Try standalone builders.
192def angle_builder(name, cpu):
193    config_os = get_os_from_name(name)
194    dimensions = {}
195    dimensions["os"] = config_os.dimension
196
197    if config_os.category in _DEFAULT_BUILDERLESS_OS_CATEGORIES:
198        dimensions["builderless"] = "1"
199
200    is_asan = "-asan" in name
201    is_tsan = "-tsan" in name
202    is_debug = "-dbg" in name
203    is_exp = "-exp" in name
204    is_perf = name.endswith("-perf")
205    is_s22 = "s22" in name
206    is_trace = name.endswith("-trace")
207    is_uwp = "winuwp" in name
208    is_msvc = is_uwp or "-msvc" in name
209
210    location_filters = None
211
212    if name.endswith("-compile"):
213        test_mode = "compile_only"
214        category = "compile"
215    elif name.endswith("-test"):
216        test_mode = "compile_and_test"
217        category = "test"
218    elif is_trace:
219        test_mode = "trace_tests"
220        category = "trace"
221
222        # Trace tests are only run on CQ if files in the capture folders change.
223        location_filters = [
224            cq.location_filter(path_regexp = "DEPS"),
225            cq.location_filter(path_regexp = "src/libANGLE/capture/.+"),
226            cq.location_filter(path_regexp = "src/tests/angle_end2end_tests_expectations.txt"),
227            cq.location_filter(path_regexp = "src/tests/capture.+"),
228            cq.location_filter(path_regexp = "src/tests/egl_tests/.+"),
229            cq.location_filter(path_regexp = "src/tests/gl_tests/.+"),
230        ]
231    elif is_perf:
232        test_mode = "compile_and_test"
233        category = "perf"
234    else:
235        print("Test mode unknown for %s" % name)
236
237    if is_msvc:
238        toolchain = "msvc"
239    else:
240        toolchain = "clang"
241
242    if is_uwp:
243        os_toolchain_name = "win-uwp"
244    elif is_msvc:
245        os_toolchain_name = "win-msvc"
246    else:
247        os_toolchain_name = config_os.console_name
248
249    if is_perf:
250        short_name = get_gpu_type_from_builder_name(name)
251    elif is_asan:
252        short_name = "asan"
253        if is_exp:
254            short_name = "asan-exp"
255    elif is_tsan:
256        short_name = "tsan"
257        if is_exp:
258            short_name = "tsan-exp"
259    elif is_debug:
260        short_name = "dbg"
261    elif is_exp:
262        short_name = "exp"
263        if is_s22:
264            # This is a little clunky, but we'd like this to be cleanly "s22" rather than "s22-exp"
265            short_name = "s22"
266    else:
267        short_name = "rel"
268
269    properties = {
270        "builder_group": "angle",
271        # TODO: crbug.com/401959048 - Remove reclient props after migration.
272        "$build/reclient": {
273            "instance": "rbe-chromium-untrusted",
274            "metrics_project": "chromium-reclient-metrics",
275            "scandeps_server": True,
276        },
277        "$build/siso": {
278            "project": "rbe-chromium-untrusted",
279            "configs": ["builder"],
280            "enable_cloud_monitoring": True,
281            "enable_cloud_profiler": True,
282            "enable_cloud_trace": True,
283            "metrics_project": "chromium-reclient-metrics",
284        },
285        "platform": config_os.console_name,
286        "toolchain": toolchain,
287        "test_mode": test_mode,
288    }
289
290    ci_properties = {
291        "builder_group": "angle",
292        # TODO: crbug.com/401959048 - Remove reclient props after migration.
293        "$build/reclient": {
294            "instance": "rbe-chromium-trusted",
295            "metrics_project": "chromium-reclient-metrics",
296            "scandeps_server": True,
297        },
298        "$build/siso": {
299            "project": "rbe-chromium-trusted",
300            "configs": ["builder"],
301            "enable_cloud_monitoring": True,
302            "enable_cloud_profiler": True,
303            "enable_cloud_trace": True,
304            "metrics_project": "chromium-reclient-metrics",
305        },
306        "platform": config_os.console_name,
307        "toolchain": toolchain,
308        "test_mode": test_mode,
309    }
310
311    # TODO(343503161): Remove sheriff_rotations after SoM is updated.
312    ci_properties["gardener_rotations"] = ["angle"]
313    ci_properties["sheriff_rotations"] = ["angle"]
314
315    if is_perf:
316        timeout_hours = 5
317    else:
318        timeout_hours = 3
319
320    luci.builder(
321        name = name,
322        bucket = "ci",
323        triggered_by = ["main-poller"],
324        executable = "recipe:angle",
325        experiments = build_experiments,
326        service_account = "angle-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
327        shadow_service_account = "angle-try-builder@chops-service-accounts.iam.gserviceaccount.com",
328        properties = ci_properties,
329        dimensions = dimensions,
330        build_numbers = True,
331        resultdb_settings = resultdb.settings(enable = True),
332        test_presentation = resultdb.test_presentation(
333            column_keys = ["v.gpu"],
334            grouping_keys = ["status", "v.test_suite"],
335        ),
336        triggering_policy = scheduler.policy(
337            kind = scheduler.LOGARITHMIC_BATCHING_KIND,
338            log_base = 2,
339        ),
340        execution_timeout = timeout_hours * time.hour,
341    )
342
343    active_experimental_builders = [
344        "android-arm64-exp-test",
345        "android-arm64-exp-s22-test",
346        "linux-exp-test",
347        "mac-exp-test",
348        "win-exp-test",
349    ]
350
351    if (not is_exp) or (name in active_experimental_builders):
352        luci.console_view_entry(
353            console_view = "ci",
354            builder = "ci/" + name,
355            category = category + "|" + os_toolchain_name + "|" + cpu,
356            short_name = short_name,
357        )
358    else:
359        luci.list_view_entry(
360            list_view = "exp",
361            builder = "ci/" + name,
362        )
363
364    # Do not include perf tests in "try".
365    if not is_perf:
366        luci.list_view_entry(
367            list_view = "try",
368            builder = "try/" + name,
369        )
370
371        luci.builder(
372            name = name,
373            bucket = "try",
374            executable = "recipe:angle",
375            experiments = build_experiments,
376            service_account = "angle-try-builder@chops-service-accounts.iam.gserviceaccount.com",
377            properties = properties,
378            dimensions = dimensions,
379            build_numbers = True,
380            resultdb_settings = resultdb.settings(enable = True),
381            test_presentation = resultdb.test_presentation(
382                column_keys = ["v.gpu"],
383                grouping_keys = ["status", "v.test_suite"],
384            ),
385        )
386
387        # Don't add experimental bots to CQ.
388        if not is_exp:
389            luci.cq_tryjob_verifier(
390                cq_group = "main",
391                builder = "angle:try/" + name,
392                location_filters = location_filters,
393            )
394
395luci.bucket(
396    name = "ci",
397    acls = [
398        acl.entry(
399            acl.BUILDBUCKET_TRIGGERER,
400            users = [
401                "angle-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
402            ],
403        ),
404    ],
405)
406
407luci.bucket(
408    name = "try",
409    acls = [
410        acl.entry(
411            acl.BUILDBUCKET_TRIGGERER,
412            groups = [
413                "project-angle-tryjob-access",
414                "service-account-cq",
415            ],
416        ),
417    ],
418)
419
420# Shadow buckets for LED jobs.
421luci.bucket(
422    name = "ci.shadow",
423    shadows = "ci",
424    constraints = luci.bucket_constraints(
425        pools = ["luci.angle.ci"],
426    ),
427    bindings = [
428        luci.binding(
429            roles = "role/buildbucket.creator",
430            groups = [
431                "mdb/chrome-build-access-sphinx",
432                "mdb/chrome-troopers",
433                "chromium-led-users",
434            ],
435            users = [
436                "angle-try-builder@chops-service-accounts.iam.gserviceaccount.com",
437            ],
438        ),
439        luci.binding(
440            roles = "role/buildbucket.triggerer",
441            users = [
442                "angle-try-builder@chops-service-accounts.iam.gserviceaccount.com",
443            ],
444        ),
445        # Allow ci builders to create invocations in their own builds.
446        luci.binding(
447            roles = "role/resultdb.invocationCreator",
448            users = [
449                "angle-try-builder@chops-service-accounts.iam.gserviceaccount.com",
450            ],
451        ),
452    ],
453    dynamic = True,
454)
455
456luci.bucket(
457    name = "try.shadow",
458    shadows = "try",
459    constraints = luci.bucket_constraints(
460        pools = ["luci.angle.try"],
461        service_accounts = [
462            "angle-try-builder@chops-service-accounts.iam.gserviceaccount.com",
463        ],
464    ),
465    bindings = [
466        luci.binding(
467            roles = "role/buildbucket.creator",
468            groups = [
469                "mdb/chrome-build-access-sphinx",
470                "mdb/chrome-troopers",
471                "chromium-led-users",
472            ],
473            users = [
474                "angle-try-builder@chops-service-accounts.iam.gserviceaccount.com",
475            ],
476        ),
477        luci.binding(
478            roles = "role/buildbucket.triggerer",
479            users = [
480                "angle-try-builder@chops-service-accounts.iam.gserviceaccount.com",
481            ],
482        ),
483        # Allow try builders to create invocations in their own builds.
484        luci.binding(
485            roles = "role/resultdb.invocationCreator",
486            groups = [
487                "project-angle-try-task-accounts",
488                "project-angle-tryjob-access",
489            ],
490        ),
491    ],
492    dynamic = True,
493)
494
495luci.builder(
496    name = "presubmit",
497    bucket = "try",
498    executable = "recipe:run_presubmit",
499    experiments = build_experiments,
500    service_account = "angle-try-builder@chops-service-accounts.iam.gserviceaccount.com",
501    build_numbers = True,
502    dimensions = {
503        "os": os.LINUX.dimension,
504    },
505    properties = {
506        "repo_name": "angle",
507        "runhooks": True,
508    },
509    resultdb_settings = resultdb.settings(enable = True),
510    test_presentation = resultdb.test_presentation(
511        column_keys = ["v.gpu"],
512        grouping_keys = ["status", "v.test_suite"],
513    ),
514)
515
516luci.gitiles_poller(
517    name = "main-poller",
518    bucket = "ci",
519    repo = "https://chromium.googlesource.com/angle/angle",
520    refs = [
521        "refs/heads/main",
522    ],
523    schedule = "with 10s interval",
524)
525
526# name, clang, debug, cpu, uwp, trace_tests
527angle_builder("android-arm-compile", cpu = "arm")
528angle_builder("android-arm-dbg-compile", cpu = "arm")
529angle_builder("android-arm64-dbg-compile", cpu = "arm64")
530angle_builder("android-arm64-exp-s22-test", cpu = "arm64")
531angle_builder("android-arm64-exp-test", cpu = "arm64")
532angle_builder("android-arm64-test", cpu = "arm64")
533angle_builder("linux-asan-test", cpu = "x64")
534angle_builder("linux-exp-asan-test", cpu = "x64")
535angle_builder("linux-exp-test", cpu = "x64")
536angle_builder("linux-exp-tsan-test", cpu = "x64")
537angle_builder("linux-tsan-test", cpu = "x64")
538angle_builder("linux-dbg-compile", cpu = "x64")
539angle_builder("linux-test", cpu = "x64")
540angle_builder("mac-dbg-compile", cpu = "x64")
541angle_builder("mac-exp-test", cpu = "x64")
542angle_builder("mac-test", cpu = "x64")
543angle_builder("win-asan-test", cpu = "x64")
544angle_builder("win-dbg-compile", cpu = "x64")
545angle_builder("win-exp-test", cpu = "x64")
546angle_builder("win-msvc-compile", cpu = "x64")
547angle_builder("win-msvc-dbg-compile", cpu = "x64")
548angle_builder("win-msvc-x86-compile", cpu = "x86")
549angle_builder("win-msvc-x86-dbg-compile", cpu = "x86")
550angle_builder("win-test", cpu = "x64")
551angle_builder("win-x86-dbg-compile", cpu = "x86")
552angle_builder("win-x86-test", cpu = "x86")
553angle_builder("winuwp-compile", cpu = "x64")
554angle_builder("winuwp-dbg-compile", cpu = "x64")
555
556angle_builder("linux-trace", cpu = "x64")
557angle_builder("win-trace", cpu = "x64")
558
559angle_builder("android-pixel4-perf", cpu = "arm64")
560angle_builder("android-pixel6-perf", cpu = "arm64")
561angle_builder("linux-intel-uhd630-perf", cpu = "x64")
562angle_builder("linux-nvidia-gtx1660-perf", cpu = "x64")
563angle_builder("win10-intel-uhd630-perf", cpu = "x64")
564angle_builder("win10-nvidia-gtx1660-perf", cpu = "x64")
565
566# Views
567
568luci.console_view(
569    name = "ci",
570    title = "ANGLE CI Builders",
571    repo = "https://chromium.googlesource.com/angle/angle",
572)
573
574luci.list_view(
575    name = "exp",
576    title = "ANGLE Experimental CI Builders",
577)
578
579luci.list_view(
580    name = "try",
581    title = "ANGLE Try Builders",
582)
583
584luci.list_view_entry(
585    list_view = "try",
586    builder = "try/presubmit",
587)
588
589# CQ
590
591luci.cq(
592    status_host = "chromium-cq-status.appspot.com",
593    submit_max_burst = 4,
594    submit_burst_delay = 480 * time.second,
595)
596
597luci.cq_group(
598    name = "main",
599    watch = cq.refset(
600        "https://chromium.googlesource.com/angle/angle",
601        refs = [r"refs/heads/main"],
602    ),
603    acls = [
604        acl.entry(
605            acl.CQ_COMMITTER,
606            groups = "project-angle-submit-access",
607        ),
608        acl.entry(
609            acl.CQ_DRY_RUNNER,
610            groups = "project-angle-tryjob-access",
611        ),
612    ],
613    verifiers = [
614        luci.cq_tryjob_verifier(
615            builder = "angle:try/presubmit",
616            disable_reuse = True,
617        ),
618        luci.cq_tryjob_verifier(
619            builder = "chromium:try/android-angle-chromium-try",
620        ),
621        luci.cq_tryjob_verifier(
622            builder = "chromium:try/fuchsia-angle-try",
623        ),
624        luci.cq_tryjob_verifier(
625            builder = "chromium:try/linux-angle-chromium-try",
626        ),
627        luci.cq_tryjob_verifier(
628            builder = "chromium:try/mac-angle-chromium-try",
629        ),
630        luci.cq_tryjob_verifier(
631            builder = "chromium:try/win-angle-chromium-x64-try",
632        ),
633        luci.cq_tryjob_verifier(
634            builder = "chromium:try/win-angle-chromium-x86-try",
635        ),
636    ],
637)
638