• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright © 2020 - 2022 Collabora Ltd.
3# Authors:
4#   Tomeu Vizoso <tomeu.vizoso@collabora.com>
5#   David Heidelberg <david.heidelberg@collabora.com>
6#
7# TODO GraphQL for dependencies
8# SPDX-License-Identifier: MIT
9
10"""
11Helper script to restrict running only required CI jobs
12and show the job(s) logs.
13"""
14
15from typing import Optional
16from functools import partial
17from concurrent.futures import ThreadPoolExecutor
18
19import os
20import re
21import time
22import argparse
23import sys
24import gitlab
25
26from colorama import Fore, Style
27
28REFRESH_WAIT_LOG = 10
29REFRESH_WAIT_JOBS = 6
30
31URL_START = "\033]8;;"
32URL_END = "\033]8;;\a"
33
34STATUS_COLORS = {
35    "created": "",
36    "running": Fore.BLUE,
37    "success": Fore.GREEN,
38    "failed": Fore.RED,
39    "canceled": Fore.MAGENTA,
40    "manual": "",
41    "pending": "",
42    "skipped": "",
43}
44
45# TODO: This hardcoded list should be replaced by querying the pipeline's
46# dependency graph to see which jobs the target jobs need
47DEPENDENCIES = [
48    "debian/x86_build-base",
49    "debian/x86_build",
50    "debian/x86_test-base",
51    "debian/x86_test-gl",
52    "debian/arm_build",
53    "debian/arm_test",
54    "kernel+rootfs_amd64",
55    "kernel+rootfs_arm64",
56    "kernel+rootfs_armhf",
57    "debian-testing",
58    "debian-arm64",
59]
60
61COMPLETED_STATUSES = ["success", "failed"]
62
63
64def get_gitlab_project(glab, name: str):
65    """Finds a specified gitlab project for given user"""
66    glab.auth()
67    username = glab.user.username
68    return glab.projects.get(f"{username}/mesa")
69
70
71def wait_for_pipeline(project, sha: str):
72    """await until pipeline appears in Gitlab"""
73    print("⏲ for the pipeline to appear..", end="")
74    while True:
75        pipelines = project.pipelines.list(sha=sha)
76        if pipelines:
77            print("", flush=True)
78            return pipelines[0]
79        print("", end=".", flush=True)
80        time.sleep(1)
81
82
83def print_job_status(job) -> None:
84    """It prints a nice, colored job status with a link to the job."""
85    if job.status == "canceled":
86        return
87
88    print(
89        STATUS_COLORS[job.status]
90        + "�� job "
91        + URL_START
92        + f"{job.web_url}\a{job.name}"
93        + URL_END
94        + f" :: {job.status}"
95        + Style.RESET_ALL
96    )
97
98
99def print_job_status_change(job) -> None:
100    """It reports job status changes."""
101    if job.status == "canceled":
102        return
103
104    print(
105        STATUS_COLORS[job.status]
106        + "�� job "
107        + URL_START
108        + f"{job.web_url}\a{job.name}"
109        + URL_END
110        + f" has new status: {job.status}"
111        + Style.RESET_ALL
112    )
113
114
115def pretty_wait(sec: int) -> None:
116    """shows progressbar in dots"""
117    for val in range(sec, 0, -1):
118        print(f"⏲  {val} seconds", end="\r")
119        time.sleep(1)
120
121
122def monitor_pipeline(
123    project, pipeline, target_job: Optional[str], dependencies, force_manual: bool
124) -> tuple[Optional[int], Optional[int]]:
125    """Monitors pipeline and delegate canceling jobs"""
126    statuses = {}
127    target_statuses = {}
128
129    if not dependencies:
130        dependencies = []
131    dependencies.extend(DEPENDENCIES)
132
133    if target_job:
134        target_jobs_regex = re.compile(target_job.strip())
135
136    while True:
137        to_cancel = []
138        for job in pipeline.jobs.list(all=True, sort="desc"):
139            # target jobs
140            if target_job and target_jobs_regex.match(job.name):
141                if force_manual and job.status == "manual":
142                    enable_job(project, job, True)
143
144                if (job.id not in target_statuses) or (
145                    job.status not in target_statuses[job.id]
146                ):
147                    print_job_status_change(job)
148                    target_statuses[job.id] = job.status
149                else:
150                    print_job_status(job)
151
152                continue
153
154            # all jobs
155            if (job.id not in statuses) or (job.status not in statuses[job.id]):
156                print_job_status_change(job)
157                statuses[job.id] = job.status
158
159            # dependencies and cancelling the rest
160            if job.name in dependencies:
161                if job.status == "manual":
162                    enable_job(project, job, False)
163
164            elif target_job and job.status not in [
165                "canceled",
166                "success",
167                "failed",
168                "skipped",
169            ]:
170                to_cancel.append(job)
171
172        if target_job:
173            cancel_jobs(project, to_cancel)
174
175        print("---------------------------------", flush=False)
176
177        if len(target_statuses) == 1 and {"running"}.intersection(
178            target_statuses.values()
179        ):
180            return next(iter(target_statuses)), None
181
182        if {"failed", "canceled"}.intersection(target_statuses.values()):
183            return None, 1
184
185        if {"success", "manual"}.issuperset(target_statuses.values()):
186            return None, 0
187
188        pretty_wait(REFRESH_WAIT_JOBS)
189
190
191def enable_job(project, job, target: bool) -> None:
192    """enable manual job"""
193    pjob = project.jobs.get(job.id, lazy=True)
194    pjob.play()
195    if target:
196        jtype = "�� "
197    else:
198        jtype = "(dependency)"
199    print(Fore.MAGENTA + f"{jtype} job {job.name} manually enabled" + Style.RESET_ALL)
200
201
202def cancel_job(project, job) -> None:
203    """Cancel GitLab job"""
204    pjob = project.jobs.get(job.id, lazy=True)
205    pjob.cancel()
206    print(f"♲ {job.name}")
207
208
209def cancel_jobs(project, to_cancel) -> None:
210    """Cancel unwanted GitLab jobs"""
211    if not to_cancel:
212        return
213
214    with ThreadPoolExecutor(max_workers=6) as exe:
215        part = partial(cancel_job, project)
216        exe.map(part, to_cancel)
217
218
219def print_log(project, job_id) -> None:
220    """Print job log into output"""
221    printed_lines = 0
222    while True:
223        job = project.jobs.get(job_id)
224
225        # GitLab's REST API doesn't offer pagination for logs, so we have to refetch it all
226        lines = job.trace().decode("unicode_escape").splitlines()
227        for line in lines[printed_lines:]:
228            print(line)
229        printed_lines = len(lines)
230
231        if job.status in COMPLETED_STATUSES:
232            print(Fore.GREEN + f"Job finished: {job.web_url}" + Style.RESET_ALL)
233            return
234        pretty_wait(REFRESH_WAIT_LOG)
235
236
237def parse_args() -> None:
238    """Parse args"""
239    parser = argparse.ArgumentParser(
240        description="Tool to trigger a subset of container jobs "
241        + "and monitor the progress of a test job",
242        epilog="Example: mesa-monitor.py --rev $(git rev-parse HEAD) "
243        + '--target ".*traces" ',
244    )
245    parser.add_argument("--target", metavar="target-job", help="Target job")
246    parser.add_argument("--deps", nargs="+", help="Job dependencies")
247    parser.add_argument(
248        "--rev", metavar="revision", help="repository git revision", required=True
249    )
250    parser.add_argument(
251        "--token",
252        metavar="token",
253        help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
254    )
255    parser.add_argument(
256        "--force-manual", action="store_true", help="Force jobs marked as manual"
257    )
258    return parser.parse_args()
259
260
261def read_token(token_arg: Optional[str]) -> str:
262    """pick token from args or file"""
263    if token_arg:
264        return token_arg
265    return (
266        open(os.path.expanduser("~/.config/gitlab-token"), encoding="utf-8")
267        .readline()
268        .rstrip()
269    )
270
271
272if __name__ == "__main__":
273    try:
274        t_start = time.perf_counter()
275
276        args = parse_args()
277
278        token = read_token(args.token)
279
280        gl = gitlab.Gitlab(url="https://gitlab.freedesktop.org", private_token=token)
281
282        cur_project = get_gitlab_project(gl, "mesa")
283
284        print(f"Revision: {args.rev}")
285        pipe = wait_for_pipeline(cur_project, args.rev)
286        print(f"Pipeline: {pipe.web_url}")
287        if args.target:
288            print("�� job: " + Fore.BLUE + args.target + Style.RESET_ALL)
289        print(f"Extra dependencies: {args.deps}")
290        target_job_id, ret = monitor_pipeline(
291            cur_project, pipe, args.target, args.deps, args.force_manual
292        )
293
294        if target_job_id:
295            print_log(cur_project, target_job_id)
296
297        t_end = time.perf_counter()
298        spend_minutes = (t_end - t_start) / 60
299        print(f"⏲ Duration of script execution: {spend_minutes:0.1f} minutes")
300
301        sys.exit(ret)
302    except KeyboardInterrupt:
303        sys.exit(1)
304