• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python3
2#
3# Copyright 2021 Advanced Micro Devices, Inc.
4#
5# SPDX-License-Identifier: MIT
6#
7
8import os
9import sys
10import argparse
11import subprocess
12import shutil
13from datetime import datetime
14import tempfile
15import itertools
16import filecmp
17import multiprocessing
18import csv
19
20
21def print_red(txt, end_line=True, prefix=None):
22    if prefix:
23        print(prefix, end="")
24    print("\033[0;31m{}\033[0m".format(txt), end="\n" if end_line else " ")
25
26
27def print_yellow(txt, end_line=True, prefix=None):
28    if prefix:
29        print(prefix, end="")
30    print("\033[1;33m{}\033[0m".format(txt), end="\n" if end_line else " ")
31
32
33def print_green(txt, end_line=True, prefix=None):
34    if prefix:
35        print(prefix, end="")
36    print("\033[1;32m{}\033[0m".format(txt), end="\n" if end_line else " ")
37
38
39parser = argparse.ArgumentParser(
40    description="radeonsi tester",
41    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
42)
43parser.add_argument(
44    "--jobs",
45    "-j",
46    type=int,
47    help="Number of processes/threads to use.",
48    default=multiprocessing.cpu_count(),
49)
50
51# The path to above the mesa directory, i.e. ../../../../../..
52path_above_mesa = os.path.realpath(os.path.join(os.path.dirname(__file__), *['..'] * 6))
53
54parser.add_argument("--piglit-path", type=str, help="Path to piglit source folder.")
55parser.add_argument("--glcts-path", type=str, help="Path to GLCTS source folder.")
56parser.add_argument("--escts-path", type=str, help="Path to GLES CTS source folder.")
57parser.add_argument("--deqp-path", type=str, help="Path to dEQP source folder.")
58parser.add_argument(
59    "--parent-path",
60    type=str,
61    help="Path to folder containing piglit/GLCTS and dEQP source folders.",
62    default=os.getenv('MAREKO_BUILD_PATH', path_above_mesa),
63)
64parser.add_argument("--verbose", "-v", action="count", default=0)
65parser.add_argument(
66    "--include-tests",
67    "-t",
68    action="append",
69    dest="include_tests",
70    default=[],
71    help="Only run the test matching this expression. This can only be a filename containing a list of failing tests to re-run.",
72)
73parser.add_argument(
74    "--baseline",
75    dest="baseline",
76    help="Folder containing expected results files",
77    default=os.path.dirname(__file__),
78)
79parser.add_argument(
80    "--no-piglit", dest="piglit", help="Disable piglit tests", action="store_false"
81)
82parser.add_argument(
83    "--no-glcts", dest="glcts", help="Disable GLCTS tests", action="store_false"
84)
85parser.add_argument(
86    "--no-escts", dest="escts", help="Disable GLES CTS tests", action="store_false"
87)
88parser.add_argument(
89    "--no-deqp", dest="deqp", help="Disable dEQP tests", action="store_false"
90)
91parser.add_argument(
92    "--slow", dest="slow", help="Include slowest glcts tests", action="store_true"
93)
94parser.add_argument(
95    "--no-deqp-egl",
96    dest="deqp_egl",
97    help="Disable dEQP-EGL tests",
98    action="store_false",
99)
100parser.add_argument(
101    "--no-deqp-gles2",
102    dest="deqp_gles2",
103    help="Disable dEQP-gles2 tests",
104    action="store_false",
105)
106parser.add_argument(
107    "--no-deqp-gles3",
108    dest="deqp_gles3",
109    help="Disable dEQP-gles3 tests",
110    action="store_false",
111)
112parser.add_argument(
113    "--no-deqp-gles31",
114    dest="deqp_gles31",
115    help="Disable dEQP-gles31 tests",
116    action="store_false",
117)
118parser.set_defaults(piglit=True)
119parser.set_defaults(glcts=True)
120parser.set_defaults(escts=True)
121parser.set_defaults(deqp=True)
122parser.set_defaults(deqp_egl=True)
123parser.set_defaults(deqp_gles2=True)
124parser.set_defaults(deqp_gles3=True)
125parser.set_defaults(deqp_gles31=True)
126parser.set_defaults(slow=False)
127
128parser.add_argument(
129    "output_folder",
130    nargs="?",
131    help="Output folder (logs, etc)",
132    default=os.path.join(
133        # Default is ../../../../../../test-results/datetime
134        os.path.join(path_above_mesa, 'test-results',
135                     datetime.now().strftime("%Y-%m-%d-%H-%M-%S"))
136    ),
137)
138
139available_gpus = []
140for f in os.listdir("/dev/dri/by-path"):
141    idx = f.find("-render")
142    if idx < 0:
143        continue
144    # gbm name is the full path, but DRI_PRIME expects a different
145    # format
146    available_gpus += [
147        (
148            os.path.join("/dev/dri/by-path", f),
149            f[:idx].replace(":", "_").replace(".", "_"),
150        )
151    ]
152
153parser.add_argument(
154    "--gpu",
155    type=int,
156    dest="gpu",
157    default=0,
158    help="Select GPU (0..{})".format(len(available_gpus) - 1),
159)
160
161args = parser.parse_args(sys.argv[1:])
162piglit_path = args.piglit_path
163glcts_path = args.glcts_path
164escts_path = args.escts_path
165deqp_path = args.deqp_path
166
167if args.parent_path:
168    if args.piglit_path or args.glcts_path or args.deqp_path:
169        parser.print_help()
170        sys.exit(0)
171    piglit_path = os.path.join(args.parent_path, "piglit")
172    glcts_path = os.path.join(args.parent_path, "glcts")
173    escts_path = os.path.join(args.parent_path, "escts")
174    deqp_path = os.path.join(args.parent_path, "deqp")
175else:
176    if not args.piglit_path or not args.glcts_path or not args.escts_path or not args.deqp_path:
177        parser.print_help()
178        sys.exit(0)
179
180base = args.baseline
181skips = os.path.join(os.path.dirname(__file__), "skips.csv")
182
183env = os.environ.copy()
184
185if "DISPLAY" not in env:
186    print_red("DISPLAY environment variable missing.")
187    sys.exit(1)
188p = subprocess.run(
189    ["deqp-runner", "--version"], capture_output="True", check=True, env=env
190)
191for line in p.stdout.decode().split("\n"):
192    if line.find("deqp-runner") >= 0:
193        s = line.split(" ")[1].split(".")
194        if args.verbose > 1:
195            print("Checking deqp-version ({})".format(s))
196        # We want at least 0.9.0
197        if not (int(s[0]) > 0 or int(s[1]) >= 9):
198            print("Expecting deqp-runner 0.9.0+ version (got {})".format(".".join(s)))
199            sys.exit(1)
200
201env["PIGLIT_PLATFORM"] = "gbm"
202
203if "DRI_PRIME" in env:
204    print("Don't use DRI_PRIME. Instead use --gpu N")
205    del env["DRI_PRIME"]
206
207assert "gpu" in args, "--gpu defaults to 0"
208
209gpu_device = available_gpus[args.gpu][1]
210env["DRI_PRIME"] = gpu_device
211env["WAFFLE_GBM_DEVICE"] = available_gpus[args.gpu][0]
212
213# Use piglit's glinfo to determine the GPU name
214gpu_name = "unknown"
215gpu_name_full = ""
216gfx_level = -1
217
218env["AMD_DEBUG"] = "info"
219p = subprocess.run(
220    ["./glinfo"],
221    capture_output="True",
222    cwd=os.path.join(piglit_path, "bin"),
223    check=True,
224    env=env,
225)
226del env["AMD_DEBUG"]
227for line in p.stdout.decode().split("\n"):
228    if "GL_RENDER" in line:
229        line = line.split("=")[1]
230        gpu_name_full = "(".join(line.split("(")[:-1]).strip()
231        gpu_name = line.replace("(TM)", "").split("(")[1].split(",")[0].lower()
232        break
233    elif "gfx_level" in line:
234        gfx_level = int(line.split("=")[1])
235
236output_folder = args.output_folder
237print_green("Tested GPU: '{}' ({}) {}".format(gpu_name_full, gpu_name, gpu_device))
238print_green("Output folder: '{}'".format(output_folder))
239
240count = 1
241while os.path.exists(output_folder):
242    output_folder = "{}.{}".format(os.path.abspath(args.output_folder), count)
243    count += 1
244
245os.makedirs(output_folder, exist_ok=True)
246
247logfile = open(os.path.join(output_folder, "{}-run-tests.log".format(gpu_name)), "w")
248
249spin = itertools.cycle("-\\|/")
250
251shutil.copy(skips, output_folder)
252skips = os.path.join(output_folder, "skips.csv")
253if not args.slow:
254    # Exclude these 4 tests slow tests
255    with open(skips, "a") as f:
256        print("KHR-GL46.copy_image.functional", file=f)
257        print("KHR-GL46.texture_swizzle.smoke", file=f)
258        print(
259            "KHR-GL46.tessellation_shader.tessellation_control_to_tessellation_evaluation.gl_MaxPatchVertices_Position_PointSize",
260            file=f,
261        )
262        print("KHR-Single-GL46.arrays_of_arrays_gl.AtomicUsage", file=f)
263
264
265def gfx_level_to_str(cl):
266    supported = ["gfx6", "gfx7", "gfx8", "gfx9", "gfx10", "gfx10_3", "gfx11"]
267    if 8 <= cl and cl < 8 + len(supported):
268        return supported[cl - 8]
269    return supported[-1]
270
271
272def run_cmd(args, verbosity):
273    if verbosity > 1:
274        print_yellow(
275            "| Command line argument '"
276            + " ".join(['"{}"'.format(a) for a in args])
277            + "'"
278        )
279    start = datetime.now()
280    proc = subprocess.Popen(
281        args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env
282    )
283    while True:
284        line = proc.stdout.readline().decode()
285        if verbosity > 0:
286            if "ERROR" in line:
287                print_red(line.strip(), prefix="| ")
288            else:
289                print("| " + line.strip())
290        else:
291            sys.stdout.write(next(spin))
292            sys.stdout.flush()
293            sys.stdout.write("\b")
294
295        logfile.write(line)
296
297        if proc.poll() is not None:
298            break
299    proc.wait()
300    end = datetime.now()
301
302    if verbosity == 0:
303        sys.stdout.write(" ... ")
304
305    print_yellow(
306        "Completed in {} seconds".format(int((end - start).total_seconds())),
307        prefix="└ " if verbosity > 0 else None,
308    )
309
310
311def verify_results(results):
312    with open(results) as file:
313        if len(file.readlines()) == 0:
314            return True
315    print_red("New results (fails or pass). Check {}".format(results))
316    return False
317
318
319def parse_test_filters(include_tests):
320    cmd = []
321    for t in include_tests:
322        if os.path.exists(t):
323            with open(t, "r") as file:
324                for row in csv.reader(file, delimiter=","):
325                    if not row or row[0][0] == "#":
326                        continue
327                    cmd += ["-t", row[0]]
328        else:
329            cmd += ["-t", t]
330    return cmd
331
332
333def select_baseline(basepath, gfx_level, gpu_name):
334    gfx_level_str = gfx_level_to_str(gfx_level)
335
336    # select the best baseline we can find
337    # 1. exact match
338    exact = os.path.join(base, "{}-{}-fail.csv".format(gfx_level_str, gpu_name))
339    if os.path.exists(exact):
340        return exact
341    # 2. any baseline with the same gfx_level
342    while gfx_level >= 8:
343        for subdir, dirs, files in os.walk(basepath):
344            for file in files:
345                if file.find(gfx_level_str) == 0 and file.endswith("-fail.csv"):
346                    return os.path.join(base, file)
347        # No match. Try an earlier class
348        gfx_level = gfx_level - 1
349        gfx_level_str = gfx_level_to_str(gfx_level)
350
351    return exact
352
353
354success = True
355filters_args = parse_test_filters(args.include_tests)
356baseline = select_baseline(base, gfx_level, gpu_name)
357flakes = [
358    f
359    for f in (
360        os.path.join(base, g)
361        for g in [
362            "radeonsi-flakes.csv",
363            "{}-{}-flakes.csv".format(gfx_level_to_str(gfx_level), gpu_name),
364        ]
365    )
366    if os.path.exists(f)
367]
368flakes_args = []
369for f in flakes:
370    flakes_args += ["--flakes", f]
371
372if os.path.exists(baseline):
373    print_yellow("Baseline: {}".format(baseline))
374if flakes_args:
375    print_yellow("Flakes: {}".format(flakes_args))
376
377# piglit test
378if args.piglit:
379    out = os.path.join(output_folder, "piglit")
380    print_yellow("Running piglit tests", args.verbose > 0)
381    cmd = [
382        "piglit-runner",
383        "run",
384        "--piglit-folder",
385        piglit_path,
386        "--profile",
387        "quick",
388        "--output",
389        out,
390        "--process-isolation",
391        "--timeout",
392        "300",
393        "--jobs",
394        str(args.jobs),
395        "--skips",
396        skips,
397        "--skips",
398        os.path.join(path_above_mesa, "mesa", ".gitlab-ci", "gbm-skips.txt")
399    ] + filters_args + flakes_args
400
401    if os.path.exists(baseline):
402        cmd += ["--baseline", baseline]
403
404    run_cmd(cmd, args.verbose)
405
406    if not verify_results(os.path.join(out, "failures.csv")):
407        success = False
408
409deqp_args = "-- --deqp-surface-width=256 --deqp-surface-height=256 --deqp-gl-config-name=rgba8888d24s8ms0 --deqp-visibility=hidden".split(
410    " "
411)
412
413# glcts test
414if args.glcts:
415    out = os.path.join(output_folder, "glcts")
416    print_yellow("Running  GLCTS tests", args.verbose > 0)
417    os.mkdir(os.path.join(output_folder, "glcts"))
418
419    cmd = [
420        "deqp-runner",
421        "run",
422        "--tests-per-group",
423        "100",
424        "--deqp",
425        "{}/build/external/openglcts/modules/glcts".format(glcts_path),
426        "--caselist",
427        "{}/build/external/openglcts/modules/gl_cts/data/mustpass/gl/khronos_mustpass/4.6.1.x/gl46-master.txt".format(
428            glcts_path
429        ),
430        "--caselist",
431        "{}/build/external/openglcts/modules/gl_cts/data/mustpass/gl/khronos_mustpass_single/4.6.1.x/gl46-khr-single.txt".format(
432            glcts_path
433        ),
434        "--caselist",
435        "{}/build/external/openglcts/modules/gl_cts/data/mustpass/gl/khronos_mustpass/4.6.1.x/gl46-gtf-master.txt".format(
436            glcts_path
437        ),
438        "--output",
439        out,
440        "--skips",
441        skips,
442        "--jobs",
443        str(args.jobs),
444        "--timeout",
445        "1000"
446    ] + filters_args + flakes_args
447
448    if os.path.exists(baseline):
449        cmd += ["--baseline", baseline]
450    cmd += deqp_args
451
452    run_cmd(cmd, args.verbose)
453
454    if not verify_results(os.path.join(out, "failures.csv")):
455        success = False
456
457# escts test
458if args.escts:
459    out = os.path.join(output_folder, "escts")
460    print_yellow("Running  ESCTS tests", args.verbose > 0)
461    os.mkdir(out)
462
463    cmd = [
464        "deqp-runner",
465        "run",
466        "--tests-per-group",
467        "100",
468        "--deqp",
469        "{}/build/external/openglcts/modules/glcts".format(escts_path),
470        "--caselist",
471        "{}/build/external/openglcts/modules/gl_cts/data/mustpass/gles/khronos_mustpass/3.2.6.x/gles2-khr-master.txt".format(
472            escts_path
473        ),
474        "--caselist",
475        "{}/build/external/openglcts/modules/gl_cts/data/mustpass/gles/khronos_mustpass/3.2.6.x/gles3-khr-master.txt".format(
476            escts_path
477        ),
478        "--caselist",
479        "{}/build/external/openglcts/modules/gl_cts/data/mustpass/gles/khronos_mustpass/3.2.6.x/gles31-khr-master.txt".format(
480            escts_path
481        ),
482        "--caselist",
483        "{}/build/external/openglcts/modules/gl_cts/data/mustpass/gles/khronos_mustpass/3.2.6.x/gles32-khr-master.txt".format(
484            escts_path
485        ),
486        "--output",
487        out,
488        "--skips",
489        skips,
490        "--jobs",
491        str(args.jobs),
492        "--timeout",
493        "1000"
494    ] + filters_args + flakes_args
495
496    if os.path.exists(baseline):
497        cmd += ["--baseline", baseline]
498    cmd += deqp_args
499
500    run_cmd(cmd, args.verbose)
501
502    if not verify_results(os.path.join(out, "failures.csv")):
503        success = False
504
505if args.deqp:
506    print_yellow("Running   dEQP tests", args.verbose > 0)
507
508    # Generate a test-suite file
509    out = os.path.join(output_folder, "deqp")
510    suite_filename = os.path.join(output_folder, "deqp-suite.toml")
511    suite = open(suite_filename, "w")
512    os.mkdir(out)
513
514    deqp_tests = {
515        "egl": args.deqp_egl,
516        "gles2": args.deqp_gles2,
517        "gles3": args.deqp_gles3,
518        "gles31": args.deqp_gles31,
519    }
520
521    for k in deqp_tests:
522        if not deqp_tests[k]:
523            continue
524
525        suite.write("[[deqp]]\n")
526        suite.write(
527            'deqp = "{}"\n'.format(
528                "{}/build/modules/{subtest}/deqp-{subtest}".format(deqp_path, subtest=k)
529            )
530        )
531        suite.write(
532            'caselists = ["{}"]\n'.format(
533                "{}/android/cts/main/{}-master.txt".format(deqp_path, k)
534            )
535        )
536        if os.path.exists(baseline):
537            suite.write('baseline = "{}"\n'.format(baseline))
538        suite.write('skips = ["{}"]\n'.format(skips))
539        suite.write("deqp_args = [\n")
540        for a in deqp_args[1:-1]:
541            suite.write('    "{}",\n'.format(a))
542        suite.write('    "{}"\n'.format(deqp_args[-1]))
543        suite.write("]\n")
544
545    suite.close()
546
547    cmd = [
548        "deqp-runner",
549        "suite",
550        "--jobs",
551        str(args.jobs),
552        "--output",
553        os.path.join(output_folder, "deqp"),
554        "--suite",
555        suite_filename,
556    ] + filters_args + flakes_args
557
558    run_cmd(cmd, args.verbose)
559
560    if not verify_results(os.path.join(out, "failures.csv")):
561        success = False
562
563sys.exit(0 if success else 1)
564