• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3#  Copyright (c) 2020, The OpenThread Authors.
4#  All rights reserved.
5#
6#  Redistribution and use in source and binary forms, with or without
7#  modification, are permitted provided that the following conditions are met:
8#  1. Redistributions of source code must retain the above copyright
9#     notice, this list of conditions and the following disclaimer.
10#  2. Redistributions in binary form must reproduce the above copyright
11#     notice, this list of conditions and the following disclaimer in the
12#     documentation and/or other materials provided with the distribution.
13#  3. Neither the name of the copyright holder nor the
14#     names of its contributors may be used to endorse or promote products
15#     derived from this software without specific prior written permission.
16#
17#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20#  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
21#  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22#  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23#  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24#  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25#  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26#  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27#  POSSIBILITY OF SUCH DAMAGE.
28#
29import logging
30import multiprocessing
31import os
32import queue
33import subprocess
34import traceback
35from collections import Counter
36from typing import List
37
38import config
39
40THREAD_VERSION = os.getenv('THREAD_VERSION')
41VIRTUAL_TIME = int(os.getenv('VIRTUAL_TIME', '1'))
42MAX_JOBS = int(os.getenv('MAX_JOBS', (multiprocessing.cpu_count() * 2 if VIRTUAL_TIME else 10)))
43
44_BACKBONE_TESTS_DIR = 'tests/scripts/thread-cert/backbone'
45
46_COLOR_PASS = '\033[0;32m'
47_COLOR_FAIL = '\033[0;31m'
48_COLOR_NONE = '\033[0m'
49
50logging.basicConfig(level=logging.DEBUG,
51                    format='File "%(pathname)s", line %(lineno)d, in %(funcName)s\n'
52                    '%(asctime)s - %(levelname)s - %(message)s')
53
54
55def bash(cmd: str, check=True, stdout=None):
56    subprocess.run(cmd, shell=True, check=check, stdout=stdout)
57
58
59def run_cert(job_id: int, port_offset: int, script: str):
60    try:
61        test_name = os.path.splitext(os.path.basename(script))[0] + '_' + str(job_id)
62        logfile = f'{test_name}.log'
63        env = os.environ.copy()
64        env['PORT_OFFSET'] = str(port_offset)
65        env['TEST_NAME'] = test_name
66
67        try:
68            print(f'Running {test_name}')
69            with open(logfile, 'wt') as output:
70                subprocess.check_call(["python3", script],
71                                      stdout=output,
72                                      stderr=output,
73                                      stdin=subprocess.DEVNULL,
74                                      env=env)
75        except subprocess.CalledProcessError:
76            bash(f'cat {logfile} 1>&2')
77            logging.error("Run test %s failed, please check the log file: %s", test_name, logfile)
78            raise
79
80    except Exception:
81        traceback.print_exc()
82        raise
83
84
85pool = multiprocessing.Pool(processes=MAX_JOBS)
86
87
88def cleanup_backbone_env():
89    logging.info("Cleaning up Backbone testing environment ...")
90    bash('pkill socat 2>/dev/null || true')
91    bash('pkill dumpcap 2>/dev/null || true')
92    bash(f'docker rm -f $(docker ps -a -q -f "name=otbr_") 2>/dev/null || true')
93    bash(f'docker network rm $(docker network ls -q -f "name=backbone") 2>/dev/null || true')
94
95
96def setup_backbone_env():
97    if THREAD_VERSION == '1.1':
98        raise RuntimeError('Backbone tests do not work with THREAD_VERSION=1.1')
99
100    if VIRTUAL_TIME:
101        raise RuntimeError('Backbone tests only work with VIRTUAL_TIME=0')
102
103    bash(f'docker image inspect {config.OTBR_DOCKER_IMAGE} >/dev/null')
104
105
106def parse_args():
107    import argparse
108    parser = argparse.ArgumentParser(description='Process some integers.')
109    parser.add_argument('--multiply', type=int, default=1, help='run each test for multiple times')
110    parser.add_argument("scripts", nargs='+', type=str, help='specify Backbone test scripts')
111
112    args = parser.parse_args()
113    logging.info("Max jobs: %d", MAX_JOBS)
114    logging.info("Multiply: %d", args.multiply)
115    logging.info("Test scripts: %d", len(args.scripts))
116    return args
117
118
119def check_has_backbone_tests(scripts):
120    for script in scripts:
121        relpath = os.path.relpath(script, _BACKBONE_TESTS_DIR)
122        if not relpath.startswith('..'):
123            return True
124
125    return False
126
127
128class PortOffsetPool:
129
130    def __init__(self, size: int):
131        self._size = size
132        self._pool = queue.Queue(maxsize=size)
133        for port_offset in range(0, size):
134            self.release(port_offset)
135
136    def allocate(self) -> int:
137        return self._pool.get()
138
139    def release(self, port_offset: int):
140        assert 0 <= port_offset < self._size, port_offset
141        self._pool.put_nowait(port_offset)
142
143
144def run_tests(scripts: List[str], multiply: int = 1):
145    script_fail_count = Counter()
146    script_succ_count = Counter()
147
148    # Run each script for multiple times
149    script_ids = [(script, i) for script in scripts for i in range(multiply)]
150    port_offset_pool = PortOffsetPool(MAX_JOBS)
151
152    def error_callback(port_offset, script, err):
153        port_offset_pool.release(port_offset)
154
155        script_fail_count[script] += 1
156        if script_succ_count[script] + script_fail_count[script] == multiply:
157            color = _COLOR_PASS if script_fail_count[script] == 0 else _COLOR_FAIL
158            print(f'{color}PASS {script_succ_count[script]} FAIL {script_fail_count[script]}{_COLOR_NONE} {script}')
159
160    def pass_callback(port_offset, script):
161        port_offset_pool.release(port_offset)
162
163        script_succ_count[script] += 1
164        if script_succ_count[script] + script_fail_count[script] == multiply:
165            color = _COLOR_PASS if script_fail_count[script] == 0 else _COLOR_FAIL
166            print(f'{color}PASS {script_succ_count[script]} FAIL {script_fail_count[script]}{_COLOR_NONE} {script}')
167
168    for script, i in script_ids:
169        port_offset = port_offset_pool.allocate()
170        pool.apply_async(
171            run_cert, [i, port_offset, script],
172            callback=lambda ret, port_offset=port_offset, script=script: pass_callback(port_offset, script),
173            error_callback=lambda err, port_offset=port_offset, script=script: error_callback(
174                port_offset, script, err))
175
176    pool.close()
177    pool.join()
178    return sum(script_fail_count.values())
179
180
181def main():
182    args = parse_args()
183
184    has_backbone_tests = check_has_backbone_tests(args.scripts)
185    logging.info('Has Backbone tests: %s', has_backbone_tests)
186
187    if has_backbone_tests:
188        cleanup_backbone_env()
189        setup_backbone_env()
190
191    try:
192        fail_count = run_tests(args.scripts, args.multiply)
193        exit(fail_count)
194    finally:
195        if has_backbone_tests:
196            cleanup_backbone_env()
197
198
199if __name__ == '__main__':
200    main()
201