• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2020 The Pigweed Authors
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5# use this file except in compliance with the License. You may obtain a copy of
6# the License at
7#
8#     https://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations under
14# the License.
15"""This script flashes and runs unit tests onto Arduino boards."""
16
17import argparse
18import logging
19import os
20import platform
21import re
22import subprocess
23import sys
24import time
25from pathlib import Path
26from typing import List
27
28import serial
29import serial.tools.list_ports
30import pw_arduino_build.log
31from pw_arduino_build import teensy_detector
32from pw_arduino_build.file_operations import decode_file_json
33
34_LOG = logging.getLogger('unit_test_runner')
35
36# Verification of test pass/failure depends on these strings. If the formatting
37# or output of the simple_printing_event_handler changes, this may need to be
38# updated.
39_TESTS_STARTING_STRING = b'[==========] Running all tests.'
40_TESTS_DONE_STRING = b'[==========] Done running all tests.'
41_TEST_FAILURE_STRING = b'[  FAILED  ]'
42
43# How long to wait for the first byte of a test to be emitted. This is longer
44# than the user-configurable timeout as there's a delay while the device is
45# flashed.
46_FLASH_TIMEOUT = 5.0
47
48
49class TestingFailure(Exception):
50    """A simple exception to be raised when a testing step fails."""
51
52
53class DeviceNotFound(Exception):
54    """A simple exception to be raised when unable to connect to a device."""
55
56
57class ArduinoCoreNotSupported(Exception):
58    """Exception raised when a given core does not support unit testing."""
59
60
61def valid_file_name(arg):
62    file_path = Path(os.path.expandvars(arg)).absolute()
63    if not file_path.is_file():
64        raise argparse.ArgumentTypeError(f"'{arg}' does not exist.")
65    return file_path
66
67
68def parse_args():
69    """Parses command-line arguments."""
70
71    parser = argparse.ArgumentParser(description=__doc__)
72    parser.add_argument(
73        'binary', help='The target test binary to run', type=valid_file_name
74    )
75    parser.add_argument(
76        '--port',
77        help='The name of the serial port to connect to when ' 'running tests',
78    )
79    parser.add_argument(
80        '--baud',
81        type=int,
82        default=115200,
83        help='Target baud rate to use for serial communication'
84        ' with target device',
85    )
86    parser.add_argument(
87        '--test-timeout',
88        type=float,
89        default=5.0,
90        help='Maximum communication delay in seconds before a '
91        'test is considered unresponsive and aborted',
92    )
93    parser.add_argument(
94        '--verbose',
95        '-v',
96        dest='verbose',
97        action='store_true',
98        help='Output additional logs as the script runs',
99    )
100
101    parser.add_argument(
102        '--flash-only',
103        action='store_true',
104        help="Don't check for test output after flashing.",
105    )
106
107    # arduino_builder arguments
108    # TODO(tonymd): Get these args from __main__.py or elsewhere.
109    parser.add_argument(
110        "-c", "--config-file", required=True, help="Path to a config file."
111    )
112    parser.add_argument(
113        "--arduino-package-path",
114        help="Path to the arduino IDE install location.",
115    )
116    parser.add_argument(
117        "--arduino-package-name",
118        help="Name of the Arduino board package to use.",
119    )
120    parser.add_argument(
121        "--compiler-path-override",
122        help="Path to arm-none-eabi-gcc bin folder. "
123        "Default: Arduino core specified gcc",
124    )
125    parser.add_argument("--board", help="Name of the Arduino board to use.")
126    parser.add_argument(
127        "--upload-tool",
128        required=True,
129        help="Name of the Arduino upload tool to use.",
130    )
131    parser.add_argument(
132        "--set-variable",
133        action="append",
134        metavar='some.variable=NEW_VALUE',
135        help="Override an Arduino recipe variable. May be "
136        "specified multiple times. For example: "
137        "--set-variable 'serial.port.label=/dev/ttyACM0' "
138        "--set-variable 'serial.port.protocol=Teensy'",
139    )
140    return parser.parse_args()
141
142
143def log_subprocess_output(level, output):
144    """Logs subprocess output line-by-line."""
145
146    lines = output.decode('utf-8', errors='replace').splitlines()
147    for line in lines:
148        _LOG.log(level, line)
149
150
151def read_serial(port, baud_rate, test_timeout) -> bytes:
152    """Reads lines from a serial port until a line read times out.
153
154    Returns bytes object containing the read serial data.
155    """
156
157    serial_data = bytearray()
158    device = serial.Serial(
159        baudrate=baud_rate, port=port, timeout=_FLASH_TIMEOUT
160    )
161    if not device.is_open:
162        raise TestingFailure('Failed to open device')
163
164    # Flush input buffer and reset the device to begin the test.
165    device.reset_input_buffer()
166
167    # Block and wait for the first byte.
168    serial_data += device.read()
169    if not serial_data:
170        raise TestingFailure('Device not producing output')
171
172    device.timeout = test_timeout
173
174    # Read with a reasonable timeout until we stop getting characters.
175    while True:
176        bytes_read = device.readline()
177        if not bytes_read:
178            break
179        serial_data += bytes_read
180        if serial_data.rfind(_TESTS_DONE_STRING) != -1:
181            # Set to much more aggressive timeout since the last one or two
182            # lines should print out immediately. (one line if all fails or all
183            # passes, two lines if mixed.)
184            device.timeout = 0.01
185
186    # Remove carriage returns.
187    serial_data = serial_data.replace(b'\r', b'')
188
189    # Try to trim captured results to only contain most recent test run.
190    test_start_index = serial_data.rfind(_TESTS_STARTING_STRING)
191    return (
192        serial_data
193        if test_start_index == -1
194        else serial_data[test_start_index:]
195    )
196
197
198def wait_for_port(port):
199    """Wait for the serial port to be available."""
200    while port not in [sp.device for sp in serial.tools.list_ports.comports()]:
201        time.sleep(1)
202
203
204def flash_device(test_runner_args, upload_tool):
205    """Flash binary to a connected device using the provided configuration."""
206
207    # TODO(tonymd): Create a library function to call rather than launching
208    # the arduino_builder script.
209    flash_tool = 'arduino_builder'
210    cmd = (
211        [flash_tool, "--quiet"]
212        + test_runner_args
213        + ["--run-objcopy", "--run-postbuilds", "--run-upload", upload_tool]
214    )
215    _LOG.info('Flashing firmware to device')
216    _LOG.debug('Running: %s', " ".join(cmd))
217
218    env = os.environ.copy()
219    process = subprocess.run(
220        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env
221    )
222    if process.returncode:
223        log_subprocess_output(logging.ERROR, process.stdout)
224        raise TestingFailure('Failed to flash target device')
225
226    log_subprocess_output(logging.DEBUG, process.stdout)
227
228    _LOG.debug('Successfully flashed firmware to device')
229
230
231def handle_test_results(test_output):
232    """Parses test output to determine whether tests passed or failed."""
233
234    if test_output.find(_TESTS_STARTING_STRING) == -1:
235        raise TestingFailure('Failed to find test start')
236
237    if test_output.rfind(_TESTS_DONE_STRING) == -1:
238        log_subprocess_output(logging.INFO, test_output)
239        raise TestingFailure('Tests did not complete')
240
241    if test_output.rfind(_TEST_FAILURE_STRING) != -1:
242        log_subprocess_output(logging.INFO, test_output)
243        raise TestingFailure('Test suite had one or more failures')
244
245    log_subprocess_output(logging.DEBUG, test_output)
246
247    _LOG.info('Test passed!')
248
249
250def run_device_test(
251    binary,
252    flash_only,
253    port,
254    baud,
255    test_timeout,
256    upload_tool,
257    arduino_package_path,
258    test_runner_args,
259) -> bool:
260    """Flashes, runs, and checks an on-device test binary.
261
262    Returns true on test pass.
263    """
264    if test_runner_args is None:
265        test_runner_args = []
266
267    if "teensy" not in arduino_package_path:
268        raise ArduinoCoreNotSupported(arduino_package_path)
269
270    if port is None or "--set-variable" not in test_runner_args:
271        _LOG.debug('Attempting to automatically detect dev board')
272        boards = teensy_detector.detect_boards(arduino_package_path)
273        if not boards:
274            error = 'Could not find an attached device'
275            _LOG.error(error)
276            raise DeviceNotFound(error)
277        test_runner_args += boards[0].test_runner_args()
278        upload_tool = boards[0].arduino_upload_tool_name
279        if port is None:
280            port = boards[0].dev_name
281
282    # TODO(tonymd): Remove this when teensy_ports is working in teensy_detector
283    if platform.system() == "Windows":
284        # Delete the incorrect serial port.
285        index_of_port = [
286            i
287            for i, l in enumerate(test_runner_args)
288            if l.startswith('serial.port=')
289        ]
290        if index_of_port:
291            # Delete the '--set-variable' arg
292            del test_runner_args[index_of_port[0] - 1]
293            # Delete the 'serial.port=*' arg
294            del test_runner_args[index_of_port[0] - 1]
295
296    _LOG.debug('Launching test binary %s', binary)
297    try:
298        result: List[bytes] = []
299        _LOG.info('Running test')
300        # Warning: A race condition is possible here. This assumes the host is
301        # able to connect to the port and that there isn't a test running on
302        # this serial port.
303        flash_device(test_runner_args, upload_tool)
304        wait_for_port(port)
305        if flash_only:
306            return True
307        result.append(read_serial(port, baud, test_timeout))
308        if result:
309            handle_test_results(result[0])
310    except TestingFailure as err:
311        _LOG.error(err)
312        return False
313
314    return True
315
316
317def get_option(key, config_file_values, args, required=False):
318    command_line_option = getattr(args, key, None)
319    final_option = config_file_values.get(key, command_line_option)
320    if required and command_line_option is None and final_option is None:
321        # Print a similar error message to argparse
322        executable = os.path.basename(sys.argv[0])
323        option = "--" + key.replace("_", "-")
324        print(
325            f"{executable}: error: the following arguments are required: "
326            f"{option}"
327        )
328        sys.exit(1)
329    return final_option
330
331
332def main():
333    """Set up runner, and then flash/run device test."""
334    args = parse_args()
335
336    json_file_options, unused_config_path = decode_file_json(args.config_file)
337
338    log_level = logging.DEBUG if args.verbose else logging.INFO
339    pw_arduino_build.log.install(log_level)
340
341    # Construct arduino_builder flash arguments for a given .elf binary.
342    arduino_package_path = get_option(
343        "arduino_package_path", json_file_options, args, required=True
344    )
345    # Arduino core args.
346    arduino_builder_args = [
347        "--arduino-package-path",
348        arduino_package_path,
349        "--arduino-package-name",
350        get_option(
351            "arduino_package_name", json_file_options, args, required=True
352        ),
353    ]
354
355    # Use CIPD installed compilers.
356    compiler_path_override = get_option(
357        "compiler_path_override", json_file_options, args
358    )
359    if compiler_path_override:
360        arduino_builder_args += [
361            "--compiler-path-override",
362            compiler_path_override,
363        ]
364
365    # Run subcommand with board selection arg.
366    arduino_builder_args += [
367        "run",
368        "--board",
369        get_option("board", json_file_options, args, required=True),
370    ]
371
372    # .elf file location args.
373    binary = args.binary
374    build_path = binary.parent.as_posix()
375    arduino_builder_args += ["--build-path", build_path]
376    build_project_name = binary.name
377    # Remove '.elf' extension.
378    match_result = re.match(r'(.*?)\.elf$', binary.name, re.IGNORECASE)
379    if match_result:
380        build_project_name = match_result[1]
381        arduino_builder_args += ["--build-project-name", build_project_name]
382
383    # USB port is passed to arduino_builder_args via --set-variable args.
384    if args.set_variable:
385        for var in args.set_variable:
386            arduino_builder_args += ["--set-variable", var]
387
388    if run_device_test(
389        binary.as_posix(),
390        args.flash_only,
391        args.port,
392        args.baud,
393        args.test_timeout,
394        args.upload_tool,
395        arduino_package_path,
396        test_runner_args=arduino_builder_args,
397    ):
398        sys.exit(0)
399    else:
400        sys.exit(1)
401
402
403if __name__ == '__main__':
404    main()
405