third_party/pigweed/repo/pw_arduino_build/py/pw_arduino_build/unit_test_runner.py

   1 #!/usr/bin/env python3
   2 # Copyright 2020 The Pigweed Authors
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
   5 # use this file except in compliance with the License. You may obtain a copy of
   6 # the License at
   7 #
   8 #     https://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13 # License for the specific language governing permissions and limitations under
  14 # the License.
  15 """This script flashes and runs unit tests onto Arduino boards."""
  16
  17 import argparse
  18 import logging
  19 import os
  20 import platform
  21 import re
  22 import subprocess
  23 import sys
  24 import time
  25 from pathlib import Path
  26 from typing import List
  27
  28 import serial  # type: ignore
  29 import serial.tools.list_ports  # type: ignore
  30 import pw_arduino_build.log
  31 from pw_arduino_build import teensy_detector
  32 from pw_arduino_build.file_operations import decode_file_json
  33
  34 _LOG = logging.getLogger('unit_test_runner')
  35
  36 # Verification of test pass/failure depends on these strings. If the formatting
  37 # or output of the simple_printing_event_handler changes, this may need to be
  38 # updated.
  39 _TESTS_STARTING_STRING = b'[==========] Running all tests.'
  40 _TESTS_DONE_STRING = b'[==========] Done running all tests.'
  41 _TEST_FAILURE_STRING = b'[  FAILED  ]'
  42
  43 # How long to wait for the first byte of a test to be emitted. This is longer
  44 # than the user-configurable timeout as there's a delay while the device is
  45 # flashed.
  46 _FLASH_TIMEOUT = 5.0
  47
  48
  49 class TestingFailure(Exception):
  50     """A simple exception to be raised when a testing step fails."""
  51
  52
  53 class DeviceNotFound(Exception):
  54     """A simple exception to be raised when unable to connect to a device."""
  55
  56
  57 class ArduinoCoreNotSupported(Exception):
  58     """Exception raised when a given core does not support unit testing."""
  59
  60
  61 def valid_file_name(arg):
  62     file_path = Path(os.path.expandvars(arg)).absolute()
  63     if not file_path.is_file():
  64         raise argparse.ArgumentTypeError(f"'{arg}' does not exist.")
  65     return file_path
  66
  67
  68 def parse_args():
  69     """Parses command-line arguments."""
  70
  71     parser = argparse.ArgumentParser(description=__doc__)
  72     parser.add_argument('binary',
  73                         help='The target test binary to run',
  74                         type=valid_file_name)
  75     parser.add_argument('--port',
  76                         help='The name of the serial port to connect to when '
  77                         'running tests')
  78     parser.add_argument('--baud',
  79                         type=int,
  80                         default=115200,
  81                         help='Target baud rate to use for serial communication'
  82                         ' with target device')
  83     parser.add_argument('--test-timeout',
  84                         type=float,
  85                         default=5.0,
  86                         help='Maximum communication delay in seconds before a '
  87                         'test is considered unresponsive and aborted')
  88     parser.add_argument('--verbose',
  89                         '-v',
  90                         dest='verbose',
  91                         action='store_true',
  92                         help='Output additional logs as the script runs')
  93
  94     parser.add_argument('--flash-only',
  95                         action='store_true',
  96                         help="Don't check for test output after flashing.")
  97
  98     # arduino_builder arguments
  99     # TODO(tonymd): Get these args from __main__.py or elsewhere.
 100     parser.add_argument("-c",
 101                         "--config-file",
 102                         required=True,
 103                         help="Path to a config file.")
 104     parser.add_argument("--arduino-package-path",
 105                         help="Path to the arduino IDE install location.")
 106     parser.add_argument("--arduino-package-name",
 107                         help="Name of the Arduino board package to use.")
 108     parser.add_argument("--compiler-path-override",
 109                         help="Path to arm-none-eabi-gcc bin folder. "
 110                         "Default: Arduino core specified gcc")
 111     parser.add_argument("--board", help="Name of the Arduino board to use.")
 112     parser.add_argument("--upload-tool",
 113                         required=True,
 114                         help="Name of the Arduino upload tool to use.")
 115     parser.add_argument("--set-variable",
 116                         action="append",
 117                         metavar='some.variable=NEW_VALUE',
 118                         help="Override an Arduino recipe variable. May be "
 119                         "specified multiple times. For example: "
 120                         "--set-variable 'serial.port.label=/dev/ttyACM0' "
 121                         "--set-variable 'serial.port.protocol=Teensy'")
 122     return parser.parse_args()
 123
 124
 125 def log_subprocess_output(level, output):
 126     """Logs subprocess output line-by-line."""
 127
 128     lines = output.decode('utf-8', errors='replace').splitlines()
 129     for line in lines:
 130         _LOG.log(level, line)
 131
 132
 133 def read_serial(port, baud_rate, test_timeout) -> bytes:
 134     """Reads lines from a serial port until a line read times out.
 135
 136     Returns bytes object containing the read serial data.
 137     """
 138
 139     serial_data = bytearray()
 140     device = serial.Serial(baudrate=baud_rate,
 141                            port=port,
 142                            timeout=_FLASH_TIMEOUT)
 143     if not device.is_open:
 144         raise TestingFailure('Failed to open device')
 145
 146     # Flush input buffer and reset the device to begin the test.
 147     device.reset_input_buffer()
 148
 149     # Block and wait for the first byte.
 150     serial_data += device.read()
 151     if not serial_data:
 152         raise TestingFailure('Device not producing output')
 153
 154     device.timeout = test_timeout
 155
 156     # Read with a reasonable timeout until we stop getting characters.
 157     while True:
 158         bytes_read = device.readline()
 159         if not bytes_read:
 160             break
 161         serial_data += bytes_read
 162         if serial_data.rfind(_TESTS_DONE_STRING) != -1:
 163             # Set to much more aggressive timeout since the last one or two
 164             # lines should print out immediately. (one line if all fails or all
 165             # passes, two lines if mixed.)
 166             device.timeout = 0.01
 167
 168     # Remove carriage returns.
 169     serial_data = serial_data.replace(b'\r', b'')
 170
 171     # Try to trim captured results to only contain most recent test run.
 172     test_start_index = serial_data.rfind(_TESTS_STARTING_STRING)
 173     return serial_data if test_start_index == -1 else serial_data[
 174         test_start_index:]
 175
 176
 177 def wait_for_port(port):
 178     """Wait for the serial port to be available."""
 179     while port not in [sp.device for sp in serial.tools.list_ports.comports()]:
 180         time.sleep(1)
 181
 182
 183 def flash_device(test_runner_args, upload_tool):
 184     """Flash binary to a connected device using the provided configuration."""
 185
 186     # TODO(tonymd): Create a library function to call rather than launching
 187     # the arduino_builder script.
 188     flash_tool = 'arduino_builder'
 189     cmd = [flash_tool, "--quiet"] + test_runner_args + [
 190         "--run-objcopy", "--run-postbuilds", "--run-upload", upload_tool
 191     ]
 192     _LOG.info('Flashing firmware to device')
 193     _LOG.debug('Running: %s', " ".join(cmd))
 194
 195     env = os.environ.copy()
 196     process = subprocess.run(cmd,
 197                              stdout=subprocess.PIPE,
 198                              stderr=subprocess.STDOUT,
 199                              env=env)
 200     if process.returncode:
 201         log_subprocess_output(logging.ERROR, process.stdout)
 202         raise TestingFailure('Failed to flash target device')
 203
 204     log_subprocess_output(logging.DEBUG, process.stdout)
 205
 206     _LOG.debug('Successfully flashed firmware to device')
 207
 208
 209 def handle_test_results(test_output):
 210     """Parses test output to determine whether tests passed or failed."""
 211
 212     if test_output.find(_TESTS_STARTING_STRING) == -1:
 213         raise TestingFailure('Failed to find test start')
 214
 215     if test_output.rfind(_TESTS_DONE_STRING) == -1:
 216         log_subprocess_output(logging.INFO, test_output)
 217         raise TestingFailure('Tests did not complete')
 218
 219     if test_output.rfind(_TEST_FAILURE_STRING) != -1:
 220         log_subprocess_output(logging.INFO, test_output)
 221         raise TestingFailure('Test suite had one or more failures')
 222
 223     log_subprocess_output(logging.DEBUG, test_output)
 224
 225     _LOG.info('Test passed!')
 226
 227
 228 def run_device_test(binary, flash_only, port, baud, test_timeout, upload_tool,
 229                     arduino_package_path, test_runner_args) -> bool:
 230     """Flashes, runs, and checks an on-device test binary.
 231
 232     Returns true on test pass.
 233     """
 234     if test_runner_args is None:
 235         test_runner_args = []
 236
 237     if "teensy" not in arduino_package_path:
 238         raise ArduinoCoreNotSupported(arduino_package_path)
 239
 240     if port is None or "--set-variable" not in test_runner_args:
 241         _LOG.debug('Attempting to automatically detect dev board')
 242         boards = teensy_detector.detect_boards(arduino_package_path)
 243         if not boards:
 244             error = 'Could not find an attached device'
 245             _LOG.error(error)
 246             raise DeviceNotFound(error)
 247         test_runner_args += boards[0].test_runner_args()
 248         upload_tool = boards[0].arduino_upload_tool_name
 249         if port is None:
 250             port = boards[0].dev_name
 251
 252     # TODO(tonymd): Remove this when teensy_ports is working in teensy_detector
 253     if platform.system() == "Windows":
 254         # Delete the incorrect serial port.
 255         index_of_port = [
 256             i for i, l in enumerate(test_runner_args)
 257             if l.startswith('serial.port=')
 258         ]
 259         if index_of_port:
 260             # Delete the '--set-variable' arg
 261             del test_runner_args[index_of_port[0] - 1]
 262             # Delete the 'serial.port=*' arg
 263             del test_runner_args[index_of_port[0] - 1]
 264
 265     _LOG.debug('Launching test binary %s', binary)
 266     try:
 267         result: List[bytes] = []
 268         _LOG.info('Running test')
 269         # Warning: A race condition is possible here. This assumes the host is
 270         # able to connect to the port and that there isn't a test running on
 271         # this serial port.
 272         flash_device(test_runner_args, upload_tool)
 273         wait_for_port(port)
 274         if flash_only:
 275             return True
 276         result.append(read_serial(port, baud, test_timeout))
 277         if result:
 278             handle_test_results(result[0])
 279     except TestingFailure as err:
 280         _LOG.error(err)
 281         return False
 282
 283     return True
 284
 285
 286 def get_option(key, config_file_values, args, required=False):
 287     command_line_option = getattr(args, key, None)
 288     final_option = config_file_values.get(key, command_line_option)
 289     if required and command_line_option is None and final_option is None:
 290         # Print a similar error message to argparse
 291         executable = os.path.basename(sys.argv[0])
 292         option = "--" + key.replace("_", "-")
 293         print(f"{executable}: error: the following arguments are required: "
 294               f"{option}")
 295         sys.exit(1)
 296     return final_option
 297
 298
 299 def main():
 300     """Set up runner, and then flash/run device test."""
 301     args = parse_args()
 302
 303     json_file_options, unused_config_path = decode_file_json(args.config_file)
 304
 305     log_level = logging.DEBUG if args.verbose else logging.INFO
 306     pw_arduino_build.log.install(log_level)
 307
 308     # Construct arduino_builder flash arguments for a given .elf binary.
 309     arduino_package_path = get_option("arduino_package_path",
 310                                       json_file_options,
 311                                       args,
 312                                       required=True)
 313     # Arduino core args.
 314     arduino_builder_args = [
 315         "--arduino-package-path",
 316         arduino_package_path,
 317         "--arduino-package-name",
 318         get_option("arduino_package_name",
 319                    json_file_options,
 320                    args,
 321                    required=True),
 322     ]
 323
 324     # Use CIPD installed compilers.
 325     compiler_path_override = get_option("compiler_path_override",
 326                                         json_file_options, args)
 327     if compiler_path_override:
 328         arduino_builder_args += [
 329             "--compiler-path-override", compiler_path_override
 330         ]
 331
 332     # Run subcommand with board selection arg.
 333     arduino_builder_args += [
 334         "run", "--board",
 335         get_option("board", json_file_options, args, required=True)
 336     ]
 337
 338     # .elf file location args.
 339     binary = args.binary
 340     build_path = binary.parent.as_posix()
 341     arduino_builder_args += ["--build-path", build_path]
 342     build_project_name = binary.name
 343     # Remove '.elf' extension.
 344     match_result = re.match(r'(.*?)\.elf$', binary.name, re.IGNORECASE)
 345     if match_result:
 346         build_project_name = match_result[1]
 347         arduino_builder_args += ["--build-project-name", build_project_name]
 348
 349     # USB port is passed to arduino_builder_args via --set-variable args.
 350     if args.set_variable:
 351         for var in args.set_variable:
 352             arduino_builder_args += ["--set-variable", var]
 353
 354     if run_device_test(binary.as_posix(),
 355                        args.flash_only,
 356                        args.port,
 357                        args.baud,
 358                        args.test_timeout,
 359                        args.upload_tool,
 360                        arduino_package_path,
 361                        test_runner_args=arduino_builder_args):
 362         sys.exit(0)
 363     else:
 364         sys.exit(1)
 365
 366
 367 if __name__ == '__main__':
 368     main()