third_party/pigweed/repo/pw_arduino_build/py/pw_arduino_build/builder.py

   1 #!/usr/bin/env python3
   2 # Copyright 2020 The Pigweed Authors
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
   5 # use this file except in compliance with the License. You may obtain a copy of
   6 # the License at
   7 #
   8 #     https://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13 # License for the specific language governing permissions and limitations under
  14 # the License.
  15 """Extracts build information from Arduino cores."""
  16
  17 import glob
  18 import logging
  19 import os
  20 import platform
  21 import pprint
  22 import re
  23 import sys
  24 import time
  25 from collections import OrderedDict
  26 from pathlib import Path
  27 from typing import List
  28
  29 from pw_arduino_build import file_operations
  30
  31 _LOG = logging.getLogger(__name__)
  32
  33 _pretty_print = pprint.PrettyPrinter(indent=1, width=120).pprint
  34 _pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat
  35
  36
  37 def arduino_runtime_os_string():
  38     arduno_platform = {
  39         "Linux": "linux",
  40         "Windows": "windows",
  41         "Darwin": "macosx"
  42     }
  43     return arduno_platform[platform.system()]
  44
  45
  46 class ArduinoBuilder:
  47     """Used to interpret arduino boards.txt and platform.txt files."""
  48     # pylint: disable=too-many-instance-attributes,too-many-public-methods
  49
  50     BOARD_MENU_REGEX = re.compile(
  51         r"^(?P<name>menu\.[^#=]+)=(?P<description>.*)$", re.MULTILINE)
  52
  53     BOARD_NAME_REGEX = re.compile(
  54         r"^(?P<name>[^\s#\.]+)\.name=(?P<description>.*)$", re.MULTILINE)
  55
  56     VARIABLE_REGEX = re.compile(r"^(?P<name>[^\s#=]+)=(?P<value>.*)$",
  57                                 re.MULTILINE)
  58
  59     MENU_OPTION_REGEX = re.compile(
  60         r"^menu\."  # starts with "menu"
  61         r"(?P<menu_option_name>[^.]+)\."  # first token after .
  62         r"(?P<menu_option_value>[^.]+)$")  # second (final) token after .
  63
  64     TOOL_NAME_REGEX = re.compile(
  65         r"^tools\."  # starts with "tools"
  66         r"(?P<tool_name>[^.]+)\.")  # first token after .
  67
  68     INTERPOLATED_VARIABLE_REGEX = re.compile(r"{[^}]+}", re.MULTILINE)
  69
  70     OBJCOPY_STEP_NAME_REGEX = re.compile(r"^recipe.objcopy.([^.]+).pattern$")
  71
  72     def __init__(self,
  73                  arduino_path,
  74                  package_name,
  75                  build_path=None,
  76                  project_path=None,
  77                  project_source_path=None,
  78                  library_path=None,
  79                  library_names=None,
  80                  build_project_name=None,
  81                  compiler_path_override=False):
  82         self.arduino_path = arduino_path
  83         self.arduino_package_name = package_name
  84         self.selected_board = None
  85         self.build_path = build_path
  86         self.project_path = project_path
  87         self.project_source_path = project_source_path
  88         self.build_project_name = build_project_name
  89         self.compiler_path_override = compiler_path_override
  90         self.variant_includes = ""
  91         self.build_variant_path = False
  92         self.library_names = library_names
  93         self.library_path = library_path
  94
  95         self.compiler_path_override_binaries = []
  96         if self.compiler_path_override:
  97             self.compiler_path_override_binaries = file_operations.find_files(
  98                 self.compiler_path_override, "*")
  99
 100         # Container dicts for boards.txt and platform.txt file data.
 101         self.board = OrderedDict()
 102         self.platform = OrderedDict()
 103         self.menu_options = OrderedDict({
 104             "global_options": {},
 105             "default_board_values": {},
 106             "selected": {}
 107         })
 108         self.tools_variables = {}
 109
 110         # Set and check for valid hardware folder.
 111         self.hardware_path = os.path.join(self.arduino_path, "hardware")
 112
 113         if not os.path.exists(self.hardware_path):
 114             raise FileNotFoundError(
 115                 "Arduino package path '{}' does not exist.".format(
 116                     self.arduino_path))
 117
 118         # Set and check for valid package name
 119         self.package_path = os.path.join(self.arduino_path, "hardware",
 120                                          package_name)
 121         # {build.arch} is the first folder name of the package (upcased)
 122         self.build_arch = os.path.split(package_name)[0].upper()
 123
 124         if not os.path.exists(self.package_path):
 125             _LOG.error("Error: Arduino package name '%s' does not exist.",
 126                        package_name)
 127             _LOG.error("Did you mean:\n")
 128             # TODO(tonymd): On Windows concatenating "/" may not work
 129             possible_alternatives = [
 130                 d.replace(self.hardware_path + os.sep, "", 1)
 131                 for d in glob.glob(self.hardware_path + "/*/*")
 132             ]
 133             _LOG.error("\n".join(possible_alternatives))
 134             sys.exit(1)
 135
 136         # Populate library paths.
 137         if not library_path:
 138             self.library_path = []
 139         # Append core libraries directory.
 140         core_lib_path = Path(self.package_path) / "libraries"
 141         if core_lib_path.is_dir():
 142             self.library_path.append(Path(self.package_path) / "libraries")
 143         if library_path:
 144             self.library_path = [
 145                 os.path.realpath(os.path.expanduser(
 146                     os.path.expandvars(l_path))) for l_path in library_path
 147             ]
 148
 149         # Grab all folder names in the cores directory. These are typically
 150         # sub-core source files.
 151         self.sub_core_folders = os.listdir(
 152             os.path.join(self.package_path, "cores"))
 153
 154         self._find_tools_variables()
 155
 156         self.boards_txt = os.path.join(self.package_path, "boards.txt")
 157         self.platform_txt = os.path.join(self.package_path, "platform.txt")
 158
 159     def select_board(self, board_name, menu_option_overrides=False):
 160         self.selected_board = board_name
 161
 162         # Load default menu options for a selected board.
 163         if not self.selected_board in self.board.keys():
 164             _LOG.error("Error board: '%s' not supported.", self.selected_board)
 165             # TODO(tonymd): Print supported boards here
 166             sys.exit(1)
 167
 168         # Override default menu options if any are specified.
 169         if menu_option_overrides:
 170             for moption in menu_option_overrides:
 171                 if not self.set_menu_option(moption):
 172                     # TODO(tonymd): Print supported menu options here
 173                     sys.exit(1)
 174
 175         self._copy_default_menu_options_to_build_variables()
 176         self._apply_recipe_overrides()
 177         self._substitute_variables()
 178
 179     def set_variables(self, variable_list: List[str]):
 180         # Convert the string list containing 'name=value' items into a dict
 181         variable_source = {}
 182         for var in variable_list:
 183             var_name, value = var.split("=")
 184             variable_source[var_name] = value
 185
 186         # Replace variables in platform
 187         for var, value in self.platform.items():
 188             self.platform[var] = self._replace_variables(
 189                 value, variable_source)
 190
 191     def _apply_recipe_overrides(self):
 192         # Override link recipes with per-core exceptions
 193         # Teensyduino cores
 194         if self.build_arch == "TEENSY":
 195             # Change {build.path}/{archive_file}
 196             # To {archive_file_path} (which should contain the core.a file)
 197             new_link_line = self.platform["recipe.c.combine.pattern"].replace(
 198                 "{object_files} \"{build.path}/{archive_file}\"",
 199                 "{object_files} {archive_file_path}", 1)
 200             # Add the teensy provided toolchain lib folder for link access to
 201             # libarm_cortexM*_math.a
 202             new_link_line = new_link_line.replace(
 203                 "\"-L{build.path}\"",
 204                 "\"-L{build.path}\" -L{compiler.path}/arm/arm-none-eabi/lib",
 205                 1)
 206             self.platform["recipe.c.combine.pattern"] = new_link_line
 207             # Remove the pre-compiled header include
 208             self.platform["recipe.cpp.o.pattern"] = self.platform[
 209                 "recipe.cpp.o.pattern"].replace("\"-I{build.path}/pch\"", "",
 210                                                 1)
 211
 212         # Adafruit-samd core
 213         # TODO(tonymd): This build_arch may clash with Arduino-SAMD core
 214         elif self.build_arch == "SAMD":
 215             new_link_line = self.platform["recipe.c.combine.pattern"].replace(
 216                 "\"{build.path}/{archive_file}\" -Wl,--end-group",
 217                 "{archive_file_path} -Wl,--end-group", 1)
 218             self.platform["recipe.c.combine.pattern"] = new_link_line
 219
 220         # STM32L4 Core:
 221         # https://github.com/GrumpyOldPizza/arduino-STM32L4
 222         elif self.build_arch == "STM32L4":
 223             # TODO(tonymd): {build.path}/{archive_file} for the link step always
 224             # seems to be core.a (except STM32 core)
 225             line_to_delete = "-Wl,--start-group \"{build.path}/{archive_file}\""
 226             new_link_line = self.platform["recipe.c.combine.pattern"].replace(
 227                 line_to_delete, "-Wl,--start-group {archive_file_path}", 1)
 228             self.platform["recipe.c.combine.pattern"] = new_link_line
 229
 230         # stm32duino core
 231         elif self.build_arch == "STM32":
 232             # Must link in SrcWrapper for all projects.
 233             if not self.library_names:
 234                 self.library_names = []
 235             self.library_names.append("SrcWrapper")
 236
 237     def _copy_default_menu_options_to_build_variables(self):
 238         # Clear existing options
 239         self.menu_options["selected"] = {}
 240         # Set default menu options for selected board
 241         for menu_key, menu_dict in self.menu_options["default_board_values"][
 242                 self.selected_board].items():
 243             for name, var in self.board[self.selected_board].items():
 244                 starting_key = "{}.{}.".format(menu_key, menu_dict["name"])
 245                 if name.startswith(starting_key):
 246                     new_var_name = name.replace(starting_key, "", 1)
 247                     self.menu_options["selected"][new_var_name] = var
 248
 249     def set_menu_option(self, moption):
 250         if moption not in self.board[self.selected_board]:
 251             _LOG.error("Error: '%s' is not a valid menu option.", moption)
 252             return False
 253
 254         # Override default menu option with new value.
 255         menu_match_result = self.MENU_OPTION_REGEX.match(moption)
 256         if menu_match_result:
 257             menu_match = menu_match_result.groupdict()
 258             menu_value = menu_match["menu_option_value"]
 259             menu_key = "menu.{}".format(menu_match["menu_option_name"])
 260             self.menu_options["default_board_values"][
 261                 self.selected_board][menu_key]["name"] = menu_value
 262
 263         # Update build variables
 264         self._copy_default_menu_options_to_build_variables()
 265         return True
 266
 267     def _set_global_arduino_variables(self):
 268         """Set some global variables defined by the Arduino-IDE.
 269
 270         See Docs:
 271         https://arduino.github.io/arduino-cli/platform-specification/#global-predefined-properties
 272         """
 273
 274         # TODO(tonymd): Make sure these variables are replaced in recipe lines
 275         # even if they are None: build_path, project_path, project_source_path,
 276         # build_project_name
 277         for current_board_name in self.board.keys():
 278             if self.build_path:
 279                 self.board[current_board_name]["build.path"] = self.build_path
 280             if self.build_project_name:
 281                 self.board[current_board_name][
 282                     "build.project_name"] = self.build_project_name
 283                 # {archive_file} is the final *.elf
 284                 archive_file = "{}.elf".format(self.build_project_name)
 285                 self.board[current_board_name]["archive_file"] = archive_file
 286                 # {archive_file_path} is the final core.a archive
 287                 if self.build_path:
 288                     self.board[current_board_name][
 289                         "archive_file_path"] = os.path.join(
 290                             self.build_path, "core.a")
 291             if self.project_source_path:
 292                 self.board[current_board_name][
 293                     "build.source.path"] = self.project_source_path
 294
 295             self.board[current_board_name]["extra.time.local"] = str(
 296                 int(time.time()))
 297             self.board[current_board_name]["runtime.ide.version"] = "10812"
 298             self.board[current_board_name][
 299                 "runtime.hardware.path"] = self.hardware_path
 300
 301             # Copy {runtime.tools.TOOL_NAME.path} vars
 302             self._set_tools_variables(self.board[current_board_name])
 303
 304             self.board[current_board_name][
 305                 "runtime.platform.path"] = self.package_path
 306             if self.platform["name"] == "Teensyduino":
 307                 # Teensyduino is installed into the arduino IDE folder
 308                 # rather than ~/.arduino15/packages/
 309                 self.board[current_board_name][
 310                     "runtime.hardware.path"] = os.path.join(
 311                         self.hardware_path, "teensy")
 312
 313             self.board[current_board_name]["build.system.path"] = os.path.join(
 314                 self.package_path, "system")
 315
 316             # Set the {build.core.path} variable that pointing to a sub-core
 317             # folder. For Teensys this is:
 318             # 'teensy/hardware/teensy/avr/cores/teensy{3,4}'. For other cores
 319             # it's typically just the 'arduino' folder. For example:
 320             # 'arduino-samd/hardware/samd/1.8.8/cores/arduino'
 321             core_path = Path(self.package_path) / "cores"
 322             core_path /= self.board[current_board_name].get(
 323                 "build.core", self.sub_core_folders[0])
 324             self.board[current_board_name][
 325                 "build.core.path"] = core_path.as_posix()
 326
 327             self.board[current_board_name]["build.arch"] = self.build_arch
 328
 329             for name, var in self.board[current_board_name].items():
 330                 self.board[current_board_name][name] = var.replace(
 331                     "{build.core.path}", core_path.as_posix())
 332
 333     def load_board_definitions(self):
 334         """Loads Arduino boards.txt and platform.txt files into dictionaries.
 335
 336         Populates the following dictionaries:
 337             self.menu_options
 338             self.boards
 339             self.platform
 340         """
 341         # Load platform.txt
 342         with open(self.platform_txt, "r") as pfile:
 343             platform_file = pfile.read()
 344             platform_var_matches = self.VARIABLE_REGEX.finditer(platform_file)
 345             for p_match in [m.groupdict() for m in platform_var_matches]:
 346                 self.platform[p_match["name"]] = p_match["value"]
 347
 348         # Load boards.txt
 349         with open(self.boards_txt, "r") as bfile:
 350             board_file = bfile.read()
 351             # Get all top-level menu options, e.g. menu.usb=USB Type
 352             board_menu_matches = self.BOARD_MENU_REGEX.finditer(board_file)
 353             for menuitem in [m.groupdict() for m in board_menu_matches]:
 354                 self.menu_options["global_options"][menuitem["name"]] = {
 355                     "description": menuitem["description"]
 356                 }
 357
 358             # Get all board names, e.g. teensy40.name=Teensy 4.0
 359             board_name_matches = self.BOARD_NAME_REGEX.finditer(board_file)
 360             for b_match in [m.groupdict() for m in board_name_matches]:
 361                 self.board[b_match["name"]] = OrderedDict()
 362                 self.menu_options["default_board_values"][
 363                     b_match["name"]] = OrderedDict()
 364
 365             # Get all board variables, e.g. teensy40.*
 366             for current_board_name in self.board.keys():
 367                 board_line_matches = re.finditer(
 368                     fr"^\s*{current_board_name}\."
 369                     fr"(?P<key>[^#=]+)=(?P<value>.*)$", board_file,
 370                     re.MULTILINE)
 371                 for b_match in [m.groupdict() for m in board_line_matches]:
 372                     # Check if this line is a menu option
 373                     # (e.g. 'menu.usb.serial') and save as default if it's the
 374                     # first one seen.
 375                     ArduinoBuilder.save_default_menu_option(
 376                         current_board_name, b_match["key"], b_match["value"],
 377                         self.menu_options)
 378                     self.board[current_board_name][
 379                         b_match["key"]] = b_match["value"].strip()
 380
 381             self._set_global_arduino_variables()
 382
 383     @staticmethod
 384     def save_default_menu_option(current_board_name, key, value, menu_options):
 385         """Save a given menu option as the default.
 386
 387         Saves the key and value into menu_options["default_board_values"]
 388         if it doesn't already exist. Assumes menu options are added in the order
 389         specified in boards.txt. The first value for a menu key is the default.
 390         """
 391         # Check if key is a menu option
 392         # e.g. menu.usb.serial
 393         #      menu.usb.serial.build.usbtype
 394         menu_match_result = re.match(
 395             r'^menu\.'  # starts with "menu"
 396             r'(?P<menu_option_name>[^.]+)\.'  # first token after .
 397             r'(?P<menu_option_value>[^.]+)'  # second token after .
 398             r'(\.(?P<rest>.+))?',  # optionally any trailing tokens after a .
 399             key)
 400         if menu_match_result:
 401             menu_match = menu_match_result.groupdict()
 402             current_menu_key = "menu.{}".format(menu_match["menu_option_name"])
 403             # If this is the first menu option seen for current_board_name, save
 404             # as the default.
 405             if current_menu_key not in menu_options["default_board_values"][
 406                     current_board_name]:
 407                 menu_options["default_board_values"][current_board_name][
 408                     current_menu_key] = {
 409                         "name": menu_match["menu_option_value"],
 410                         "description": value
 411                     }
 412
 413     def _replace_variables(self, line, variable_lookup_source):
 414         """Replace {variables} from loaded boards.txt or platform.txt.
 415
 416         Replace interpolated variables surrounded by curly braces in line with
 417         definitions from variable_lookup_source.
 418         """
 419         new_line = line
 420         for current_var_match in self.INTERPOLATED_VARIABLE_REGEX.findall(
 421                 line):
 422             # {build.flags.c} --> build.flags.c
 423             current_var = current_var_match.strip("{}")
 424
 425             # check for matches in board definition
 426             if current_var in variable_lookup_source:
 427                 replacement = variable_lookup_source.get(current_var, "")
 428                 new_line = new_line.replace(current_var_match, replacement)
 429         return new_line
 430
 431     def _find_tools_variables(self):
 432         # Gather tool directories in order of increasing precedence
 433         runtime_tool_paths = []
 434
 435         # Check for tools installed in ~/.arduino15/packages/arduino/tools/
 436         # TODO(tonymd): Is this Mac & Linux specific?
 437         runtime_tool_paths += glob.glob(
 438             os.path.join(
 439                 os.path.realpath(os.path.expanduser(os.path.expandvars("~"))),
 440                 ".arduino15", "packages", "arduino", "tools", "*"))
 441
 442         # <ARDUINO_PATH>/tools/<OS_STRING>/<TOOL_NAMES>
 443         runtime_tool_paths += glob.glob(
 444             os.path.join(self.arduino_path, "tools",
 445                          arduino_runtime_os_string(), "*"))
 446         # <ARDUINO_PATH>/tools/<TOOL_NAMES>
 447         # This will grab linux/windows/macosx/share as <TOOL_NAMES>.
 448         runtime_tool_paths += glob.glob(
 449             os.path.join(self.arduino_path, "tools", "*"))
 450
 451         # Process package tools after arduino tools.
 452         # They should overwrite vars & take precedence.
 453
 454         # <PACKAGE_PATH>/tools/<OS_STRING>/<TOOL_NAMES>
 455         runtime_tool_paths += glob.glob(
 456             os.path.join(self.package_path, "tools",
 457                          arduino_runtime_os_string(), "*"))
 458         # <PACKAGE_PATH>/tools/<TOOL_NAMES>
 459         # This will grab linux/windows/macosx/share as <TOOL_NAMES>.
 460         runtime_tool_paths += glob.glob(
 461             os.path.join(self.package_path, "tools", "*"))
 462
 463         for path in runtime_tool_paths:
 464             # Make sure TOOL_NAME is not an OS string
 465             if not (path.endswith("linux") or path.endswith("windows")
 466                     or path.endswith("macosx") or path.endswith("share")):
 467                 # TODO(tonymd): Check if a file & do nothing?
 468
 469                 # Check if it's a directory with subdir(s) as a version string
 470                 #   create all 'runtime.tools.{tool_folder}-{version.path}'
 471                 #     (for each version)
 472                 #   create 'runtime.tools.{tool_folder}.path'
 473                 #     (with latest version)
 474                 if os.path.isdir(path):
 475                     # Grab the tool name (folder) by itself.
 476                     tool_folder = os.path.basename(path)
 477                     # Sort so that [-1] is the latest version.
 478                     version_paths = sorted(glob.glob(os.path.join(path, "*")))
 479                     # Check if all sub folders start with a version string.
 480                     if len(version_paths) == sum(
 481                             bool(re.match(r"^[0-9.]+", os.path.basename(vp)))
 482                             for vp in version_paths):
 483                         for version_path in version_paths:
 484                             version_string = os.path.basename(version_path)
 485                             var_name = "runtime.tools.{}-{}.path".format(
 486                                 tool_folder, version_string)
 487                             self.tools_variables[var_name] = os.path.join(
 488                                 path, version_string)
 489                         var_name = "runtime.tools.{}.path".format(tool_folder)
 490                         self.tools_variables[var_name] = os.path.join(
 491                             path, os.path.basename(version_paths[-1]))
 492                     # Else set toolpath to path.
 493                     else:
 494                         var_name = "runtime.tools.{}.path".format(tool_folder)
 495                         self.tools_variables[var_name] = path
 496
 497         _LOG.debug("TOOL VARIABLES: %s", _pretty_format(self.tools_variables))
 498
 499     # Copy self.tools_variables into destination
 500     def _set_tools_variables(self, destination):
 501         for key, value in self.tools_variables.items():
 502             destination[key] = value
 503
 504     def _substitute_variables(self):
 505         """Perform variable substitution in board and platform metadata."""
 506
 507         # menu -> board
 508         # Copy selected menu variables into board definiton
 509         for name, value in self.menu_options["selected"].items():
 510             self.board[self.selected_board][name] = value
 511
 512         # board -> board
 513         # Replace any {vars} in the selected board with values defined within
 514         # (and from copied in menu options).
 515         for var, value in self.board[self.selected_board].items():
 516             self.board[self.selected_board][var] = self._replace_variables(
 517                 value, self.board[self.selected_board])
 518
 519         # Check for build.variant variable
 520         # This will be set in selected board after menu options substitution
 521         build_variant = self.board[self.selected_board].get(
 522             "build.variant", None)
 523         if build_variant:
 524             # Set build.variant.path
 525             bvp = os.path.join(self.package_path, "variants", build_variant)
 526             self.build_variant_path = bvp
 527             self.board[self.selected_board]["build.variant.path"] = bvp
 528             # Add the variant folder as an include directory
 529             # (used in stm32l4 core)
 530             self.variant_includes = "-I{}".format(bvp)
 531
 532         _LOG.debug("PLATFORM INITIAL: %s", _pretty_format(self.platform))
 533
 534         # board -> platform
 535         # Replace {vars} in platform from the selected board definition
 536         for var, value in self.platform.items():
 537             self.platform[var] = self._replace_variables(
 538                 value, self.board[self.selected_board])
 539
 540         # platform -> platform
 541         # Replace any remaining {vars} in platform from platform
 542         for var, value in self.platform.items():
 543             self.platform[var] = self._replace_variables(value, self.platform)
 544
 545         # Repeat platform -> platform for any lingering variables
 546         # Example: {build.opt.name} in STM32 core
 547         for var, value in self.platform.items():
 548             self.platform[var] = self._replace_variables(value, self.platform)
 549
 550         _LOG.debug("MENU_OPTIONS: %s", _pretty_format(self.menu_options))
 551         _LOG.debug("SELECTED_BOARD: %s",
 552                    _pretty_format(self.board[self.selected_board]))
 553         _LOG.debug("PLATFORM: %s", _pretty_format(self.platform))
 554
 555     def selected_board_spec(self):
 556         return self.board[self.selected_board]
 557
 558     def get_menu_options(self):
 559         all_options = []
 560         max_string_length = [0, 0]
 561
 562         for key_name, description in self.board[self.selected_board].items():
 563             menu_match_result = self.MENU_OPTION_REGEX.match(key_name)
 564             if menu_match_result:
 565                 menu_match = menu_match_result.groupdict()
 566                 name = "menu.{}.{}".format(menu_match["menu_option_name"],
 567                                            menu_match["menu_option_value"])
 568                 if len(name) > max_string_length[0]:
 569                     max_string_length[0] = len(name)
 570                 if len(description) > max_string_length[1]:
 571                     max_string_length[1] = len(description)
 572                 all_options.append((name, description))
 573
 574         return all_options, max_string_length
 575
 576     def get_default_menu_options(self):
 577         default_options = []
 578         max_string_length = [0, 0]
 579
 580         for key_name, value in self.menu_options["default_board_values"][
 581                 self.selected_board].items():
 582             full_key = key_name + "." + value["name"]
 583             if len(full_key) > max_string_length[0]:
 584                 max_string_length[0] = len(full_key)
 585             if len(value["description"]) > max_string_length[1]:
 586                 max_string_length[1] = len(value["description"])
 587             default_options.append((full_key, value["description"]))
 588
 589         return default_options, max_string_length
 590
 591     @staticmethod
 592     def split_binary_from_arguments(compile_line):
 593         compile_binary = None
 594         rest_of_line = compile_line
 595
 596         compile_binary_match = re.search(r'^("[^"]+") ', compile_line)
 597         if compile_binary_match:
 598             compile_binary = compile_binary_match[1]
 599             rest_of_line = compile_line.replace(compile_binary_match[0], "", 1)
 600
 601         return compile_binary, rest_of_line
 602
 603     def _strip_includes_source_file_object_file_vars(self, compile_line):
 604         line = compile_line
 605         if self.variant_includes:
 606             line = compile_line.replace(
 607                 "{includes} \"{source_file}\" -o \"{object_file}\"",
 608                 self.variant_includes, 1)
 609         else:
 610             line = compile_line.replace(
 611                 "{includes} \"{source_file}\" -o \"{object_file}\"", "", 1)
 612         return line
 613
 614     def _get_tool_name(self, line):
 615         tool_match_result = self.TOOL_NAME_REGEX.match(line)
 616         if tool_match_result:
 617             return tool_match_result[1]
 618         return False
 619
 620     def get_upload_tool_names(self):
 621         return [
 622             self._get_tool_name(t) for t in self.platform.keys()
 623             if self.TOOL_NAME_REGEX.match(t) and 'upload.pattern' in t
 624         ]
 625
 626     # TODO(tonymd): Use these getters in _replace_variables() or
 627     # _substitute_variables()
 628
 629     def _get_platform_variable(self, variable):
 630         # TODO(tonymd): Check for '.macos' '.linux' '.windows' in variable name,
 631         # compare with platform.system() and return that instead.
 632         return self.platform.get(variable, False)
 633
 634     def _get_platform_variable_with_substitutions(self, variable, namespace):
 635         line = self.platform.get(variable, False)
 636         # Get all unique variables used in this line in line.
 637         unique_vars = sorted(
 638             set(self.INTERPOLATED_VARIABLE_REGEX.findall(line)))
 639         # Search for each unique_vars in namespace and global.
 640         for var in unique_vars:
 641             v_raw_name = var.strip("{}")
 642
 643             # Check for namespace.variable
 644             #   eg: 'tools.stm32CubeProg.cmd'
 645             possible_var_name = "{}.{}".format(namespace, v_raw_name)
 646             result = self._get_platform_variable(possible_var_name)
 647             # Check for os overriden variable
 648             #   eg:
 649             #     ('tools.stm32CubeProg.cmd', 'stm32CubeProg.sh'),
 650             #     ('tools.stm32CubeProg.cmd.windows', 'stm32CubeProg.bat'),
 651             possible_var_name = "{}.{}.{}".format(namespace, v_raw_name,
 652                                                   arduino_runtime_os_string())
 653             os_override_result = self._get_platform_variable(possible_var_name)
 654
 655             if os_override_result:
 656                 line = line.replace(var, os_override_result)
 657             elif result:
 658                 line = line.replace(var, result)
 659             # Check for variable at top level?
 660             # elif self._get_platform_variable(v_raw_name):
 661             #     line = line.replace(self._get_platform_variable(v_raw_name),
 662             #                         result)
 663         return line
 664
 665     def get_upload_line(self, tool_name, serial_port=False):
 666         # TODO(tonymd): Error if tool_name does not exist
 667         tool_namespace = "tools.{}".format(tool_name)
 668         pattern = "tools.{}.upload.pattern".format(tool_name)
 669
 670         if not self._get_platform_variable(pattern):
 671             _LOG.error("Error: upload tool '%s' does not exist.", tool_name)
 672             tools = self.get_upload_tool_names()
 673             _LOG.error("Valid tools: %s", ", ".join(tools))
 674             return sys.exit(1)
 675
 676         line = self._get_platform_variable_with_substitutions(
 677             pattern, tool_namespace)
 678
 679         # TODO(tonymd): Teensy specific tool overrides.
 680         if tool_name == "teensyloader":
 681             # Remove un-necessary lines
 682             # {serial.port.label} and {serial.port.protocol} are returned by
 683             # the teensy_ports binary.
 684             line = line.replace("\"-portlabel={serial.port.label}\"", "", 1)
 685             line = line.replace("\"-portprotocol={serial.port.protocol}\"", "",
 686                                 1)
 687
 688             if serial_port == "UNKNOWN" or not serial_port:
 689                 line = line.replace('"-port={serial.port}"', "", 1)
 690             else:
 691                 line = line.replace("{serial.port}", serial_port, 1)
 692
 693         return line
 694
 695     def _get_binary_path(self, variable_pattern):
 696         compile_line = self.replace_compile_binary_with_override_path(
 697             self._get_platform_variable(variable_pattern))
 698         compile_binary, _ = ArduinoBuilder.split_binary_from_arguments(
 699             compile_line)
 700         return compile_binary
 701
 702     def get_cc_binary(self):
 703         return self._get_binary_path("recipe.c.o.pattern")
 704
 705     def get_cxx_binary(self):
 706         return self._get_binary_path("recipe.cpp.o.pattern")
 707
 708     def get_objcopy_binary(self):
 709         objcopy_step_name = self.get_objcopy_step_names()[0]
 710         objcopy_binary = self._get_binary_path(objcopy_step_name)
 711         return objcopy_binary
 712
 713     def get_ar_binary(self):
 714         return self._get_binary_path("recipe.ar.pattern")
 715
 716     def get_size_binary(self):
 717         return self._get_binary_path("recipe.size.pattern")
 718
 719     def replace_command_args_with_compiler_override_path(self, compile_line):
 720         if not self.compiler_path_override:
 721             return compile_line
 722         replacement_line = compile_line
 723         replacement_line_args = compile_line.split()
 724         for arg in replacement_line_args:
 725             compile_binary_basename = os.path.basename(arg.strip("\""))
 726             if compile_binary_basename in self.compiler_path_override_binaries:
 727                 new_compiler = os.path.join(self.compiler_path_override,
 728                                             compile_binary_basename)
 729                 replacement_line = replacement_line.replace(
 730                     arg, new_compiler, 1)
 731         return replacement_line
 732
 733     def replace_compile_binary_with_override_path(self, compile_line):
 734         replacement_compile_line = compile_line
 735
 736         # Change the compiler path if there's an override path set
 737         if self.compiler_path_override:
 738             compile_binary, line = ArduinoBuilder.split_binary_from_arguments(
 739                 compile_line)
 740             compile_binary_basename = os.path.basename(
 741                 compile_binary.strip("\""))
 742             new_compiler = os.path.join(self.compiler_path_override,
 743                                         compile_binary_basename)
 744             if platform.system() == "Windows" and not re.match(
 745                     r".*\.exe$", new_compiler, flags=re.IGNORECASE):
 746                 new_compiler += ".exe"
 747
 748             if os.path.isfile(new_compiler):
 749                 replacement_compile_line = "\"{}\" {}".format(
 750                     new_compiler, line)
 751
 752         return replacement_compile_line
 753
 754     def get_c_compile_line(self):
 755         _LOG.debug("ARDUINO_C_COMPILE: %s",
 756                    _pretty_format(self.platform["recipe.c.o.pattern"]))
 757
 758         compile_line = self.platform["recipe.c.o.pattern"]
 759         compile_line = self._strip_includes_source_file_object_file_vars(
 760             compile_line)
 761         compile_line += " -I{}".format(
 762             self.board[self.selected_board]["build.core.path"])
 763
 764         compile_line = self.replace_compile_binary_with_override_path(
 765             compile_line)
 766         return compile_line
 767
 768     def get_s_compile_line(self):
 769         _LOG.debug("ARDUINO_S_COMPILE %s",
 770                    _pretty_format(self.platform["recipe.S.o.pattern"]))
 771
 772         compile_line = self.platform["recipe.S.o.pattern"]
 773         compile_line = self._strip_includes_source_file_object_file_vars(
 774             compile_line)
 775         compile_line += " -I{}".format(
 776             self.board[self.selected_board]["build.core.path"])
 777
 778         compile_line = self.replace_compile_binary_with_override_path(
 779             compile_line)
 780         return compile_line
 781
 782     def get_ar_compile_line(self):
 783         _LOG.debug("ARDUINO_AR_COMPILE: %s",
 784                    _pretty_format(self.platform["recipe.ar.pattern"]))
 785
 786         compile_line = self.platform["recipe.ar.pattern"].replace(
 787             "\"{object_file}\"", "", 1)
 788
 789         compile_line = self.replace_compile_binary_with_override_path(
 790             compile_line)
 791         return compile_line
 792
 793     def get_cpp_compile_line(self):
 794         _LOG.debug("ARDUINO_CPP_COMPILE: %s",
 795                    _pretty_format(self.platform["recipe.cpp.o.pattern"]))
 796
 797         compile_line = self.platform["recipe.cpp.o.pattern"]
 798         compile_line = self._strip_includes_source_file_object_file_vars(
 799             compile_line)
 800         compile_line += " -I{}".format(
 801             self.board[self.selected_board]["build.core.path"])
 802
 803         compile_line = self.replace_compile_binary_with_override_path(
 804             compile_line)
 805         return compile_line
 806
 807     def get_link_line(self):
 808         _LOG.debug("ARDUINO_LINK: %s",
 809                    _pretty_format(self.platform["recipe.c.combine.pattern"]))
 810
 811         compile_line = self.platform["recipe.c.combine.pattern"]
 812
 813         compile_line = self.replace_compile_binary_with_override_path(
 814             compile_line)
 815         return compile_line
 816
 817     def get_objcopy_step_names(self):
 818         names = [
 819             name for name, line in self.platform.items()
 820             if self.OBJCOPY_STEP_NAME_REGEX.match(name)
 821         ]
 822         return names
 823
 824     def get_objcopy_steps(self) -> List[str]:
 825         lines = [
 826             line for name, line in self.platform.items()
 827             if self.OBJCOPY_STEP_NAME_REGEX.match(name)
 828         ]
 829         lines = [
 830             self.replace_compile_binary_with_override_path(line)
 831             for line in lines
 832         ]
 833         return lines
 834
 835     # TODO(tonymd): These recipes are probably run in sorted order
 836     def get_objcopy(self, suffix):
 837         # Expected vars:
 838         # teensy:
 839         #   recipe.objcopy.eep.pattern
 840         #   recipe.objcopy.hex.pattern
 841
 842         pattern = "recipe.objcopy.{}.pattern".format(suffix)
 843         objcopy_step_names = self.get_objcopy_step_names()
 844
 845         objcopy_suffixes = [
 846             m[1] for m in [
 847                 self.OBJCOPY_STEP_NAME_REGEX.match(line)
 848                 for line in objcopy_step_names
 849             ] if m
 850         ]
 851         if pattern not in objcopy_step_names:
 852             _LOG.error("Error: objcopy suffix '%s' does not exist.", suffix)
 853             _LOG.error("Valid suffixes: %s", ", ".join(objcopy_suffixes))
 854             return sys.exit(1)
 855
 856         line = self._get_platform_variable(pattern)
 857
 858         _LOG.debug("ARDUINO_OBJCOPY_%s: %s", suffix, line)
 859
 860         line = self.replace_compile_binary_with_override_path(line)
 861
 862         return line
 863
 864     def get_objcopy_flags(self, suffix):
 865         # TODO(tonymd): Possibly teensy specific variables.
 866         flags = ""
 867         if suffix == "hex":
 868             flags = self.platform.get("compiler.elf2hex.flags", "")
 869         elif suffix == "bin":
 870             flags = self.platform.get("compiler.elf2bin.flags", "")
 871         elif suffix == "eep":
 872             flags = self.platform.get("compiler.objcopy.eep.flags", "")
 873         return flags
 874
 875     # TODO(tonymd): There are more recipe hooks besides postbuild.
 876     #   They are run in sorted order.
 877     # TODO(tonymd): Rename this to get_hooks(hook_name, step).
 878     # TODO(tonymd): Add a list-hooks and or run-hooks command
 879     def get_postbuild_line(self, step_number):
 880         line = self.platform["recipe.hooks.postbuild.{}.pattern".format(
 881             step_number)]
 882         line = self.replace_command_args_with_compiler_override_path(line)
 883         return line
 884
 885     def get_prebuild_steps(self) -> List[str]:
 886         # Teensy core uses recipe.hooks.sketch.prebuild.1.pattern
 887         # stm32 core uses recipe.hooks.prebuild.1.pattern
 888         # TODO(tonymd): STM32 core uses recipe.hooks.prebuild.1.pattern.windows
 889         #   (should override non-windows key)
 890         lines = [
 891             line for name, line in self.platform.items() if re.match(
 892                 r"^recipe.hooks.(?:sketch.)?prebuild.[^.]+.pattern$", name)
 893         ]
 894         # TODO(tonymd): Write a function to fetch/replace OS specific patterns
 895         #   (ending in an OS string)
 896         lines = [
 897             self.replace_compile_binary_with_override_path(line)
 898             for line in lines
 899         ]
 900         return lines
 901
 902     def get_postbuild_steps(self) -> List[str]:
 903         lines = [
 904             line for name, line in self.platform.items()
 905             if re.match(r"^recipe.hooks.postbuild.[^.]+.pattern$", name)
 906         ]
 907
 908         lines = [
 909             self.replace_command_args_with_compiler_override_path(line)
 910             for line in lines
 911         ]
 912         return lines
 913
 914     def get_s_flags(self):
 915         compile_line = self.get_s_compile_line()
 916         _, compile_line = ArduinoBuilder.split_binary_from_arguments(
 917             compile_line)
 918         compile_line = compile_line.replace("-c", "", 1)
 919         return compile_line.strip()
 920
 921     def get_c_flags(self):
 922         compile_line = self.get_c_compile_line()
 923         _, compile_line = ArduinoBuilder.split_binary_from_arguments(
 924             compile_line)
 925         compile_line = compile_line.replace("-c", "", 1)
 926         return compile_line.strip()
 927
 928     def get_cpp_flags(self):
 929         compile_line = self.get_cpp_compile_line()
 930         _, compile_line = ArduinoBuilder.split_binary_from_arguments(
 931             compile_line)
 932         compile_line = compile_line.replace("-c", "", 1)
 933         return compile_line.strip()
 934
 935     def get_ar_flags(self):
 936         compile_line = self.get_ar_compile_line()
 937         _, compile_line = ArduinoBuilder.split_binary_from_arguments(
 938             compile_line)
 939         return compile_line.strip()
 940
 941     def get_ld_flags(self):
 942         compile_line = self.get_link_line()
 943         _, compile_line = ArduinoBuilder.split_binary_from_arguments(
 944             compile_line)
 945
 946         # TODO(tonymd): This is teensy specific
 947         line_to_delete = "-o \"{build.path}/{build.project_name}.elf\" " \
 948             "{object_files} \"-L{build.path}\""
 949         if self.build_path:
 950             line_to_delete = line_to_delete.replace("{build.path}",
 951                                                     self.build_path)
 952         if self.build_project_name:
 953             line_to_delete = line_to_delete.replace("{build.project_name}",
 954                                                     self.build_project_name)
 955
 956         compile_line = compile_line.replace(line_to_delete, "", 1)
 957         libs = re.findall(r'(-l[^ ]+ ?)', compile_line)
 958         for lib in libs:
 959             compile_line = compile_line.replace(lib, "", 1)
 960         libs = [lib.strip() for lib in libs]
 961
 962         return compile_line.strip()
 963
 964     def get_ld_libs(self, name_only=False):
 965         compile_line = self.get_link_line()
 966         libs = re.findall(r'(?P<arg>-l(?P<name>[^ ]+) ?)', compile_line)
 967         if name_only:
 968             libs = [lib_name.strip() for lib_arg, lib_name in libs]
 969         else:
 970             libs = [lib_arg.strip() for lib_arg, lib_name in libs]
 971         return " ".join(libs)
 972
 973     def library_folders(self):
 974         # Arduino library format documentation:
 975         # https://arduino.github.io/arduino-cli/library-specification/#layout-of-folders-and-files
 976         # - If src folder exists,
 977         #   use that as the root include directory -Ilibraries/libname/src
 978         # - Else lib folder as root include -Ilibraries/libname
 979         #   (exclude source files in the examples folder in this case)
 980
 981         if not self.library_names or not self.library_path:
 982             return []
 983
 984         folder_patterns = ["*"]
 985         if self.library_names:
 986             folder_patterns = self.library_names
 987
 988         library_folders = OrderedDict()
 989         for library_dir in self.library_path:
 990             found_library_names = file_operations.find_files(
 991                 library_dir, folder_patterns, directories_only=True)
 992             _LOG.debug("Found Libraries %s: %s", library_dir,
 993                        found_library_names)
 994             for lib_name in found_library_names:
 995                 lib_dir = os.path.join(library_dir, lib_name)
 996                 src_dir = os.path.join(lib_dir, "src")
 997                 if os.path.exists(src_dir) and os.path.isdir(src_dir):
 998                     library_folders[lib_name] = src_dir
 999                 else:
1000                     library_folders[lib_name] = lib_dir
1001
1002         return list(library_folders.values())
1003
1004     def library_include_dirs(self):
1005         return [Path(lib).as_posix() for lib in self.library_folders()]
1006
1007     def library_includes(self):
1008         include_args = []
1009         library_folders = self.library_folders()
1010         for lib_dir in library_folders:
1011             include_args.append("-I{}".format(os.path.relpath(lib_dir)))
1012         return include_args
1013
1014     def library_files(self, pattern, only_library_name=None):
1015         sources = []
1016         library_folders = self.library_folders()
1017         if only_library_name:
1018             library_folders = [
1019                 lf for lf in self.library_folders() if only_library_name in lf
1020             ]
1021         for lib_dir in library_folders:
1022             for file_path in file_operations.find_files(lib_dir, [pattern]):
1023                 if not file_path.startswith("examples"):
1024                     sources.append((Path(lib_dir) / file_path).as_posix())
1025         return sources
1026
1027     def library_c_files(self):
1028         return self.library_files("**/*.c")
1029
1030     def library_s_files(self):
1031         return self.library_files("**/*.S")
1032
1033     def library_cpp_files(self):
1034         return self.library_files("**/*.cpp")
1035
1036     def get_core_path(self):
1037         return self.board[self.selected_board]["build.core.path"]
1038
1039     def core_files(self, pattern):
1040         sources = []
1041         for file_path in file_operations.find_files(self.get_core_path(),
1042                                                     [pattern]):
1043             sources.append(os.path.join(self.get_core_path(), file_path))
1044         return sources
1045
1046     def core_c_files(self):
1047         return self.core_files("**/*.c")
1048
1049     def core_s_files(self):
1050         return self.core_files("**/*.S")
1051
1052     def core_cpp_files(self):
1053         return self.core_files("**/*.cpp")
1054
1055     def get_variant_path(self):
1056         return self.build_variant_path
1057
1058     def variant_files(self, pattern):
1059         sources = []
1060         if self.build_variant_path:
1061             for file_path in file_operations.find_files(
1062                     self.get_variant_path(), [pattern]):
1063                 sources.append(os.path.join(self.get_variant_path(),
1064                                             file_path))
1065         return sources
1066
1067     def variant_c_files(self):
1068         return self.variant_files("**/*.c")
1069
1070     def variant_s_files(self):
1071         return self.variant_files("**/*.S")
1072
1073     def variant_cpp_files(self):
1074         return self.variant_files("**/*.cpp")
1075
1076     def project_files(self, pattern):
1077         sources = []
1078         for file_path in file_operations.find_files(self.project_path,
1079                                                     [pattern]):
1080             if not file_path.startswith(
1081                     "examples") and not file_path.startswith("libraries"):
1082                 sources.append(file_path)
1083         return sources
1084
1085     def project_c_files(self):
1086         return self.project_files("**/*.c")
1087
1088     def project_cpp_files(self):
1089         return self.project_files("**/*.cpp")
1090
1091     def project_ino_files(self):
1092         return self.project_files("**/*.ino")