1#!/usr/bin/env python3 2# Copyright 2020 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Extracts build information from Arduino cores.""" 16 17import glob 18import logging 19import os 20import platform 21import pprint 22import re 23import sys 24import time 25from collections import OrderedDict 26from pathlib import Path 27from typing import List 28 29from pw_arduino_build import file_operations 30 31_LOG = logging.getLogger(__name__) 32 33_pretty_print = pprint.PrettyPrinter(indent=1, width=120).pprint 34_pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat 35 36 37def arduino_runtime_os_string(): 38 arduno_platform = { 39 "Linux": "linux", 40 "Windows": "windows", 41 "Darwin": "macosx", 42 } 43 return arduno_platform[platform.system()] 44 45 46class ArduinoBuilder: 47 """Used to interpret arduino boards.txt and platform.txt files.""" 48 49 # pylint: disable=too-many-instance-attributes,too-many-public-methods 50 51 BOARD_MENU_REGEX = re.compile( 52 r"^(?P<name>menu\.[^#=]+)=(?P<description>.*)$", re.MULTILINE 53 ) 54 55 BOARD_NAME_REGEX = re.compile( 56 r"^(?P<name>[^\s#\.]+)\.name=(?P<description>.*)$", re.MULTILINE 57 ) 58 59 VARIABLE_REGEX = re.compile( 60 r"^(?P<name>[^\s#=]+)=(?P<value>.*)$", re.MULTILINE 61 ) 62 63 MENU_OPTION_REGEX = re.compile( 64 r"^menu\." # starts with "menu" 65 r"(?P<menu_option_name>[^.]+)\." # first token after . 66 r"(?P<menu_option_value>[^.]+)$" 67 ) # second (final) token after . 68 69 TOOL_NAME_REGEX = re.compile( 70 r"^tools\." r"(?P<tool_name>[^.]+)\." # starts with "tools" 71 ) # first token after . 72 73 INTERPOLATED_VARIABLE_REGEX = re.compile(r"{[^}]+}", re.MULTILINE) 74 75 OBJCOPY_STEP_NAME_REGEX = re.compile(r"^recipe.objcopy.([^.]+).pattern$") 76 77 def __init__( 78 self, 79 arduino_path, 80 package_name, 81 build_path=None, 82 project_path=None, 83 project_source_path=None, 84 library_path=None, 85 library_names=None, 86 build_project_name=None, 87 compiler_path_override=False, 88 ): 89 self.arduino_path = arduino_path 90 self.arduino_package_name = package_name 91 self.selected_board = None 92 self.build_path = build_path 93 self.project_path = project_path 94 self.project_source_path = project_source_path 95 self.build_project_name = build_project_name 96 self.compiler_path_override = compiler_path_override 97 self.variant_includes = "" 98 self.build_variant_path = False 99 self.library_names = library_names 100 self.library_path = library_path 101 102 self.compiler_path_override_binaries = [] 103 if self.compiler_path_override: 104 self.compiler_path_override_binaries = file_operations.find_files( 105 self.compiler_path_override, "*" 106 ) 107 108 # Container dicts for boards.txt and platform.txt file data. 109 self.board = OrderedDict() 110 self.platform = OrderedDict() 111 self.menu_options = OrderedDict( 112 {"global_options": {}, "default_board_values": {}, "selected": {}} 113 ) 114 self.tools_variables = {} 115 116 # Set and check for valid hardware folder. 117 self.hardware_path = os.path.join(self.arduino_path, "hardware") 118 119 if not os.path.exists(self.hardware_path): 120 raise FileNotFoundError( 121 "Arduino package path '{}' does not exist.".format( 122 self.hardware_path 123 ) 124 ) 125 126 # Set and check for valid package name 127 self.package_path = os.path.join( 128 self.arduino_path, "hardware", package_name 129 ) 130 # {build.arch} is the first folder name of the package (upcased) 131 self.build_arch = os.path.split(package_name)[0].upper() 132 133 if not os.path.exists(self.package_path): 134 _LOG.error( 135 "Error: Arduino package name '%s' does not exist.", package_name 136 ) 137 _LOG.error("Did you mean:\n") 138 # TODO(tonymd): On Windows concatenating "/" may not work 139 possible_alternatives = [ 140 d.replace(self.hardware_path + os.sep, "", 1) 141 for d in glob.glob(self.hardware_path + "/*/*") 142 ] 143 _LOG.error("\n".join(possible_alternatives)) 144 sys.exit(1) 145 146 # Populate library paths. 147 if not library_path: 148 self.library_path = [] 149 # Append core libraries directory. 150 core_lib_path = Path(self.package_path) / "libraries" 151 if core_lib_path.is_dir(): 152 self.library_path.append(Path(self.package_path) / "libraries") 153 if library_path: 154 self.library_path = [ 155 os.path.realpath(os.path.expanduser(os.path.expandvars(l_path))) 156 for l_path in library_path 157 ] 158 159 # Grab all folder names in the cores directory. These are typically 160 # sub-core source files. 161 self.sub_core_folders = os.listdir( 162 os.path.join(self.package_path, "cores") 163 ) 164 165 self._find_tools_variables() 166 167 self.boards_txt = os.path.join(self.package_path, "boards.txt") 168 self.platform_txt = os.path.join(self.package_path, "platform.txt") 169 170 def select_board(self, board_name, menu_option_overrides=False): 171 self.selected_board = board_name 172 173 # Load default menu options for a selected board. 174 if not self.selected_board in self.board.keys(): 175 _LOG.error("Error board: '%s' not supported.", self.selected_board) 176 # TODO(tonymd): Print supported boards here 177 sys.exit(1) 178 179 # Override default menu options if any are specified. 180 if menu_option_overrides: 181 for moption in menu_option_overrides: 182 if not self.set_menu_option(moption): 183 # TODO(tonymd): Print supported menu options here 184 sys.exit(1) 185 186 self._copy_default_menu_options_to_build_variables() 187 self._apply_recipe_overrides() 188 self._substitute_variables() 189 190 def set_variables(self, variable_list: List[str]): 191 # Convert the string list containing 'name=value' items into a dict 192 variable_source = {} 193 for var in variable_list: 194 var_name, value = var.split("=") 195 variable_source[var_name] = value 196 197 # Replace variables in platform 198 for var, value in self.platform.items(): 199 self.platform[var] = self._replace_variables(value, variable_source) 200 201 def _apply_recipe_overrides(self): 202 # Override link recipes with per-core exceptions 203 # Teensyduino cores 204 if self.build_arch == "TEENSY": 205 # Change {build.path}/{archive_file} 206 # To {archive_file_path} (which should contain the core.a file) 207 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 208 "{object_files} \"{build.path}/{archive_file}\"", 209 "{object_files} {archive_file_path}", 210 1, 211 ) 212 # Add the teensy provided toolchain lib folder for link access to 213 # libarm_cortexM*_math.a 214 new_link_line = new_link_line.replace( 215 "\"-L{build.path}\"", 216 "\"-L{build.path}\" -L{compiler.path}/arm/arm-none-eabi/lib", 217 1, 218 ) 219 self.platform["recipe.c.combine.pattern"] = new_link_line 220 # Remove the pre-compiled header include 221 self.platform["recipe.cpp.o.pattern"] = self.platform[ 222 "recipe.cpp.o.pattern" 223 ].replace("\"-I{build.path}/pch\"", "", 1) 224 225 # Adafruit-samd core 226 # TODO(tonymd): This build_arch may clash with Arduino-SAMD core 227 elif self.build_arch == "SAMD": 228 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 229 "\"{build.path}/{archive_file}\" -Wl,--end-group", 230 "{archive_file_path} -Wl,--end-group", 231 1, 232 ) 233 self.platform["recipe.c.combine.pattern"] = new_link_line 234 235 # STM32L4 Core: 236 # https://github.com/GrumpyOldPizza/arduino-STM32L4 237 elif self.build_arch == "STM32L4": 238 # TODO(tonymd): {build.path}/{archive_file} for the link step always 239 # seems to be core.a (except STM32 core) 240 line_to_delete = "-Wl,--start-group \"{build.path}/{archive_file}\"" 241 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 242 line_to_delete, "-Wl,--start-group {archive_file_path}", 1 243 ) 244 self.platform["recipe.c.combine.pattern"] = new_link_line 245 246 # stm32duino core 247 elif self.build_arch == "STM32": 248 # Must link in SrcWrapper for all projects. 249 if not self.library_names: 250 self.library_names = [] 251 self.library_names.append("SrcWrapper") 252 253 def _copy_default_menu_options_to_build_variables(self): 254 # Clear existing options 255 self.menu_options["selected"] = {} 256 # Set default menu options for selected board 257 for menu_key, menu_dict in self.menu_options["default_board_values"][ 258 self.selected_board 259 ].items(): 260 for name, var in self.board[self.selected_board].items(): 261 starting_key = "{}.{}.".format(menu_key, menu_dict["name"]) 262 if name.startswith(starting_key): 263 new_var_name = name.replace(starting_key, "", 1) 264 self.menu_options["selected"][new_var_name] = var 265 266 def set_menu_option(self, moption): 267 if moption not in self.board[self.selected_board]: 268 _LOG.error("Error: '%s' is not a valid menu option.", moption) 269 return False 270 271 # Override default menu option with new value. 272 menu_match_result = self.MENU_OPTION_REGEX.match(moption) 273 if menu_match_result: 274 menu_match = menu_match_result.groupdict() 275 menu_value = menu_match["menu_option_value"] 276 menu_key = "menu.{}".format(menu_match["menu_option_name"]) 277 self.menu_options["default_board_values"][self.selected_board][ 278 menu_key 279 ]["name"] = menu_value 280 281 # Update build variables 282 self._copy_default_menu_options_to_build_variables() 283 return True 284 285 def _set_global_arduino_variables(self): 286 """Set some global variables defined by the Arduino-IDE. 287 288 See Docs: 289 https://arduino.github.io/arduino-cli/platform-specification/#global-predefined-properties 290 """ 291 292 # TODO(tonymd): Make sure these variables are replaced in recipe lines 293 # even if they are None: build_path, project_path, project_source_path, 294 # build_project_name 295 for current_board in self.board.values(): 296 if self.build_path: 297 current_board["build.path"] = self.build_path 298 if self.build_project_name: 299 current_board["build.project_name"] = self.build_project_name 300 # {archive_file} is the final *.elf 301 archive_file = "{}.elf".format(self.build_project_name) 302 current_board["archive_file"] = archive_file 303 # {archive_file_path} is the final core.a archive 304 if self.build_path: 305 current_board["archive_file_path"] = os.path.join( 306 self.build_path, "core.a" 307 ) 308 if self.project_source_path: 309 current_board["build.source.path"] = self.project_source_path 310 311 current_board["extra.time.local"] = str(int(time.time())) 312 current_board["runtime.ide.version"] = "10812" 313 current_board["runtime.hardware.path"] = self.hardware_path 314 315 # Copy {runtime.tools.TOOL_NAME.path} vars 316 self._set_tools_variables(current_board) 317 318 current_board["runtime.platform.path"] = self.package_path 319 if self.platform["name"] == "Teensyduino": 320 # Teensyduino is installed into the arduino IDE folder 321 # rather than ~/.arduino15/packages/ 322 current_board["runtime.hardware.path"] = os.path.join( 323 self.hardware_path, "teensy" 324 ) 325 326 current_board["build.system.path"] = os.path.join( 327 self.package_path, "system" 328 ) 329 330 # Set the {build.core.path} variable that pointing to a sub-core 331 # folder. For Teensys this is: 332 # 'teensy/hardware/teensy/avr/cores/teensy{3,4}'. For other cores 333 # it's typically just the 'arduino' folder. For example: 334 # 'arduino-samd/hardware/samd/1.8.8/cores/arduino' 335 core_path = Path(self.package_path) / "cores" 336 core_path /= current_board.get( 337 "build.core", self.sub_core_folders[0] 338 ) 339 current_board["build.core.path"] = core_path.as_posix() 340 341 current_board["build.arch"] = self.build_arch 342 343 for name, var in current_board.items(): 344 current_board[name] = var.replace( 345 "{build.core.path}", core_path.as_posix() 346 ) 347 348 def load_board_definitions(self): 349 """Loads Arduino boards.txt and platform.txt files into dictionaries. 350 351 Populates the following dictionaries: 352 self.menu_options 353 self.boards 354 self.platform 355 """ 356 # Load platform.txt 357 with open(self.platform_txt, "r") as pfile: 358 platform_file = pfile.read() 359 platform_var_matches = self.VARIABLE_REGEX.finditer(platform_file) 360 for p_match in [m.groupdict() for m in platform_var_matches]: 361 self.platform[p_match["name"]] = p_match["value"] 362 363 # Load boards.txt 364 with open(self.boards_txt, "r") as bfile: 365 board_file = bfile.read() 366 # Get all top-level menu options, e.g. menu.usb=USB Type 367 board_menu_matches = self.BOARD_MENU_REGEX.finditer(board_file) 368 for menuitem in [m.groupdict() for m in board_menu_matches]: 369 self.menu_options["global_options"][menuitem["name"]] = { 370 "description": menuitem["description"] 371 } 372 373 # Get all board names, e.g. teensy40.name=Teensy 4.0 374 board_name_matches = self.BOARD_NAME_REGEX.finditer(board_file) 375 for b_match in [m.groupdict() for m in board_name_matches]: 376 self.board[b_match["name"]] = OrderedDict() 377 self.menu_options["default_board_values"][ 378 b_match["name"] 379 ] = OrderedDict() 380 381 # Get all board variables, e.g. teensy40.* 382 for current_board_name, current_board in self.board.items(): 383 board_line_matches = re.finditer( 384 fr"^\s*{current_board_name}\." 385 fr"(?P<key>[^#=]+)=(?P<value>.*)$", 386 board_file, 387 re.MULTILINE, 388 ) 389 for b_match in [m.groupdict() for m in board_line_matches]: 390 # Check if this line is a menu option 391 # (e.g. 'menu.usb.serial') and save as default if it's the 392 # first one seen. 393 ArduinoBuilder.save_default_menu_option( 394 current_board_name, 395 b_match["key"], 396 b_match["value"], 397 self.menu_options, 398 ) 399 current_board[b_match["key"]] = b_match["value"].strip() 400 401 self._set_global_arduino_variables() 402 403 @staticmethod 404 def save_default_menu_option(current_board_name, key, value, menu_options): 405 """Save a given menu option as the default. 406 407 Saves the key and value into menu_options["default_board_values"] 408 if it doesn't already exist. Assumes menu options are added in the order 409 specified in boards.txt. The first value for a menu key is the default. 410 """ 411 # Check if key is a menu option 412 # e.g. menu.usb.serial 413 # menu.usb.serial.build.usbtype 414 menu_match_result = re.match( 415 r'^menu\.' # starts with "menu" 416 r'(?P<menu_option_name>[^.]+)\.' # first token after . 417 r'(?P<menu_option_value>[^.]+)' # second token after . 418 r'(\.(?P<rest>.+))?', # optionally any trailing tokens after a . 419 key, 420 ) 421 if menu_match_result: 422 menu_match = menu_match_result.groupdict() 423 current_menu_key = "menu.{}".format(menu_match["menu_option_name"]) 424 # If this is the first menu option seen for current_board_name, save 425 # as the default. 426 if ( 427 current_menu_key 428 not in menu_options["default_board_values"][current_board_name] 429 ): 430 menu_options["default_board_values"][current_board_name][ 431 current_menu_key 432 ] = { 433 "name": menu_match["menu_option_value"], 434 "description": value, 435 } 436 437 def _replace_variables(self, line, variable_lookup_source): 438 """Replace {variables} from loaded boards.txt or platform.txt. 439 440 Replace interpolated variables surrounded by curly braces in line with 441 definitions from variable_lookup_source. 442 """ 443 new_line = line 444 for current_var_match in self.INTERPOLATED_VARIABLE_REGEX.findall(line): 445 # {build.flags.c} --> build.flags.c 446 current_var = current_var_match.strip("{}") 447 448 # check for matches in board definition 449 if current_var in variable_lookup_source: 450 replacement = variable_lookup_source.get(current_var, "") 451 new_line = new_line.replace(current_var_match, replacement) 452 return new_line 453 454 def _find_tools_variables(self): 455 # Gather tool directories in order of increasing precedence 456 runtime_tool_paths = [] 457 458 # Check for tools installed in ~/.arduino15/packages/arduino/tools/ 459 # TODO(tonymd): Is this Mac & Linux specific? 460 runtime_tool_paths += glob.glob( 461 os.path.join( 462 os.path.realpath(os.path.expanduser(os.path.expandvars("~"))), 463 ".arduino15", 464 "packages", 465 "arduino", 466 "tools", 467 "*", 468 ) 469 ) 470 471 # <ARDUINO_PATH>/tools/<OS_STRING>/<TOOL_NAMES> 472 runtime_tool_paths += glob.glob( 473 os.path.join( 474 self.arduino_path, "tools", arduino_runtime_os_string(), "*" 475 ) 476 ) 477 # <ARDUINO_PATH>/tools/<TOOL_NAMES> 478 # This will grab linux/windows/macosx/share as <TOOL_NAMES>. 479 runtime_tool_paths += glob.glob( 480 os.path.join(self.arduino_path, "tools", "*") 481 ) 482 483 # Process package tools after arduino tools. 484 # They should overwrite vars & take precedence. 485 486 # <PACKAGE_PATH>/tools/<OS_STRING>/<TOOL_NAMES> 487 runtime_tool_paths += glob.glob( 488 os.path.join( 489 self.package_path, "tools", arduino_runtime_os_string(), "*" 490 ) 491 ) 492 # <PACKAGE_PATH>/tools/<TOOL_NAMES> 493 # This will grab linux/windows/macosx/share as <TOOL_NAMES>. 494 runtime_tool_paths += glob.glob( 495 os.path.join(self.package_path, "tools", "*") 496 ) 497 498 for path in runtime_tool_paths: 499 # Make sure TOOL_NAME is not an OS string 500 if not ( 501 path.endswith("linux") 502 or path.endswith("windows") 503 or path.endswith("macosx") 504 or path.endswith("share") 505 ): 506 # TODO(tonymd): Check if a file & do nothing? 507 508 # Check if it's a directory with subdir(s) as a version string 509 # create all 'runtime.tools.{tool_folder}-{version.path}' 510 # (for each version) 511 # create 'runtime.tools.{tool_folder}.path' 512 # (with latest version) 513 if os.path.isdir(path): 514 # Grab the tool name (folder) by itself. 515 tool_folder = os.path.basename(path) 516 # Sort so that [-1] is the latest version. 517 version_paths = sorted(glob.glob(os.path.join(path, "*"))) 518 # Check if all sub folders start with a version string. 519 if len(version_paths) == sum( 520 bool(re.match(r"^[0-9.]+", os.path.basename(vp))) 521 for vp in version_paths 522 ): 523 for version_path in version_paths: 524 version_string = os.path.basename(version_path) 525 var_name = "runtime.tools.{}-{}.path".format( 526 tool_folder, version_string 527 ) 528 self.tools_variables[var_name] = os.path.join( 529 path, version_string 530 ) 531 var_name = "runtime.tools.{}.path".format(tool_folder) 532 self.tools_variables[var_name] = os.path.join( 533 path, os.path.basename(version_paths[-1]) 534 ) 535 # Else set toolpath to path. 536 else: 537 var_name = "runtime.tools.{}.path".format(tool_folder) 538 self.tools_variables[var_name] = path 539 540 _LOG.debug("TOOL VARIABLES: %s", _pretty_format(self.tools_variables)) 541 542 # Copy self.tools_variables into destination 543 def _set_tools_variables(self, destination): 544 for key, value in self.tools_variables.items(): 545 destination[key] = value 546 547 def _substitute_variables(self): 548 """Perform variable substitution in board and platform metadata.""" 549 550 # menu -> board 551 # Copy selected menu variables into board definiton 552 for name, value in self.menu_options["selected"].items(): 553 self.board[self.selected_board][name] = value 554 555 # board -> board 556 # Replace any {vars} in the selected board with values defined within 557 # (and from copied in menu options). 558 for var, value in self.board[self.selected_board].items(): 559 self.board[self.selected_board][var] = self._replace_variables( 560 value, self.board[self.selected_board] 561 ) 562 563 # Check for build.variant variable 564 # This will be set in selected board after menu options substitution 565 build_variant = self.board[self.selected_board].get( 566 "build.variant", None 567 ) 568 if build_variant: 569 # Set build.variant.path 570 bvp = os.path.join(self.package_path, "variants", build_variant) 571 self.build_variant_path = bvp 572 self.board[self.selected_board]["build.variant.path"] = bvp 573 # Add the variant folder as an include directory 574 # (used in stm32l4 core) 575 self.variant_includes = "-I{}".format(bvp) 576 577 _LOG.debug("PLATFORM INITIAL: %s", _pretty_format(self.platform)) 578 579 # board -> platform 580 # Replace {vars} in platform from the selected board definition 581 for var, value in self.platform.items(): 582 self.platform[var] = self._replace_variables( 583 value, self.board[self.selected_board] 584 ) 585 586 # platform -> platform 587 # Replace any remaining {vars} in platform from platform 588 for var, value in self.platform.items(): 589 self.platform[var] = self._replace_variables(value, self.platform) 590 591 # Repeat platform -> platform for any lingering variables 592 # Example: {build.opt.name} in STM32 core 593 for var, value in self.platform.items(): 594 self.platform[var] = self._replace_variables(value, self.platform) 595 596 _LOG.debug("MENU_OPTIONS: %s", _pretty_format(self.menu_options)) 597 _LOG.debug( 598 "SELECTED_BOARD: %s", 599 _pretty_format(self.board[self.selected_board]), 600 ) 601 _LOG.debug("PLATFORM: %s", _pretty_format(self.platform)) 602 603 def selected_board_spec(self): 604 return self.board[self.selected_board] 605 606 def get_menu_options(self): 607 all_options = [] 608 max_string_length = [0, 0] 609 610 for key_name, description in self.board[self.selected_board].items(): 611 menu_match_result = self.MENU_OPTION_REGEX.match(key_name) 612 if menu_match_result: 613 menu_match = menu_match_result.groupdict() 614 name = "menu.{}.{}".format( 615 menu_match["menu_option_name"], 616 menu_match["menu_option_value"], 617 ) 618 if len(name) > max_string_length[0]: 619 max_string_length[0] = len(name) 620 if len(description) > max_string_length[1]: 621 max_string_length[1] = len(description) 622 all_options.append((name, description)) 623 624 return all_options, max_string_length 625 626 def get_default_menu_options(self): 627 default_options = [] 628 max_string_length = [0, 0] 629 630 for key_name, value in self.menu_options["default_board_values"][ 631 self.selected_board 632 ].items(): 633 full_key = key_name + "." + value["name"] 634 if len(full_key) > max_string_length[0]: 635 max_string_length[0] = len(full_key) 636 if len(value["description"]) > max_string_length[1]: 637 max_string_length[1] = len(value["description"]) 638 default_options.append((full_key, value["description"])) 639 640 return default_options, max_string_length 641 642 @staticmethod 643 def split_binary_from_arguments(compile_line): 644 compile_binary = None 645 rest_of_line = compile_line 646 647 compile_binary_match = re.search(r'^("[^"]+") ', compile_line) 648 if compile_binary_match: 649 compile_binary = compile_binary_match[1] 650 rest_of_line = compile_line.replace(compile_binary_match[0], "", 1) 651 652 return compile_binary, rest_of_line 653 654 def _strip_includes_source_file_object_file_vars(self, compile_line): 655 line = compile_line 656 if self.variant_includes: 657 line = compile_line.replace( 658 "{includes} \"{source_file}\" -o \"{object_file}\"", 659 self.variant_includes, 660 1, 661 ) 662 else: 663 line = compile_line.replace( 664 "{includes} \"{source_file}\" -o \"{object_file}\"", "", 1 665 ) 666 return line 667 668 def _get_tool_name(self, line): 669 tool_match_result = self.TOOL_NAME_REGEX.match(line) 670 if tool_match_result: 671 return tool_match_result[1] 672 return False 673 674 def get_upload_tool_names(self): 675 return [ 676 self._get_tool_name(t) 677 for t in self.platform.keys() 678 if self.TOOL_NAME_REGEX.match(t) and 'upload.pattern' in t 679 ] 680 681 # TODO(tonymd): Use these getters in _replace_variables() or 682 # _substitute_variables() 683 684 def _get_platform_variable(self, variable): 685 # TODO(tonymd): Check for '.macos' '.linux' '.windows' in variable name, 686 # compare with platform.system() and return that instead. 687 return self.platform.get(variable, False) 688 689 def _get_platform_variable_with_substitutions(self, variable, namespace): 690 line = self.platform.get(variable, False) 691 # Get all unique variables used in this line in line. 692 unique_vars = sorted( 693 set(self.INTERPOLATED_VARIABLE_REGEX.findall(line)) 694 ) 695 # Search for each unique_vars in namespace and global. 696 for var in unique_vars: 697 v_raw_name = var.strip("{}") 698 699 # Check for namespace.variable 700 # eg: 'tools.stm32CubeProg.cmd' 701 possible_var_name = "{}.{}".format(namespace, v_raw_name) 702 result = self._get_platform_variable(possible_var_name) 703 # Check for os overriden variable 704 # eg: 705 # ('tools.stm32CubeProg.cmd', 'stm32CubeProg.sh'), 706 # ('tools.stm32CubeProg.cmd.windows', 'stm32CubeProg.bat'), 707 possible_var_name = "{}.{}.{}".format( 708 namespace, v_raw_name, arduino_runtime_os_string() 709 ) 710 os_override_result = self._get_platform_variable(possible_var_name) 711 712 if os_override_result: 713 line = line.replace(var, os_override_result) 714 elif result: 715 line = line.replace(var, result) 716 # Check for variable at top level? 717 # elif self._get_platform_variable(v_raw_name): 718 # line = line.replace(self._get_platform_variable(v_raw_name), 719 # result) 720 return line 721 722 def get_upload_line(self, tool_name, serial_port=False): 723 """TODO(tonymd) Add docstring.""" 724 # TODO(tonymd): Error if tool_name does not exist 725 tool_namespace = "tools.{}".format(tool_name) 726 pattern = "tools.{}.upload.pattern".format(tool_name) 727 728 if not self._get_platform_variable(pattern): 729 _LOG.error("Error: upload tool '%s' does not exist.", tool_name) 730 tools = self.get_upload_tool_names() 731 _LOG.error("Valid tools: %s", ", ".join(tools)) 732 return sys.exit(1) 733 734 line = self._get_platform_variable_with_substitutions( 735 pattern, tool_namespace 736 ) 737 738 # TODO(tonymd): Teensy specific tool overrides. 739 if tool_name == "teensyloader": 740 # Remove un-necessary lines 741 # {serial.port.label} and {serial.port.protocol} are returned by 742 # the teensy_ports binary. 743 line = line.replace("\"-portlabel={serial.port.label}\"", "", 1) 744 line = line.replace( 745 "\"-portprotocol={serial.port.protocol}\"", "", 1 746 ) 747 748 if serial_port == "UNKNOWN" or not serial_port: 749 line = line.replace('"-port={serial.port}"', "", 1) 750 else: 751 line = line.replace("{serial.port}", serial_port, 1) 752 753 return line 754 755 def _get_binary_path(self, variable_pattern): 756 compile_line = self.replace_compile_binary_with_override_path( 757 self._get_platform_variable(variable_pattern) 758 ) 759 compile_binary, _ = ArduinoBuilder.split_binary_from_arguments( 760 compile_line 761 ) 762 return compile_binary 763 764 def get_cc_binary(self): 765 return self._get_binary_path("recipe.c.o.pattern") 766 767 def get_cxx_binary(self): 768 return self._get_binary_path("recipe.cpp.o.pattern") 769 770 def get_objcopy_binary(self): 771 objcopy_step_name = self.get_objcopy_step_names()[0] 772 objcopy_binary = self._get_binary_path(objcopy_step_name) 773 return objcopy_binary 774 775 def get_ar_binary(self): 776 return self._get_binary_path("recipe.ar.pattern") 777 778 def get_size_binary(self): 779 return self._get_binary_path("recipe.size.pattern") 780 781 def replace_command_args_with_compiler_override_path(self, compile_line): 782 if not self.compiler_path_override: 783 return compile_line 784 replacement_line = compile_line 785 replacement_line_args = compile_line.split() 786 for arg in replacement_line_args: 787 compile_binary_basename = os.path.basename(arg.strip("\"")) 788 if compile_binary_basename in self.compiler_path_override_binaries: 789 new_compiler = os.path.join( 790 self.compiler_path_override, compile_binary_basename 791 ) 792 replacement_line = replacement_line.replace( 793 arg, new_compiler, 1 794 ) 795 return replacement_line 796 797 def replace_compile_binary_with_override_path(self, compile_line): 798 replacement_compile_line = compile_line 799 800 # Change the compiler path if there's an override path set 801 if self.compiler_path_override: 802 compile_binary, line = ArduinoBuilder.split_binary_from_arguments( 803 compile_line 804 ) 805 compile_binary_basename = os.path.basename( 806 compile_binary.strip("\"") 807 ) 808 new_compiler = os.path.join( 809 self.compiler_path_override, compile_binary_basename 810 ) 811 if platform.system() == "Windows" and not re.match( 812 r".*\.exe$", new_compiler, flags=re.IGNORECASE 813 ): 814 new_compiler += ".exe" 815 816 if os.path.isfile(new_compiler): 817 replacement_compile_line = "\"{}\" {}".format( 818 new_compiler, line 819 ) 820 821 return replacement_compile_line 822 823 def get_c_compile_line(self): 824 _LOG.debug( 825 "ARDUINO_C_COMPILE: %s", 826 _pretty_format(self.platform["recipe.c.o.pattern"]), 827 ) 828 829 compile_line = self.platform["recipe.c.o.pattern"] 830 compile_line = self._strip_includes_source_file_object_file_vars( 831 compile_line 832 ) 833 compile_line += " -I{}".format( 834 self.board[self.selected_board]["build.core.path"] 835 ) 836 837 compile_line = self.replace_compile_binary_with_override_path( 838 compile_line 839 ) 840 return compile_line 841 842 def get_s_compile_line(self): 843 _LOG.debug( 844 "ARDUINO_S_COMPILE %s", 845 _pretty_format(self.platform["recipe.S.o.pattern"]), 846 ) 847 848 compile_line = self.platform["recipe.S.o.pattern"] 849 compile_line = self._strip_includes_source_file_object_file_vars( 850 compile_line 851 ) 852 compile_line += " -I{}".format( 853 self.board[self.selected_board]["build.core.path"] 854 ) 855 856 compile_line = self.replace_compile_binary_with_override_path( 857 compile_line 858 ) 859 return compile_line 860 861 def get_ar_compile_line(self): 862 _LOG.debug( 863 "ARDUINO_AR_COMPILE: %s", 864 _pretty_format(self.platform["recipe.ar.pattern"]), 865 ) 866 867 compile_line = self.platform["recipe.ar.pattern"].replace( 868 "\"{object_file}\"", "", 1 869 ) 870 871 compile_line = self.replace_compile_binary_with_override_path( 872 compile_line 873 ) 874 return compile_line 875 876 def get_cpp_compile_line(self): 877 _LOG.debug( 878 "ARDUINO_CPP_COMPILE: %s", 879 _pretty_format(self.platform["recipe.cpp.o.pattern"]), 880 ) 881 882 compile_line = self.platform["recipe.cpp.o.pattern"] 883 compile_line = self._strip_includes_source_file_object_file_vars( 884 compile_line 885 ) 886 compile_line += " -I{}".format( 887 self.board[self.selected_board]["build.core.path"] 888 ) 889 890 compile_line = self.replace_compile_binary_with_override_path( 891 compile_line 892 ) 893 return compile_line 894 895 def get_link_line(self): 896 _LOG.debug( 897 "ARDUINO_LINK: %s", 898 _pretty_format(self.platform["recipe.c.combine.pattern"]), 899 ) 900 901 compile_line = self.platform["recipe.c.combine.pattern"] 902 903 compile_line = self.replace_compile_binary_with_override_path( 904 compile_line 905 ) 906 return compile_line 907 908 def get_objcopy_step_names(self): 909 names = [ 910 name 911 for name, line in self.platform.items() 912 if self.OBJCOPY_STEP_NAME_REGEX.match(name) 913 ] 914 return names 915 916 def get_objcopy_steps(self) -> List[str]: 917 lines = [ 918 line 919 for name, line in self.platform.items() 920 if self.OBJCOPY_STEP_NAME_REGEX.match(name) 921 ] 922 lines = [ 923 self.replace_compile_binary_with_override_path(line) 924 for line in lines 925 ] 926 return lines 927 928 # TODO(tonymd): These recipes are probably run in sorted order 929 def get_objcopy(self, suffix): 930 # Expected vars: 931 # teensy: 932 # recipe.objcopy.eep.pattern 933 # recipe.objcopy.hex.pattern 934 935 pattern = "recipe.objcopy.{}.pattern".format(suffix) 936 objcopy_step_names = self.get_objcopy_step_names() 937 938 objcopy_suffixes = [ 939 m[1] 940 for m in [ 941 self.OBJCOPY_STEP_NAME_REGEX.match(line) 942 for line in objcopy_step_names 943 ] 944 if m 945 ] 946 if pattern not in objcopy_step_names: 947 _LOG.error("Error: objcopy suffix '%s' does not exist.", suffix) 948 _LOG.error("Valid suffixes: %s", ", ".join(objcopy_suffixes)) 949 return sys.exit(1) 950 951 line = self._get_platform_variable(pattern) 952 953 _LOG.debug("ARDUINO_OBJCOPY_%s: %s", suffix, line) 954 955 line = self.replace_compile_binary_with_override_path(line) 956 957 return line 958 959 def get_objcopy_flags(self, suffix): 960 # TODO(tonymd): Possibly teensy specific variables. 961 flags = "" 962 if suffix == "hex": 963 flags = self.platform.get("compiler.elf2hex.flags", "") 964 elif suffix == "bin": 965 flags = self.platform.get("compiler.elf2bin.flags", "") 966 elif suffix == "eep": 967 flags = self.platform.get("compiler.objcopy.eep.flags", "") 968 return flags 969 970 # TODO(tonymd): There are more recipe hooks besides postbuild. 971 # They are run in sorted order. 972 # TODO(tonymd): Rename this to get_hooks(hook_name, step). 973 # TODO(tonymd): Add a list-hooks and or run-hooks command 974 def get_postbuild_line(self, step_number): 975 line = self.platform[ 976 "recipe.hooks.postbuild.{}.pattern".format(step_number) 977 ] 978 line = self.replace_command_args_with_compiler_override_path(line) 979 return line 980 981 def get_prebuild_steps(self) -> List[str]: 982 # Teensy core uses recipe.hooks.sketch.prebuild.1.pattern 983 # stm32 core uses recipe.hooks.prebuild.1.pattern 984 # TODO(tonymd): STM32 core uses recipe.hooks.prebuild.1.pattern.windows 985 # (should override non-windows key) 986 lines = [ 987 line 988 for name, line in self.platform.items() 989 if re.match( 990 r"^recipe.hooks.(?:sketch.)?prebuild.[^.]+.pattern$", name 991 ) 992 ] 993 # TODO(tonymd): Write a function to fetch/replace OS specific patterns 994 # (ending in an OS string) 995 lines = [ 996 self.replace_compile_binary_with_override_path(line) 997 for line in lines 998 ] 999 return lines 1000 1001 def get_postbuild_steps(self) -> List[str]: 1002 lines = [ 1003 line 1004 for name, line in self.platform.items() 1005 if re.match(r"^recipe.hooks.postbuild.[^.]+.pattern$", name) 1006 ] 1007 1008 lines = [ 1009 self.replace_command_args_with_compiler_override_path(line) 1010 for line in lines 1011 ] 1012 return lines 1013 1014 def get_s_flags(self): 1015 compile_line = self.get_s_compile_line() 1016 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 1017 compile_line 1018 ) 1019 compile_line = compile_line.replace("-c", "", 1) 1020 return compile_line.strip() 1021 1022 def get_c_flags(self): 1023 compile_line = self.get_c_compile_line() 1024 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 1025 compile_line 1026 ) 1027 compile_line = compile_line.replace("-c", "", 1) 1028 return compile_line.strip() 1029 1030 def get_cpp_flags(self): 1031 compile_line = self.get_cpp_compile_line() 1032 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 1033 compile_line 1034 ) 1035 compile_line = compile_line.replace("-c", "", 1) 1036 return compile_line.strip() 1037 1038 def get_ar_flags(self): 1039 compile_line = self.get_ar_compile_line() 1040 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 1041 compile_line 1042 ) 1043 return compile_line.strip() 1044 1045 def get_ld_flags(self): 1046 compile_line = self.get_link_line() 1047 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 1048 compile_line 1049 ) 1050 1051 # TODO(tonymd): This is teensy specific 1052 line_to_delete = ( 1053 "-o \"{build.path}/{build.project_name}.elf\" " 1054 "{object_files} \"-L{build.path}\"" 1055 ) 1056 if self.build_path: 1057 line_to_delete = line_to_delete.replace( 1058 "{build.path}", self.build_path 1059 ) 1060 if self.build_project_name: 1061 line_to_delete = line_to_delete.replace( 1062 "{build.project_name}", self.build_project_name 1063 ) 1064 1065 compile_line = compile_line.replace(line_to_delete, "", 1) 1066 libs = re.findall(r'(-l[^ ]+ ?)', compile_line) 1067 for lib in libs: 1068 compile_line = compile_line.replace(lib, "", 1) 1069 libs = [lib.strip() for lib in libs] 1070 1071 return compile_line.strip() 1072 1073 def get_ld_libs(self, name_only=False): 1074 compile_line = self.get_link_line() 1075 libs = re.findall(r'(?P<arg>-l(?P<name>[^ ]+) ?)', compile_line) 1076 if name_only: 1077 libs = [lib_name.strip() for lib_arg, lib_name in libs] 1078 else: 1079 libs = [lib_arg.strip() for lib_arg, lib_name in libs] 1080 return " ".join(libs) 1081 1082 def library_folders(self): 1083 """TODO(tonymd) Add docstring.""" 1084 # Arduino library format documentation: 1085 # https://arduino.github.io/arduino-cli/library-specification/#layout-of-folders-and-files 1086 # - If src folder exists, 1087 # use that as the root include directory -Ilibraries/libname/src 1088 # - Else lib folder as root include -Ilibraries/libname 1089 # (exclude source files in the examples folder in this case) 1090 1091 if not self.library_names or not self.library_path: 1092 return [] 1093 1094 folder_patterns = ["*"] 1095 if self.library_names: 1096 folder_patterns = self.library_names 1097 1098 library_folders = OrderedDict() 1099 for library_dir in self.library_path: 1100 found_library_names = file_operations.find_files( 1101 library_dir, folder_patterns, directories_only=True 1102 ) 1103 _LOG.debug( 1104 "Found Libraries %s: %s", library_dir, found_library_names 1105 ) 1106 for lib_name in found_library_names: 1107 lib_dir = os.path.join(library_dir, lib_name) 1108 src_dir = os.path.join(lib_dir, "src") 1109 if os.path.exists(src_dir) and os.path.isdir(src_dir): 1110 library_folders[lib_name] = src_dir 1111 else: 1112 library_folders[lib_name] = lib_dir 1113 1114 return list(library_folders.values()) 1115 1116 def library_include_dirs(self): 1117 return [Path(lib).as_posix() for lib in self.library_folders()] 1118 1119 def library_includes(self): 1120 include_args = [] 1121 library_folders = self.library_folders() 1122 for lib_dir in library_folders: 1123 include_args.append("-I{}".format(os.path.relpath(lib_dir))) 1124 return include_args 1125 1126 def library_files(self, pattern, only_library_name=None): 1127 sources = [] 1128 library_folders = self.library_folders() 1129 if only_library_name: 1130 library_folders = [ 1131 lf for lf in self.library_folders() if only_library_name in lf 1132 ] 1133 for lib_dir in library_folders: 1134 for file_path in file_operations.find_files(lib_dir, [pattern]): 1135 if not file_path.startswith("examples"): 1136 sources.append((Path(lib_dir) / file_path).as_posix()) 1137 return sources 1138 1139 def library_c_files(self): 1140 return self.library_files("**/*.c") 1141 1142 def library_s_files(self): 1143 return self.library_files("**/*.S") 1144 1145 def library_cpp_files(self): 1146 return self.library_files("**/*.cpp") 1147 1148 def get_core_path(self): 1149 return self.board[self.selected_board]["build.core.path"] 1150 1151 def core_files(self, pattern): 1152 sources = [] 1153 for file_path in file_operations.find_files( 1154 self.get_core_path(), [pattern] 1155 ): 1156 sources.append(os.path.join(self.get_core_path(), file_path)) 1157 return sources 1158 1159 def core_c_files(self): 1160 return self.core_files("**/*.c") 1161 1162 def core_s_files(self): 1163 return self.core_files("**/*.S") 1164 1165 def core_cpp_files(self): 1166 return self.core_files("**/*.cpp") 1167 1168 def get_variant_path(self): 1169 return self.build_variant_path 1170 1171 def variant_files(self, pattern): 1172 sources = [] 1173 if self.build_variant_path: 1174 for file_path in file_operations.find_files( 1175 self.get_variant_path(), [pattern] 1176 ): 1177 sources.append(os.path.join(self.get_variant_path(), file_path)) 1178 return sources 1179 1180 def variant_c_files(self): 1181 return self.variant_files("**/*.c") 1182 1183 def variant_s_files(self): 1184 return self.variant_files("**/*.S") 1185 1186 def variant_cpp_files(self): 1187 return self.variant_files("**/*.cpp") 1188 1189 def project_files(self, pattern): 1190 sources = [] 1191 for file_path in file_operations.find_files( 1192 self.project_path, [pattern] 1193 ): 1194 if not file_path.startswith( 1195 "examples" 1196 ) and not file_path.startswith("libraries"): 1197 sources.append(file_path) 1198 return sources 1199 1200 def project_c_files(self): 1201 return self.project_files("**/*.c") 1202 1203 def project_cpp_files(self): 1204 return self.project_files("**/*.cpp") 1205 1206 def project_ino_files(self): 1207 return self.project_files("**/*.ino") 1208