1#!/usr/bin/env python3 2# Copyright 2020 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Extracts build information from Arduino cores.""" 16 17import glob 18import logging 19import os 20import platform 21import pprint 22import re 23import sys 24import time 25from collections import OrderedDict 26from pathlib import Path 27from typing import List 28 29from pw_arduino_build import file_operations 30 31_LOG = logging.getLogger(__name__) 32 33_pretty_print = pprint.PrettyPrinter(indent=1, width=120).pprint 34_pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat 35 36 37def arduino_runtime_os_string(): 38 arduno_platform = { 39 "Linux": "linux", 40 "Windows": "windows", 41 "Darwin": "macosx" 42 } 43 return arduno_platform[platform.system()] 44 45 46class ArduinoBuilder: 47 """Used to interpret arduino boards.txt and platform.txt files.""" 48 # pylint: disable=too-many-instance-attributes,too-many-public-methods 49 50 BOARD_MENU_REGEX = re.compile( 51 r"^(?P<name>menu\.[^#=]+)=(?P<description>.*)$", re.MULTILINE) 52 53 BOARD_NAME_REGEX = re.compile( 54 r"^(?P<name>[^\s#\.]+)\.name=(?P<description>.*)$", re.MULTILINE) 55 56 VARIABLE_REGEX = re.compile(r"^(?P<name>[^\s#=]+)=(?P<value>.*)$", 57 re.MULTILINE) 58 59 MENU_OPTION_REGEX = re.compile( 60 r"^menu\." # starts with "menu" 61 r"(?P<menu_option_name>[^.]+)\." # first token after . 62 r"(?P<menu_option_value>[^.]+)$") # second (final) token after . 63 64 TOOL_NAME_REGEX = re.compile( 65 r"^tools\." # starts with "tools" 66 r"(?P<tool_name>[^.]+)\.") # first token after . 67 68 INTERPOLATED_VARIABLE_REGEX = re.compile(r"{[^}]+}", re.MULTILINE) 69 70 OBJCOPY_STEP_NAME_REGEX = re.compile(r"^recipe.objcopy.([^.]+).pattern$") 71 72 def __init__(self, 73 arduino_path, 74 package_name, 75 build_path=None, 76 project_path=None, 77 project_source_path=None, 78 library_path=None, 79 library_names=None, 80 build_project_name=None, 81 compiler_path_override=False): 82 self.arduino_path = arduino_path 83 self.arduino_package_name = package_name 84 self.selected_board = None 85 self.build_path = build_path 86 self.project_path = project_path 87 self.project_source_path = project_source_path 88 self.build_project_name = build_project_name 89 self.compiler_path_override = compiler_path_override 90 self.variant_includes = "" 91 self.build_variant_path = False 92 self.library_names = library_names 93 self.library_path = library_path 94 95 self.compiler_path_override_binaries = [] 96 if self.compiler_path_override: 97 self.compiler_path_override_binaries = file_operations.find_files( 98 self.compiler_path_override, "*") 99 100 # Container dicts for boards.txt and platform.txt file data. 101 self.board = OrderedDict() 102 self.platform = OrderedDict() 103 self.menu_options = OrderedDict({ 104 "global_options": {}, 105 "default_board_values": {}, 106 "selected": {} 107 }) 108 self.tools_variables = {} 109 110 # Set and check for valid hardware folder. 111 self.hardware_path = os.path.join(self.arduino_path, "hardware") 112 113 if not os.path.exists(self.hardware_path): 114 raise FileNotFoundError( 115 "Arduino package path '{}' does not exist.".format( 116 self.arduino_path)) 117 118 # Set and check for valid package name 119 self.package_path = os.path.join(self.arduino_path, "hardware", 120 package_name) 121 # {build.arch} is the first folder name of the package (upcased) 122 self.build_arch = os.path.split(package_name)[0].upper() 123 124 if not os.path.exists(self.package_path): 125 _LOG.error("Error: Arduino package name '%s' does not exist.", 126 package_name) 127 _LOG.error("Did you mean:\n") 128 # TODO(tonymd): On Windows concatenating "/" may not work 129 possible_alternatives = [ 130 d.replace(self.hardware_path + os.sep, "", 1) 131 for d in glob.glob(self.hardware_path + "/*/*") 132 ] 133 _LOG.error("\n".join(possible_alternatives)) 134 sys.exit(1) 135 136 # Populate library paths. 137 if not library_path: 138 self.library_path = [] 139 # Append core libraries directory. 140 core_lib_path = Path(self.package_path) / "libraries" 141 if core_lib_path.is_dir(): 142 self.library_path.append(Path(self.package_path) / "libraries") 143 if library_path: 144 self.library_path = [ 145 os.path.realpath(os.path.expanduser( 146 os.path.expandvars(l_path))) for l_path in library_path 147 ] 148 149 # Grab all folder names in the cores directory. These are typically 150 # sub-core source files. 151 self.sub_core_folders = os.listdir( 152 os.path.join(self.package_path, "cores")) 153 154 self._find_tools_variables() 155 156 self.boards_txt = os.path.join(self.package_path, "boards.txt") 157 self.platform_txt = os.path.join(self.package_path, "platform.txt") 158 159 def select_board(self, board_name, menu_option_overrides=False): 160 self.selected_board = board_name 161 162 # Load default menu options for a selected board. 163 if not self.selected_board in self.board.keys(): 164 _LOG.error("Error board: '%s' not supported.", self.selected_board) 165 # TODO(tonymd): Print supported boards here 166 sys.exit(1) 167 168 # Override default menu options if any are specified. 169 if menu_option_overrides: 170 for moption in menu_option_overrides: 171 if not self.set_menu_option(moption): 172 # TODO(tonymd): Print supported menu options here 173 sys.exit(1) 174 175 self._copy_default_menu_options_to_build_variables() 176 self._apply_recipe_overrides() 177 self._substitute_variables() 178 179 def set_variables(self, variable_list: List[str]): 180 # Convert the string list containing 'name=value' items into a dict 181 variable_source = {} 182 for var in variable_list: 183 var_name, value = var.split("=") 184 variable_source[var_name] = value 185 186 # Replace variables in platform 187 for var, value in self.platform.items(): 188 self.platform[var] = self._replace_variables( 189 value, variable_source) 190 191 def _apply_recipe_overrides(self): 192 # Override link recipes with per-core exceptions 193 # Teensyduino cores 194 if self.build_arch == "TEENSY": 195 # Change {build.path}/{archive_file} 196 # To {archive_file_path} (which should contain the core.a file) 197 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 198 "{object_files} \"{build.path}/{archive_file}\"", 199 "{object_files} {archive_file_path}", 1) 200 # Add the teensy provided toolchain lib folder for link access to 201 # libarm_cortexM*_math.a 202 new_link_line = new_link_line.replace( 203 "\"-L{build.path}\"", 204 "\"-L{build.path}\" -L{compiler.path}/arm/arm-none-eabi/lib", 205 1) 206 self.platform["recipe.c.combine.pattern"] = new_link_line 207 # Remove the pre-compiled header include 208 self.platform["recipe.cpp.o.pattern"] = self.platform[ 209 "recipe.cpp.o.pattern"].replace("\"-I{build.path}/pch\"", "", 210 1) 211 212 # Adafruit-samd core 213 # TODO(tonymd): This build_arch may clash with Arduino-SAMD core 214 elif self.build_arch == "SAMD": 215 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 216 "\"{build.path}/{archive_file}\" -Wl,--end-group", 217 "{archive_file_path} -Wl,--end-group", 1) 218 self.platform["recipe.c.combine.pattern"] = new_link_line 219 220 # STM32L4 Core: 221 # https://github.com/GrumpyOldPizza/arduino-STM32L4 222 elif self.build_arch == "STM32L4": 223 # TODO(tonymd): {build.path}/{archive_file} for the link step always 224 # seems to be core.a (except STM32 core) 225 line_to_delete = "-Wl,--start-group \"{build.path}/{archive_file}\"" 226 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 227 line_to_delete, "-Wl,--start-group {archive_file_path}", 1) 228 self.platform["recipe.c.combine.pattern"] = new_link_line 229 230 # stm32duino core 231 elif self.build_arch == "STM32": 232 # Must link in SrcWrapper for all projects. 233 if not self.library_names: 234 self.library_names = [] 235 self.library_names.append("SrcWrapper") 236 237 def _copy_default_menu_options_to_build_variables(self): 238 # Clear existing options 239 self.menu_options["selected"] = {} 240 # Set default menu options for selected board 241 for menu_key, menu_dict in self.menu_options["default_board_values"][ 242 self.selected_board].items(): 243 for name, var in self.board[self.selected_board].items(): 244 starting_key = "{}.{}.".format(menu_key, menu_dict["name"]) 245 if name.startswith(starting_key): 246 new_var_name = name.replace(starting_key, "", 1) 247 self.menu_options["selected"][new_var_name] = var 248 249 def set_menu_option(self, moption): 250 if moption not in self.board[self.selected_board]: 251 _LOG.error("Error: '%s' is not a valid menu option.", moption) 252 return False 253 254 # Override default menu option with new value. 255 menu_match_result = self.MENU_OPTION_REGEX.match(moption) 256 if menu_match_result: 257 menu_match = menu_match_result.groupdict() 258 menu_value = menu_match["menu_option_value"] 259 menu_key = "menu.{}".format(menu_match["menu_option_name"]) 260 self.menu_options["default_board_values"][ 261 self.selected_board][menu_key]["name"] = menu_value 262 263 # Update build variables 264 self._copy_default_menu_options_to_build_variables() 265 return True 266 267 def _set_global_arduino_variables(self): 268 """Set some global variables defined by the Arduino-IDE. 269 270 See Docs: 271 https://arduino.github.io/arduino-cli/platform-specification/#global-predefined-properties 272 """ 273 274 # TODO(tonymd): Make sure these variables are replaced in recipe lines 275 # even if they are None: build_path, project_path, project_source_path, 276 # build_project_name 277 for current_board in self.board.values(): 278 if self.build_path: 279 current_board["build.path"] = self.build_path 280 if self.build_project_name: 281 current_board["build.project_name"] = self.build_project_name 282 # {archive_file} is the final *.elf 283 archive_file = "{}.elf".format(self.build_project_name) 284 current_board["archive_file"] = archive_file 285 # {archive_file_path} is the final core.a archive 286 if self.build_path: 287 current_board["archive_file_path"] = os.path.join( 288 self.build_path, "core.a") 289 if self.project_source_path: 290 current_board["build.source.path"] = self.project_source_path 291 292 current_board["extra.time.local"] = str(int(time.time())) 293 current_board["runtime.ide.version"] = "10812" 294 current_board["runtime.hardware.path"] = self.hardware_path 295 296 # Copy {runtime.tools.TOOL_NAME.path} vars 297 self._set_tools_variables(current_board) 298 299 current_board["runtime.platform.path"] = self.package_path 300 if self.platform["name"] == "Teensyduino": 301 # Teensyduino is installed into the arduino IDE folder 302 # rather than ~/.arduino15/packages/ 303 current_board["runtime.hardware.path"] = os.path.join( 304 self.hardware_path, "teensy") 305 306 current_board["build.system.path"] = os.path.join( 307 self.package_path, "system") 308 309 # Set the {build.core.path} variable that pointing to a sub-core 310 # folder. For Teensys this is: 311 # 'teensy/hardware/teensy/avr/cores/teensy{3,4}'. For other cores 312 # it's typically just the 'arduino' folder. For example: 313 # 'arduino-samd/hardware/samd/1.8.8/cores/arduino' 314 core_path = Path(self.package_path) / "cores" 315 core_path /= current_board.get("build.core", 316 self.sub_core_folders[0]) 317 current_board["build.core.path"] = core_path.as_posix() 318 319 current_board["build.arch"] = self.build_arch 320 321 for name, var in current_board.items(): 322 current_board[name] = var.replace("{build.core.path}", 323 core_path.as_posix()) 324 325 def load_board_definitions(self): 326 """Loads Arduino boards.txt and platform.txt files into dictionaries. 327 328 Populates the following dictionaries: 329 self.menu_options 330 self.boards 331 self.platform 332 """ 333 # Load platform.txt 334 with open(self.platform_txt, "r") as pfile: 335 platform_file = pfile.read() 336 platform_var_matches = self.VARIABLE_REGEX.finditer(platform_file) 337 for p_match in [m.groupdict() for m in platform_var_matches]: 338 self.platform[p_match["name"]] = p_match["value"] 339 340 # Load boards.txt 341 with open(self.boards_txt, "r") as bfile: 342 board_file = bfile.read() 343 # Get all top-level menu options, e.g. menu.usb=USB Type 344 board_menu_matches = self.BOARD_MENU_REGEX.finditer(board_file) 345 for menuitem in [m.groupdict() for m in board_menu_matches]: 346 self.menu_options["global_options"][menuitem["name"]] = { 347 "description": menuitem["description"] 348 } 349 350 # Get all board names, e.g. teensy40.name=Teensy 4.0 351 board_name_matches = self.BOARD_NAME_REGEX.finditer(board_file) 352 for b_match in [m.groupdict() for m in board_name_matches]: 353 self.board[b_match["name"]] = OrderedDict() 354 self.menu_options["default_board_values"][ 355 b_match["name"]] = OrderedDict() 356 357 # Get all board variables, e.g. teensy40.* 358 for current_board_name, current_board in self.board.items(): 359 board_line_matches = re.finditer( 360 fr"^\s*{current_board_name}\." 361 fr"(?P<key>[^#=]+)=(?P<value>.*)$", board_file, 362 re.MULTILINE) 363 for b_match in [m.groupdict() for m in board_line_matches]: 364 # Check if this line is a menu option 365 # (e.g. 'menu.usb.serial') and save as default if it's the 366 # first one seen. 367 ArduinoBuilder.save_default_menu_option( 368 current_board_name, b_match["key"], b_match["value"], 369 self.menu_options) 370 current_board[b_match["key"]] = b_match["value"].strip() 371 372 self._set_global_arduino_variables() 373 374 @staticmethod 375 def save_default_menu_option(current_board_name, key, value, menu_options): 376 """Save a given menu option as the default. 377 378 Saves the key and value into menu_options["default_board_values"] 379 if it doesn't already exist. Assumes menu options are added in the order 380 specified in boards.txt. The first value for a menu key is the default. 381 """ 382 # Check if key is a menu option 383 # e.g. menu.usb.serial 384 # menu.usb.serial.build.usbtype 385 menu_match_result = re.match( 386 r'^menu\.' # starts with "menu" 387 r'(?P<menu_option_name>[^.]+)\.' # first token after . 388 r'(?P<menu_option_value>[^.]+)' # second token after . 389 r'(\.(?P<rest>.+))?', # optionally any trailing tokens after a . 390 key) 391 if menu_match_result: 392 menu_match = menu_match_result.groupdict() 393 current_menu_key = "menu.{}".format(menu_match["menu_option_name"]) 394 # If this is the first menu option seen for current_board_name, save 395 # as the default. 396 if current_menu_key not in menu_options["default_board_values"][ 397 current_board_name]: 398 menu_options["default_board_values"][current_board_name][ 399 current_menu_key] = { 400 "name": menu_match["menu_option_value"], 401 "description": value 402 } 403 404 def _replace_variables(self, line, variable_lookup_source): 405 """Replace {variables} from loaded boards.txt or platform.txt. 406 407 Replace interpolated variables surrounded by curly braces in line with 408 definitions from variable_lookup_source. 409 """ 410 new_line = line 411 for current_var_match in self.INTERPOLATED_VARIABLE_REGEX.findall( 412 line): 413 # {build.flags.c} --> build.flags.c 414 current_var = current_var_match.strip("{}") 415 416 # check for matches in board definition 417 if current_var in variable_lookup_source: 418 replacement = variable_lookup_source.get(current_var, "") 419 new_line = new_line.replace(current_var_match, replacement) 420 return new_line 421 422 def _find_tools_variables(self): 423 # Gather tool directories in order of increasing precedence 424 runtime_tool_paths = [] 425 426 # Check for tools installed in ~/.arduino15/packages/arduino/tools/ 427 # TODO(tonymd): Is this Mac & Linux specific? 428 runtime_tool_paths += glob.glob( 429 os.path.join( 430 os.path.realpath(os.path.expanduser(os.path.expandvars("~"))), 431 ".arduino15", "packages", "arduino", "tools", "*")) 432 433 # <ARDUINO_PATH>/tools/<OS_STRING>/<TOOL_NAMES> 434 runtime_tool_paths += glob.glob( 435 os.path.join(self.arduino_path, "tools", 436 arduino_runtime_os_string(), "*")) 437 # <ARDUINO_PATH>/tools/<TOOL_NAMES> 438 # This will grab linux/windows/macosx/share as <TOOL_NAMES>. 439 runtime_tool_paths += glob.glob( 440 os.path.join(self.arduino_path, "tools", "*")) 441 442 # Process package tools after arduino tools. 443 # They should overwrite vars & take precedence. 444 445 # <PACKAGE_PATH>/tools/<OS_STRING>/<TOOL_NAMES> 446 runtime_tool_paths += glob.glob( 447 os.path.join(self.package_path, "tools", 448 arduino_runtime_os_string(), "*")) 449 # <PACKAGE_PATH>/tools/<TOOL_NAMES> 450 # This will grab linux/windows/macosx/share as <TOOL_NAMES>. 451 runtime_tool_paths += glob.glob( 452 os.path.join(self.package_path, "tools", "*")) 453 454 for path in runtime_tool_paths: 455 # Make sure TOOL_NAME is not an OS string 456 if not (path.endswith("linux") or path.endswith("windows") 457 or path.endswith("macosx") or path.endswith("share")): 458 # TODO(tonymd): Check if a file & do nothing? 459 460 # Check if it's a directory with subdir(s) as a version string 461 # create all 'runtime.tools.{tool_folder}-{version.path}' 462 # (for each version) 463 # create 'runtime.tools.{tool_folder}.path' 464 # (with latest version) 465 if os.path.isdir(path): 466 # Grab the tool name (folder) by itself. 467 tool_folder = os.path.basename(path) 468 # Sort so that [-1] is the latest version. 469 version_paths = sorted(glob.glob(os.path.join(path, "*"))) 470 # Check if all sub folders start with a version string. 471 if len(version_paths) == sum( 472 bool(re.match(r"^[0-9.]+", os.path.basename(vp))) 473 for vp in version_paths): 474 for version_path in version_paths: 475 version_string = os.path.basename(version_path) 476 var_name = "runtime.tools.{}-{}.path".format( 477 tool_folder, version_string) 478 self.tools_variables[var_name] = os.path.join( 479 path, version_string) 480 var_name = "runtime.tools.{}.path".format(tool_folder) 481 self.tools_variables[var_name] = os.path.join( 482 path, os.path.basename(version_paths[-1])) 483 # Else set toolpath to path. 484 else: 485 var_name = "runtime.tools.{}.path".format(tool_folder) 486 self.tools_variables[var_name] = path 487 488 _LOG.debug("TOOL VARIABLES: %s", _pretty_format(self.tools_variables)) 489 490 # Copy self.tools_variables into destination 491 def _set_tools_variables(self, destination): 492 for key, value in self.tools_variables.items(): 493 destination[key] = value 494 495 def _substitute_variables(self): 496 """Perform variable substitution in board and platform metadata.""" 497 498 # menu -> board 499 # Copy selected menu variables into board definiton 500 for name, value in self.menu_options["selected"].items(): 501 self.board[self.selected_board][name] = value 502 503 # board -> board 504 # Replace any {vars} in the selected board with values defined within 505 # (and from copied in menu options). 506 for var, value in self.board[self.selected_board].items(): 507 self.board[self.selected_board][var] = self._replace_variables( 508 value, self.board[self.selected_board]) 509 510 # Check for build.variant variable 511 # This will be set in selected board after menu options substitution 512 build_variant = self.board[self.selected_board].get( 513 "build.variant", None) 514 if build_variant: 515 # Set build.variant.path 516 bvp = os.path.join(self.package_path, "variants", build_variant) 517 self.build_variant_path = bvp 518 self.board[self.selected_board]["build.variant.path"] = bvp 519 # Add the variant folder as an include directory 520 # (used in stm32l4 core) 521 self.variant_includes = "-I{}".format(bvp) 522 523 _LOG.debug("PLATFORM INITIAL: %s", _pretty_format(self.platform)) 524 525 # board -> platform 526 # Replace {vars} in platform from the selected board definition 527 for var, value in self.platform.items(): 528 self.platform[var] = self._replace_variables( 529 value, self.board[self.selected_board]) 530 531 # platform -> platform 532 # Replace any remaining {vars} in platform from platform 533 for var, value in self.platform.items(): 534 self.platform[var] = self._replace_variables(value, self.platform) 535 536 # Repeat platform -> platform for any lingering variables 537 # Example: {build.opt.name} in STM32 core 538 for var, value in self.platform.items(): 539 self.platform[var] = self._replace_variables(value, self.platform) 540 541 _LOG.debug("MENU_OPTIONS: %s", _pretty_format(self.menu_options)) 542 _LOG.debug("SELECTED_BOARD: %s", 543 _pretty_format(self.board[self.selected_board])) 544 _LOG.debug("PLATFORM: %s", _pretty_format(self.platform)) 545 546 def selected_board_spec(self): 547 return self.board[self.selected_board] 548 549 def get_menu_options(self): 550 all_options = [] 551 max_string_length = [0, 0] 552 553 for key_name, description in self.board[self.selected_board].items(): 554 menu_match_result = self.MENU_OPTION_REGEX.match(key_name) 555 if menu_match_result: 556 menu_match = menu_match_result.groupdict() 557 name = "menu.{}.{}".format(menu_match["menu_option_name"], 558 menu_match["menu_option_value"]) 559 if len(name) > max_string_length[0]: 560 max_string_length[0] = len(name) 561 if len(description) > max_string_length[1]: 562 max_string_length[1] = len(description) 563 all_options.append((name, description)) 564 565 return all_options, max_string_length 566 567 def get_default_menu_options(self): 568 default_options = [] 569 max_string_length = [0, 0] 570 571 for key_name, value in self.menu_options["default_board_values"][ 572 self.selected_board].items(): 573 full_key = key_name + "." + value["name"] 574 if len(full_key) > max_string_length[0]: 575 max_string_length[0] = len(full_key) 576 if len(value["description"]) > max_string_length[1]: 577 max_string_length[1] = len(value["description"]) 578 default_options.append((full_key, value["description"])) 579 580 return default_options, max_string_length 581 582 @staticmethod 583 def split_binary_from_arguments(compile_line): 584 compile_binary = None 585 rest_of_line = compile_line 586 587 compile_binary_match = re.search(r'^("[^"]+") ', compile_line) 588 if compile_binary_match: 589 compile_binary = compile_binary_match[1] 590 rest_of_line = compile_line.replace(compile_binary_match[0], "", 1) 591 592 return compile_binary, rest_of_line 593 594 def _strip_includes_source_file_object_file_vars(self, compile_line): 595 line = compile_line 596 if self.variant_includes: 597 line = compile_line.replace( 598 "{includes} \"{source_file}\" -o \"{object_file}\"", 599 self.variant_includes, 1) 600 else: 601 line = compile_line.replace( 602 "{includes} \"{source_file}\" -o \"{object_file}\"", "", 1) 603 return line 604 605 def _get_tool_name(self, line): 606 tool_match_result = self.TOOL_NAME_REGEX.match(line) 607 if tool_match_result: 608 return tool_match_result[1] 609 return False 610 611 def get_upload_tool_names(self): 612 return [ 613 self._get_tool_name(t) for t in self.platform.keys() 614 if self.TOOL_NAME_REGEX.match(t) and 'upload.pattern' in t 615 ] 616 617 # TODO(tonymd): Use these getters in _replace_variables() or 618 # _substitute_variables() 619 620 def _get_platform_variable(self, variable): 621 # TODO(tonymd): Check for '.macos' '.linux' '.windows' in variable name, 622 # compare with platform.system() and return that instead. 623 return self.platform.get(variable, False) 624 625 def _get_platform_variable_with_substitutions(self, variable, namespace): 626 line = self.platform.get(variable, False) 627 # Get all unique variables used in this line in line. 628 unique_vars = sorted( 629 set(self.INTERPOLATED_VARIABLE_REGEX.findall(line))) 630 # Search for each unique_vars in namespace and global. 631 for var in unique_vars: 632 v_raw_name = var.strip("{}") 633 634 # Check for namespace.variable 635 # eg: 'tools.stm32CubeProg.cmd' 636 possible_var_name = "{}.{}".format(namespace, v_raw_name) 637 result = self._get_platform_variable(possible_var_name) 638 # Check for os overriden variable 639 # eg: 640 # ('tools.stm32CubeProg.cmd', 'stm32CubeProg.sh'), 641 # ('tools.stm32CubeProg.cmd.windows', 'stm32CubeProg.bat'), 642 possible_var_name = "{}.{}.{}".format(namespace, v_raw_name, 643 arduino_runtime_os_string()) 644 os_override_result = self._get_platform_variable(possible_var_name) 645 646 if os_override_result: 647 line = line.replace(var, os_override_result) 648 elif result: 649 line = line.replace(var, result) 650 # Check for variable at top level? 651 # elif self._get_platform_variable(v_raw_name): 652 # line = line.replace(self._get_platform_variable(v_raw_name), 653 # result) 654 return line 655 656 def get_upload_line(self, tool_name, serial_port=False): 657 # TODO(tonymd): Error if tool_name does not exist 658 tool_namespace = "tools.{}".format(tool_name) 659 pattern = "tools.{}.upload.pattern".format(tool_name) 660 661 if not self._get_platform_variable(pattern): 662 _LOG.error("Error: upload tool '%s' does not exist.", tool_name) 663 tools = self.get_upload_tool_names() 664 _LOG.error("Valid tools: %s", ", ".join(tools)) 665 return sys.exit(1) 666 667 line = self._get_platform_variable_with_substitutions( 668 pattern, tool_namespace) 669 670 # TODO(tonymd): Teensy specific tool overrides. 671 if tool_name == "teensyloader": 672 # Remove un-necessary lines 673 # {serial.port.label} and {serial.port.protocol} are returned by 674 # the teensy_ports binary. 675 line = line.replace("\"-portlabel={serial.port.label}\"", "", 1) 676 line = line.replace("\"-portprotocol={serial.port.protocol}\"", "", 677 1) 678 679 if serial_port == "UNKNOWN" or not serial_port: 680 line = line.replace('"-port={serial.port}"', "", 1) 681 else: 682 line = line.replace("{serial.port}", serial_port, 1) 683 684 return line 685 686 def _get_binary_path(self, variable_pattern): 687 compile_line = self.replace_compile_binary_with_override_path( 688 self._get_platform_variable(variable_pattern)) 689 compile_binary, _ = ArduinoBuilder.split_binary_from_arguments( 690 compile_line) 691 return compile_binary 692 693 def get_cc_binary(self): 694 return self._get_binary_path("recipe.c.o.pattern") 695 696 def get_cxx_binary(self): 697 return self._get_binary_path("recipe.cpp.o.pattern") 698 699 def get_objcopy_binary(self): 700 objcopy_step_name = self.get_objcopy_step_names()[0] 701 objcopy_binary = self._get_binary_path(objcopy_step_name) 702 return objcopy_binary 703 704 def get_ar_binary(self): 705 return self._get_binary_path("recipe.ar.pattern") 706 707 def get_size_binary(self): 708 return self._get_binary_path("recipe.size.pattern") 709 710 def replace_command_args_with_compiler_override_path(self, compile_line): 711 if not self.compiler_path_override: 712 return compile_line 713 replacement_line = compile_line 714 replacement_line_args = compile_line.split() 715 for arg in replacement_line_args: 716 compile_binary_basename = os.path.basename(arg.strip("\"")) 717 if compile_binary_basename in self.compiler_path_override_binaries: 718 new_compiler = os.path.join(self.compiler_path_override, 719 compile_binary_basename) 720 replacement_line = replacement_line.replace( 721 arg, new_compiler, 1) 722 return replacement_line 723 724 def replace_compile_binary_with_override_path(self, compile_line): 725 replacement_compile_line = compile_line 726 727 # Change the compiler path if there's an override path set 728 if self.compiler_path_override: 729 compile_binary, line = ArduinoBuilder.split_binary_from_arguments( 730 compile_line) 731 compile_binary_basename = os.path.basename( 732 compile_binary.strip("\"")) 733 new_compiler = os.path.join(self.compiler_path_override, 734 compile_binary_basename) 735 if platform.system() == "Windows" and not re.match( 736 r".*\.exe$", new_compiler, flags=re.IGNORECASE): 737 new_compiler += ".exe" 738 739 if os.path.isfile(new_compiler): 740 replacement_compile_line = "\"{}\" {}".format( 741 new_compiler, line) 742 743 return replacement_compile_line 744 745 def get_c_compile_line(self): 746 _LOG.debug("ARDUINO_C_COMPILE: %s", 747 _pretty_format(self.platform["recipe.c.o.pattern"])) 748 749 compile_line = self.platform["recipe.c.o.pattern"] 750 compile_line = self._strip_includes_source_file_object_file_vars( 751 compile_line) 752 compile_line += " -I{}".format( 753 self.board[self.selected_board]["build.core.path"]) 754 755 compile_line = self.replace_compile_binary_with_override_path( 756 compile_line) 757 return compile_line 758 759 def get_s_compile_line(self): 760 _LOG.debug("ARDUINO_S_COMPILE %s", 761 _pretty_format(self.platform["recipe.S.o.pattern"])) 762 763 compile_line = self.platform["recipe.S.o.pattern"] 764 compile_line = self._strip_includes_source_file_object_file_vars( 765 compile_line) 766 compile_line += " -I{}".format( 767 self.board[self.selected_board]["build.core.path"]) 768 769 compile_line = self.replace_compile_binary_with_override_path( 770 compile_line) 771 return compile_line 772 773 def get_ar_compile_line(self): 774 _LOG.debug("ARDUINO_AR_COMPILE: %s", 775 _pretty_format(self.platform["recipe.ar.pattern"])) 776 777 compile_line = self.platform["recipe.ar.pattern"].replace( 778 "\"{object_file}\"", "", 1) 779 780 compile_line = self.replace_compile_binary_with_override_path( 781 compile_line) 782 return compile_line 783 784 def get_cpp_compile_line(self): 785 _LOG.debug("ARDUINO_CPP_COMPILE: %s", 786 _pretty_format(self.platform["recipe.cpp.o.pattern"])) 787 788 compile_line = self.platform["recipe.cpp.o.pattern"] 789 compile_line = self._strip_includes_source_file_object_file_vars( 790 compile_line) 791 compile_line += " -I{}".format( 792 self.board[self.selected_board]["build.core.path"]) 793 794 compile_line = self.replace_compile_binary_with_override_path( 795 compile_line) 796 return compile_line 797 798 def get_link_line(self): 799 _LOG.debug("ARDUINO_LINK: %s", 800 _pretty_format(self.platform["recipe.c.combine.pattern"])) 801 802 compile_line = self.platform["recipe.c.combine.pattern"] 803 804 compile_line = self.replace_compile_binary_with_override_path( 805 compile_line) 806 return compile_line 807 808 def get_objcopy_step_names(self): 809 names = [ 810 name for name, line in self.platform.items() 811 if self.OBJCOPY_STEP_NAME_REGEX.match(name) 812 ] 813 return names 814 815 def get_objcopy_steps(self) -> List[str]: 816 lines = [ 817 line for name, line in self.platform.items() 818 if self.OBJCOPY_STEP_NAME_REGEX.match(name) 819 ] 820 lines = [ 821 self.replace_compile_binary_with_override_path(line) 822 for line in lines 823 ] 824 return lines 825 826 # TODO(tonymd): These recipes are probably run in sorted order 827 def get_objcopy(self, suffix): 828 # Expected vars: 829 # teensy: 830 # recipe.objcopy.eep.pattern 831 # recipe.objcopy.hex.pattern 832 833 pattern = "recipe.objcopy.{}.pattern".format(suffix) 834 objcopy_step_names = self.get_objcopy_step_names() 835 836 objcopy_suffixes = [ 837 m[1] for m in [ 838 self.OBJCOPY_STEP_NAME_REGEX.match(line) 839 for line in objcopy_step_names 840 ] if m 841 ] 842 if pattern not in objcopy_step_names: 843 _LOG.error("Error: objcopy suffix '%s' does not exist.", suffix) 844 _LOG.error("Valid suffixes: %s", ", ".join(objcopy_suffixes)) 845 return sys.exit(1) 846 847 line = self._get_platform_variable(pattern) 848 849 _LOG.debug("ARDUINO_OBJCOPY_%s: %s", suffix, line) 850 851 line = self.replace_compile_binary_with_override_path(line) 852 853 return line 854 855 def get_objcopy_flags(self, suffix): 856 # TODO(tonymd): Possibly teensy specific variables. 857 flags = "" 858 if suffix == "hex": 859 flags = self.platform.get("compiler.elf2hex.flags", "") 860 elif suffix == "bin": 861 flags = self.platform.get("compiler.elf2bin.flags", "") 862 elif suffix == "eep": 863 flags = self.platform.get("compiler.objcopy.eep.flags", "") 864 return flags 865 866 # TODO(tonymd): There are more recipe hooks besides postbuild. 867 # They are run in sorted order. 868 # TODO(tonymd): Rename this to get_hooks(hook_name, step). 869 # TODO(tonymd): Add a list-hooks and or run-hooks command 870 def get_postbuild_line(self, step_number): 871 line = self.platform["recipe.hooks.postbuild.{}.pattern".format( 872 step_number)] 873 line = self.replace_command_args_with_compiler_override_path(line) 874 return line 875 876 def get_prebuild_steps(self) -> List[str]: 877 # Teensy core uses recipe.hooks.sketch.prebuild.1.pattern 878 # stm32 core uses recipe.hooks.prebuild.1.pattern 879 # TODO(tonymd): STM32 core uses recipe.hooks.prebuild.1.pattern.windows 880 # (should override non-windows key) 881 lines = [ 882 line for name, line in self.platform.items() if re.match( 883 r"^recipe.hooks.(?:sketch.)?prebuild.[^.]+.pattern$", name) 884 ] 885 # TODO(tonymd): Write a function to fetch/replace OS specific patterns 886 # (ending in an OS string) 887 lines = [ 888 self.replace_compile_binary_with_override_path(line) 889 for line in lines 890 ] 891 return lines 892 893 def get_postbuild_steps(self) -> List[str]: 894 lines = [ 895 line for name, line in self.platform.items() 896 if re.match(r"^recipe.hooks.postbuild.[^.]+.pattern$", name) 897 ] 898 899 lines = [ 900 self.replace_command_args_with_compiler_override_path(line) 901 for line in lines 902 ] 903 return lines 904 905 def get_s_flags(self): 906 compile_line = self.get_s_compile_line() 907 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 908 compile_line) 909 compile_line = compile_line.replace("-c", "", 1) 910 return compile_line.strip() 911 912 def get_c_flags(self): 913 compile_line = self.get_c_compile_line() 914 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 915 compile_line) 916 compile_line = compile_line.replace("-c", "", 1) 917 return compile_line.strip() 918 919 def get_cpp_flags(self): 920 compile_line = self.get_cpp_compile_line() 921 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 922 compile_line) 923 compile_line = compile_line.replace("-c", "", 1) 924 return compile_line.strip() 925 926 def get_ar_flags(self): 927 compile_line = self.get_ar_compile_line() 928 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 929 compile_line) 930 return compile_line.strip() 931 932 def get_ld_flags(self): 933 compile_line = self.get_link_line() 934 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 935 compile_line) 936 937 # TODO(tonymd): This is teensy specific 938 line_to_delete = "-o \"{build.path}/{build.project_name}.elf\" " \ 939 "{object_files} \"-L{build.path}\"" 940 if self.build_path: 941 line_to_delete = line_to_delete.replace("{build.path}", 942 self.build_path) 943 if self.build_project_name: 944 line_to_delete = line_to_delete.replace("{build.project_name}", 945 self.build_project_name) 946 947 compile_line = compile_line.replace(line_to_delete, "", 1) 948 libs = re.findall(r'(-l[^ ]+ ?)', compile_line) 949 for lib in libs: 950 compile_line = compile_line.replace(lib, "", 1) 951 libs = [lib.strip() for lib in libs] 952 953 return compile_line.strip() 954 955 def get_ld_libs(self, name_only=False): 956 compile_line = self.get_link_line() 957 libs = re.findall(r'(?P<arg>-l(?P<name>[^ ]+) ?)', compile_line) 958 if name_only: 959 libs = [lib_name.strip() for lib_arg, lib_name in libs] 960 else: 961 libs = [lib_arg.strip() for lib_arg, lib_name in libs] 962 return " ".join(libs) 963 964 def library_folders(self): 965 # Arduino library format documentation: 966 # https://arduino.github.io/arduino-cli/library-specification/#layout-of-folders-and-files 967 # - If src folder exists, 968 # use that as the root include directory -Ilibraries/libname/src 969 # - Else lib folder as root include -Ilibraries/libname 970 # (exclude source files in the examples folder in this case) 971 972 if not self.library_names or not self.library_path: 973 return [] 974 975 folder_patterns = ["*"] 976 if self.library_names: 977 folder_patterns = self.library_names 978 979 library_folders = OrderedDict() 980 for library_dir in self.library_path: 981 found_library_names = file_operations.find_files( 982 library_dir, folder_patterns, directories_only=True) 983 _LOG.debug("Found Libraries %s: %s", library_dir, 984 found_library_names) 985 for lib_name in found_library_names: 986 lib_dir = os.path.join(library_dir, lib_name) 987 src_dir = os.path.join(lib_dir, "src") 988 if os.path.exists(src_dir) and os.path.isdir(src_dir): 989 library_folders[lib_name] = src_dir 990 else: 991 library_folders[lib_name] = lib_dir 992 993 return list(library_folders.values()) 994 995 def library_include_dirs(self): 996 return [Path(lib).as_posix() for lib in self.library_folders()] 997 998 def library_includes(self): 999 include_args = [] 1000 library_folders = self.library_folders() 1001 for lib_dir in library_folders: 1002 include_args.append("-I{}".format(os.path.relpath(lib_dir))) 1003 return include_args 1004 1005 def library_files(self, pattern, only_library_name=None): 1006 sources = [] 1007 library_folders = self.library_folders() 1008 if only_library_name: 1009 library_folders = [ 1010 lf for lf in self.library_folders() if only_library_name in lf 1011 ] 1012 for lib_dir in library_folders: 1013 for file_path in file_operations.find_files(lib_dir, [pattern]): 1014 if not file_path.startswith("examples"): 1015 sources.append((Path(lib_dir) / file_path).as_posix()) 1016 return sources 1017 1018 def library_c_files(self): 1019 return self.library_files("**/*.c") 1020 1021 def library_s_files(self): 1022 return self.library_files("**/*.S") 1023 1024 def library_cpp_files(self): 1025 return self.library_files("**/*.cpp") 1026 1027 def get_core_path(self): 1028 return self.board[self.selected_board]["build.core.path"] 1029 1030 def core_files(self, pattern): 1031 sources = [] 1032 for file_path in file_operations.find_files(self.get_core_path(), 1033 [pattern]): 1034 sources.append(os.path.join(self.get_core_path(), file_path)) 1035 return sources 1036 1037 def core_c_files(self): 1038 return self.core_files("**/*.c") 1039 1040 def core_s_files(self): 1041 return self.core_files("**/*.S") 1042 1043 def core_cpp_files(self): 1044 return self.core_files("**/*.cpp") 1045 1046 def get_variant_path(self): 1047 return self.build_variant_path 1048 1049 def variant_files(self, pattern): 1050 sources = [] 1051 if self.build_variant_path: 1052 for file_path in file_operations.find_files( 1053 self.get_variant_path(), [pattern]): 1054 sources.append(os.path.join(self.get_variant_path(), 1055 file_path)) 1056 return sources 1057 1058 def variant_c_files(self): 1059 return self.variant_files("**/*.c") 1060 1061 def variant_s_files(self): 1062 return self.variant_files("**/*.S") 1063 1064 def variant_cpp_files(self): 1065 return self.variant_files("**/*.cpp") 1066 1067 def project_files(self, pattern): 1068 sources = [] 1069 for file_path in file_operations.find_files(self.project_path, 1070 [pattern]): 1071 if not file_path.startswith( 1072 "examples") and not file_path.startswith("libraries"): 1073 sources.append(file_path) 1074 return sources 1075 1076 def project_c_files(self): 1077 return self.project_files("**/*.c") 1078 1079 def project_cpp_files(self): 1080 return self.project_files("**/*.cpp") 1081 1082 def project_ino_files(self): 1083 return self.project_files("**/*.ino") 1084