1# Copyright (C) 2024 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15upstream_git_repo = 'https://gitlab.com/kernel-firmware/linux-firmware.git' 16 17# ########################################################## 18# Parsing of linux-firmware WHENCE file 19# ########################################################## 20 21section_divider = '--------------------------------------------------------------------------' 22 23# Handle paths that might be quoted or use backslashes to escape spaces. 24# 25# For some reason some of the paths listed in WHENCE seem to be quoted or 26# use '\ ' to escape spaces in the files, even though this doesn't seem like 27# it should be necessary. Handle it. This takes a path `p`. If it's surrounded 28# by quotes it just strips the quotes off. If it isn't quoted but we see 29# '\ ' we'll transform that to just simple spaces. 30def unquote_path(p): 31 if p.startswith('"') and p.endswith('"'): 32 p = p[1:-1] 33 else: 34 p = p.replace('\\ ', ' ') 35 36 return p 37 38# Parse WHENCE from the upstream repository and return dict w/ info about files. 39# 40# This will read the upstream whence and return a dictionary keyed by files 41# referenced in the upstream WHENCE (anything tagged 'File:', 'RawFile:', or 42# 'Link:'). Values in the dictionary will be another dict that looks like: 43# { 44# 'driver': Name of the driver line associated with this file, 45# 'kind': The kind of file ('File', 'RawFile', or 'Link') 46# 'license': If this key is present it's the text of the license; if this 47# key is not present then the license is unknown. 48# 'link': Only present for 'kind' == 'Link'. This is where the link 49# should point to. 50# } 51def parse_whence(whence_text): 52 file_info_map = {} 53 54 driver = 'UNKNOWN' 55 unlicensed_files = [] 56 license_text = None 57 58 for line in whence_text.splitlines() + [section_divider]: 59 # Take out trailing spaces / carriage returns 60 line = line.rstrip() 61 62 # Look for tags, which are lines that look like: 63 # tag: other stuff 64 # 65 # Tags always need to start the line and can't have any spaces in 66 # them, which helps ID them as tags. 67 # 68 # Note that normally we require a space after the colon which keeps 69 # us from getting confused when we're parsing license text that 70 # has a URL. We special-case allow lines to end with a colon 71 # since some tags (like "License:") are sometimes multiline tags 72 # and the first line might just be blank. 73 if ': ' in line or line.endswith(':'): 74 tag, _, rest = line.partition(': ') 75 tag = tag.rstrip(':') 76 if ' ' in tag or '\t' in tag: 77 tag = None 78 else: 79 rest = rest.lstrip() 80 else: 81 tag = None 82 83 # Any new tag or a full separator ends a license. 84 if line == section_divider or (tag and license_text): 85 if license_text: 86 for f in unlicensed_files: 87 # Clear out blank lines at the start and end 88 for i, text in enumerate(license_text): 89 if text: 90 break 91 for j, text in reversed(list(enumerate(license_text))): 92 if text: 93 break 94 license_text = license_text[i:j+1] 95 96 file_info_map[f]['license'] = '\n'.join(license_text) 97 unlicensed_files = [] 98 license_text = None 99 100 if line == section_divider: 101 driver = 'UNKNOWN' 102 103 if tag == 'Driver': 104 driver = rest 105 elif tag in ['File', 'RawFile', 'Link']: 106 if tag == 'Link': 107 rest, _, link_dest = rest.partition(' -> ') 108 rest = rest.rstrip() 109 link_dest = unquote_path(link_dest.lstrip()) 110 111 rest = unquote_path(rest) 112 113 file_info_map[rest] = { 'driver': driver, 'kind': tag } 114 if tag == 'Link': 115 file_info_map[rest]['link'] = link_dest 116 117 unlicensed_files.append(rest) 118 elif tag in ['License', 'Licence']: 119 license_text = [rest] 120 elif license_text: 121 license_text.append(line) 122 123 return file_info_map 124 125# Look at the license and see if it references other files. 126# 127# Many of the licenses in WHENCE refer to other files in the same directory. 128# This will detect those and return a list of indirectly referenced files. 129def find_indirect_files(license_text): 130 license_files = [] 131 132 # Our regex match works better if there are no carriage returns, so 133 # split everything else and join with spaces. All we care about is 134 # detecting indirectly referenced files anyway. 135 license_text_oneline = ' '.join(license_text.splitlines()) 136 137 # The only phrasing that appears present right now refer to one or two 138 # other files and looks like: 139 # See <filename> for details 140 # See <filename1> and <filename2> for details 141 # 142 # Detect those two. More can be added later. 143 pattern = re2.compile(r'.*[Ss]ee (.*) for details.*') 144 matcher = pattern.matcher(license_text_oneline) 145 if matcher.matches(): 146 for i in range(matcher.group_count()): 147 license_files.extend(matcher.group(i + 1).split(' and ')) 148 149 return license_files 150 151# ########################################################## 152# Templates for generated files 153# ########################################################## 154 155# NOTES: 156# - Right now license_type is always BY_EXCEPTION_ONLY. If this is 157# ever not right we can always add a lookup table by license_kind. 158metadata_template = \ 159'''name: "linux-firmware-{name}" 160description: 161 "Contains an import of upstream linux-firmware for {name}." 162 163third_party {{ 164 homepage: "{upstream_git_repo}" 165 identifier {{ 166 type: "Git" 167 value: "{upstream_git_repo}" 168 primary_source: true 169 version: "{version}" 170 }} 171 version: "{version}" 172 last_upgrade_date {{ year: {year} month: {month} day: {day} }} 173 license_type: BY_EXCEPTION_ONLY 174}} 175''' 176 177# Automatically create the METADATA file under the directory `name`. 178def create_metadata_file(ctx, name): 179 output = metadata_template.format( 180 name = name, 181 year = ctx.now_as_string('yyyy'), 182 month = ctx.now_as_string('M'), 183 day = ctx.now_as_string('d'), 184 version = ctx.fill_template('${GIT_SHA1}'), 185 upstream_git_repo = upstream_git_repo, 186 ) 187 188 ctx.write_path(ctx.new_path(name + '/METADATA'), output) 189 190android_bp_template = \ 191'''// Copyright (C) {year} The Android Open Source Project 192// 193// Licensed under the Apache License, Version 2.0 (the "License"); 194// you may not use this file except in compliance with the License. 195// You may obtain a copy of the License at 196// 197// http://www.apache.org/licenses/LICENSE-2.0 198// 199// Unless required by applicable law or agreed to in writing, software 200// distributed under the License is distributed on an "AS IS" BASIS, 201// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 202// See the License for the specific language governing permissions and 203// limitations under the License. 204// 205// This file is autogenerated by copybara, please do not edit. 206 207license {{ 208 name: "linux_firmware_{name}_license", 209 visibility: ["//visibility:private"], 210 license_kinds: [{license_kinds}], 211 license_text: [{license_files}], 212}}{symlink_stanzas} 213 214prebuilt_firmware {{ 215 name: "linux_firmware_{name}", 216 // In general linux-firmware gets pulled in by a product adding it 217 // to PRODUCT_PACKAGES, so we don't need visibility to anything outside 218 // of linux-firmware. We allow visibility between all the linux-firmware 219 // sub-packages so that one linux-firmare can use `required` for other 220 // linux-firmware. 221 visibility: ["//external/linux-firmware:__subpackages__"], 222 licenses: ["linux_firmware_{name}_license"], 223 srcs: [{fw_files}], 224 dsts: [{fw_files}], 225 vendor: true, 226 required: [{required}], 227}} 228''' 229 230# Format the an array of strings to go into Android.bp 231def format_android_bp_string_arr(a): 232 if len(a) == 0: 233 return "" 234 if len(a) == 1: 235 return '"%s"' % a[0] 236 237 indent_per = ' ' 238 indent = indent_per * 2 239 return '\n' + indent + \ 240 (',\n' + indent).join(['"%s"' % s for s in a]) + \ 241 ',\n' + indent_per 242 243# Automatically create the Android.bp file under the directory `name`. 244def create_android_bp_file(ctx, name, license_kind, fw_files, required, symlink_stanzas): 245 output = android_bp_template.format( 246 name = name, 247 license_kinds = format_android_bp_string_arr([license_kind]), 248 license_files = format_android_bp_string_arr(['LICENSE']), 249 fw_files = format_android_bp_string_arr(fw_files), 250 required = format_android_bp_string_arr(required), 251 symlink_stanzas = '\n\n'.join(symlink_stanzas), 252 year = ctx.now_as_string('yyyy'), 253 ) 254 ctx.write_path(ctx.new_path(name + '/Android.bp'), output) 255 256# Create the LICENSE file containing all relevant license text. 257def create_license_file(ctx, name, license_text, license_files, fw_files): 258 license_header_strs = [] 259 license_header_strs.append("For the files:") 260 for f in fw_files: 261 license_header_strs.append("- %s" % f) 262 license_header_strs.append("\nThe license is as follows:\n") 263 license_header_strs.append(license_text) 264 265 # Even though the indrectly referenced files are copied to the directory 266 # too, policy says to copy them into the main LICENSE file for easy 267 # reference. 268 license_strs = ['\n'.join(license_header_strs)] 269 for f in license_files: 270 license_strs.append("The text of %s is:\n\n%s" % 271 (f, ctx.read_path(ctx.new_path(name + '/' + f)))) 272 273 ctx.write_path(ctx.new_path(name + '/LICENSE'), 274 '\n\n---\n\n'.join(license_strs)) 275 276commit_message_template = \ 277'''IMPORT: {name} 278 279Import linux-firmware "{name}" using copybara. 280 281NOTE: linux-firmware is not like most android3p reviews. See 282go/android-linux-firmware for some details. 283 284Third-Party Import of: {upstream_git_repo} 285Request Document: go/android3p 286For CL Reviewers: go/android-linux-firmware:reviewers 287Bug: {{REPLACE WITH motivation bug without b: prefix}} 288Bug: {{REPLACE WITH bug filed by go/android3p process without b: prefix}} 289Test: None 290''' 291 292# Automatically set the commit message. 293def set_commit_message(ctx, name): 294 output = commit_message_template.format( 295 name = name, 296 upstream_git_repo = upstream_git_repo, 297 ) 298 ctx.set_message(output) 299 300symlink_template = \ 301'''install_symlink {{ 302 name: "{module_name}", 303 visibility: ["//visibility:private"], 304 symlink_target: "{target}", 305 installed_location: "firmware/{link}", 306 vendor: true, 307}}''' 308 309# ########################################################## 310# Main transformation 311# ########################################################## 312 313def _firmware_import_transform(ctx): 314 name = ctx.params['name'] 315 license_kind = ctx.params['license_kind'] 316 expected_license_files = ctx.params['license_files'] 317 fw_files = ctx.params['fw_files'] 318 required = list(ctx.params['required']) 319 symlink_stanzas = [''] 320 non_symlink_fw_files = [] 321 322 # We will read the WHENCE to validate that the license upstream lists for 323 # the fw_files matches the license we think we have. 324 whence_text = ctx.read_path(ctx.new_path('WHENCE')) 325 file_info_map = parse_whence(whence_text) 326 327 # To be a valid import then every fw_file we're importing must have the 328 # same license. Validate that. 329 license_text = file_info_map[fw_files[0]].get('license', '') 330 if not license_text: 331 ctx.console.error('Missing license for "%s"' % fw_files[0]) 332 333 bad_licenses = [f for f in fw_files 334 if file_info_map[f].get('license', '') != license_text] 335 if bad_licenses: 336 ctx.console.error( 337 ('All files in a module must have a matching license. ' + 338 'The license(s) for "%s" don\'t match the license for "%s".') % 339 (', '.join(bad_licenses), fw_files[0]) 340 ) 341 342 for f in expected_license_files: 343 ctx.run(core.move(f, name + '/' + f)) 344 345 for f in fw_files: 346 if file_info_map[f]['kind'] == 'Link': 347 symlink_module_name = "linux_firmware_%s-symlink-%d" % (name, len(symlink_stanzas)) 348 symlink_stanzas.append(symlink_template.format( 349 module_name = symlink_module_name, 350 link = f, 351 target = file_info_map[f]['link'], 352 )) 353 required.append(symlink_module_name) 354 else: 355 ctx.run(core.move(f, name + '/' + f)) 356 non_symlink_fw_files.append(f) 357 358 # Look for indirectly referenced license files since we'll need those too. 359 license_files = find_indirect_files(license_text) 360 361 # copybara required us to specify all the origin files. To make this work 362 # firmware_import_workflow() requires the callers to provide the list of 363 # indirectly referenced license files. Those should match what we detected. 364 if tuple(sorted(license_files)) != tuple(sorted(expected_license_files)): 365 ctx.console.error( 366 ('Upstream WHENCE specified that licenses were %r but we expected %r.') % 367 (license_files, expected_license_files) 368 ) 369 370 create_license_file(ctx, name, license_text, license_files, fw_files) 371 create_metadata_file(ctx, name) 372 create_android_bp_file(ctx, name, license_kind, non_symlink_fw_files, 373 required, symlink_stanzas) 374 set_commit_message(ctx, name) 375 376 # We don't actually want 'WHENCE' in the destination but we need to read it 377 # so we need to list it in the input files. We can't core.remove() since 378 # that yells at us. Just replace it with some placeholder text. 379 ctx.write_path(ctx.new_path('WHENCE'), 380 'Upstream WHENCE is parsed to confirm licenses; not copied here.\n') 381 382# Create a workflow for the given files. 383def firmware_import_workflow(name, license_kind, license_files, fw_files, required=[]): 384 return core.workflow( 385 name = name, 386 authoring = authoring.overwrite('linux-firmware importer <noreply@google.com>'), 387 388 origin = git.origin( 389 url = upstream_git_repo, 390 ref = 'main', 391 ), 392 393 # The below is just a placeholder and will be overridden by command 394 # line arguments passed by `run_copybara.sh`. The script is used 395 # because our copybara flow is different than others. Our flow is: 396 # 1. Add the new firmware import to copy.bara.sky and create a 397 # 'ANDROID:' CL for this. 398 # 2. Run copybara locally which creates an 'IMPORT:' CL. Validate 399 # that it looks OK. 400 # 3. Upload both CLs in a chain and get review. 401 destination = git.destination( 402 url = 'please-use-run_copybara.sh', 403 push = 'please-use-run_copybara.sh', 404 ), 405 406 origin_files = glob( 407 include = license_files + fw_files + ['WHENCE'], 408 ), 409 destination_files = glob( 410 include = [name + '/**'] + ['WHENCE'], 411 ), 412 mode = 'SQUASH', 413 transformations = [ 414 core.dynamic_transform( 415 impl = _firmware_import_transform, 416 params = { 417 'name': name, 418 'license_kind': license_kind, 419 'license_files': license_files, 420 'fw_files': fw_files, 421 'required': required, 422 }, 423 ), 424 ] 425 ) 426 427# ########################################################## 428# Firmware that we manage, sorted alphabetically 429# ########################################################## 430 431# Intel AX201 Bluetooth chip 432firmware_import_workflow( 433 name = 'btusb-ibt_ax201', 434 license_kind = 'BSD-Binary-Only', 435 license_files = [ 436 'LICENCE.ibt_firmware', 437 ], 438 fw_files = [ 439 'intel/ibt-19-0-0.ddc', 440 'intel/ibt-19-0-0.sfi', 441 'intel/ibt-19-0-1.ddc', 442 'intel/ibt-19-0-1.sfi', 443 'intel/ibt-19-0-4.ddc', 444 'intel/ibt-19-0-4.sfi', 445 'intel/ibt-19-16-4.ddc', 446 'intel/ibt-19-16-4.sfi', 447 'intel/ibt-19-32-0.ddc', 448 'intel/ibt-19-32-0.sfi', 449 'intel/ibt-19-32-1.ddc', 450 'intel/ibt-19-32-1.sfi', 451 'intel/ibt-19-32-4.ddc', 452 'intel/ibt-19-32-4.sfi', 453 'intel/ibt-19-240-1.ddc', 454 'intel/ibt-19-240-1.sfi', 455 'intel/ibt-19-240-4.ddc', 456 'intel/ibt-19-240-4.sfi', 457 ], 458) 459 460# Realtek RTL8822C Bluetooth chipset 461firmware_import_workflow( 462 name = 'btusb-r8822cu', 463 license_kind = 'BSD-Binary-Only', 464 license_files = [ 465 'LICENCE.rtlwifi_firmware.txt', 466 ], 467 fw_files = [ 468 'rtl_bt/rtl8822cu_fw.bin', 469 'rtl_bt/rtl8822cu_config.bin', 470 ], 471 required = [ 472 # One of our files is a symlink to this one so depend on it. 473 'linux_firmware_btusb-rtl8761bu_config', 474 ], 475) 476 477# Realtek RTL8761BU Bluetooth Config 478# Separated out since several similar BT chipsets symlink to it. 479firmware_import_workflow( 480 name = 'btusb-rtl8761bu_config', 481 license_kind = 'BSD-Binary-Only', 482 license_files = [ 483 'LICENCE.rtlwifi_firmware.txt', 484 ], 485 fw_files = [ 486 'rtl_bt/rtl8761bu_config.bin', 487 ], 488) 489 490# Intel AX201 Wireless MAC (API Revision 0x77) 491firmware_import_workflow( 492 name = 'iwlwifi-QuZ-a0-hr-b0-77', 493 license_kind = 'BSD-Binary-Only', 494 license_files = [ 495 'LICENCE.iwlwifi_firmware', 496 ], 497 fw_files = [ 498 'iwlwifi-QuZ-a0-hr-b0-77.ucode', 499 ] 500) 501 502# MediaTek MT7921 Wireless MACs 503# 504# NOTE: though the filename uses 7961, the description in upstream WHENCE 505# and ChromeOS call this 7921. 506firmware_import_workflow( 507 name = 'mt7921', 508 license_kind = 'legacy_by_exception_only', 509 license_files = [ 510 'LICENCE.mediatek', 511 ], 512 fw_files = [ 513 'mediatek/WIFI_MT7961_patch_mcu_1_2_hdr.bin', 514 'mediatek/WIFI_RAM_CODE_MT7961_1.bin', 515 ], 516) 517 518# MediaTek MT7921 bluetooth chipset 519# 520# NOTE: though the filename uses 7961, the description in upstream WHENCE 521# and ChromeOS call this 7921. 522firmware_import_workflow( 523 name = 'mt7921-bt', 524 license_kind = 'legacy_by_exception_only', 525 license_files = [ 526 'LICENCE.mediatek', 527 ], 528 fw_files = [ 529 'mediatek/BT_RAM_CODE_MT7961_1_2_hdr.bin', 530 ], 531) 532 533# MediaTek MT7922 Wireless MACs 534firmware_import_workflow( 535 name = 'mt7922', 536 license_kind = 'legacy_by_exception_only', 537 license_files = [ 538 'LICENCE.mediatek', 539 ], 540 fw_files = [ 541 'mediatek/WIFI_MT7922_patch_mcu_1_1_hdr.bin', 542 'mediatek/WIFI_RAM_CODE_MT7922_1.bin', 543 ], 544) 545 546# MediaTek MT7922 bluetooth chipset 547firmware_import_workflow( 548 name = 'mt7922-bt', 549 license_kind = 'legacy_by_exception_only', 550 license_files = [ 551 'LICENCE.mediatek', 552 ], 553 fw_files = [ 554 'mediatek/BT_RAM_CODE_MT7922_1_1_hdr.bin', 555 ], 556) 557 558# Realtek r8152 (and related) USB Ethernet adapters 559firmware_import_workflow( 560 name = 'r8152', 561 license_kind = 'BSD-Binary-Only', 562 license_files = [ 563 'LICENCE.rtlwifi_firmware.txt', 564 ], 565 fw_files = [ 566 'rtl_nic/rtl8153a-2.fw', 567 'rtl_nic/rtl8153a-3.fw', 568 'rtl_nic/rtl8153a-4.fw', 569 'rtl_nic/rtl8153b-2.fw', 570 'rtl_nic/rtl8153c-1.fw', 571 'rtl_nic/rtl8156a-2.fw', 572 'rtl_nic/rtl8156b-2.fw', 573 ], 574) 575 576# Ralink RT2800 (and related) USB wireless MACs 577firmware_import_workflow( 578 name = 'rt2800usb', 579 license_kind = 'BSD-Binary-Only', 580 license_files = [ 581 'LICENCE.ralink-firmware.txt', 582 ], 583 fw_files = [ 584 'rt2870.bin', 585 ], 586) 587 588# Realtek RTL8822C Wireless MAC 589firmware_import_workflow( 590 name = 'rtw88-rtw8822c', 591 license_kind = 'BSD-Binary-Only', 592 license_files = [ 593 'LICENCE.rtlwifi_firmware.txt', 594 ], 595 fw_files = [ 596 'rtw88/rtw8822c_fw.bin', 597 'rtw88/rtw8822c_wow_fw.bin', 598 ], 599) 600