1# !/usr/bin/env python3 2# 3# Copyright (C) 2024 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Generate the SBOM of the current target product in SPDX format. 19Usage example: 20 gen_sbom.py --output_file out/soong/sbom/aosp_cf_x86_64_phone/sbom.spdx \ 21 --metadata out/soong/metadata/aosp_cf_x86_64_phone/metadata.db \ 22 --product_out out/target/vsoc_x86_64 23 --soong_out out/soong 24 --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \ 25 --product_mfr=Google 26""" 27 28import argparse 29import compliance_metadata 30import datetime 31import google.protobuf.text_format as text_format 32import hashlib 33import os 34import pathlib 35import queue 36import metadata_file_pb2 37import sbom_data 38import sbom_writers 39 40# Package type 41PKG_SOURCE = 'SOURCE' 42PKG_UPSTREAM = 'UPSTREAM' 43PKG_PREBUILT = 'PREBUILT' 44 45# Security tag 46NVD_CPE23 = 'NVD-CPE2.3:' 47 48# Report 49ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:' 50ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:' 51ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:' 52ISSUE_UNKNOWN_SECURITY_TAG_TYPE = 'Unknown security tag type:' 53ISSUE_INSTALLED_FILE_NOT_EXIST = 'Non-existent installed files:' 54ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP = 'No module found for static dependency files:' 55INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:' 56 57SOONG_PREBUILT_MODULE_TYPES = [ 58 'android_app_import', 59 'android_library_import', 60 'cc_prebuilt_binary', 61 'cc_prebuilt_library', 62 'cc_prebuilt_library_headers', 63 'cc_prebuilt_library_shared', 64 'cc_prebuilt_library_static', 65 'cc_prebuilt_object', 66 'dex_import', 67 'java_import', 68 'java_sdk_library_import', 69 'java_system_modules_import', 70 'libclang_rt_prebuilt_library_static', 71 'libclang_rt_prebuilt_library_shared', 72 'llvm_prebuilt_library_static', 73 'ndk_prebuilt_object', 74 'ndk_prebuilt_shared_stl', 75 'nkd_prebuilt_static_stl', 76 'prebuilt_apex', 77 'prebuilt_bootclasspath_fragment', 78 'prebuilt_dsp', 79 'prebuilt_firmware', 80 'prebuilt_kernel_modules', 81 'prebuilt_rfsa', 82 'prebuilt_root', 83 'rust_prebuilt_dylib', 84 'rust_prebuilt_library', 85 'rust_prebuilt_rlib', 86 'vndk_prebuilt_shared', 87] 88 89THIRD_PARTY_IDENTIFIER_TYPES = [ 90 # Types defined in metadata_file.proto 91 'Git', 92 'SVN', 93 'Hg', 94 'Darcs', 95 'Piper', 96 'VCS', 97 'Archive', 98 'PrebuiltByAlphabet', 99 'LocalSource', 100 'Other', 101 # OSV ecosystems defined at https://ossf.github.io/osv-schema/#affectedpackage-field. 102 'Go', 103 'npm', 104 'OSS-Fuzz', 105 'PyPI', 106 'RubyGems', 107 'crates.io', 108 'Hackage', 109 'GHC', 110 'Packagist', 111 'Maven', 112 'NuGet', 113 'Linux', 114 'Debian', 115 'Alpine', 116 'Hex', 117 'Android', 118 'GitHub Actions', 119 'Pub', 120 'ConanCenter', 121 'Rocky Linux', 122 'AlmaLinux', 123 'Bitnami', 124 'Photon OS', 125 'CRAN', 126 'Bioconductor', 127 'SwiftURL' 128] 129 130 131def get_args(): 132 parser = argparse.ArgumentParser() 133 parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.') 134 parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode') 135 parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.') 136 parser.add_argument('--metadata', required=True, help='The metadata DB file path.') 137 parser.add_argument('--product_out', required=True, help='The path of PRODUCT_OUT, e.g. out/target/product/vsoc_x86_64.') 138 parser.add_argument('--soong_out', required=True, help='The path of Soong output directory, e.g. out/soong') 139 parser.add_argument('--build_version', required=True, help='The build version.') 140 parser.add_argument('--product_mfr', required=True, help='The product manufacturer.') 141 parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format') 142 143 return parser.parse_args() 144 145 146def log(*info): 147 if args.verbose: 148 for i in info: 149 print(i) 150 151 152def new_package_id(package_name, type): 153 return f'SPDXRef-{type}-{sbom_data.encode_for_spdxid(package_name)}' 154 155 156def new_file_id(file_path): 157 return f'SPDXRef-{sbom_data.encode_for_spdxid(file_path)}' 158 159 160def new_license_id(license_name): 161 return f'LicenseRef-{sbom_data.encode_for_spdxid(license_name)}' 162 163 164def checksum(file_path): 165 h = hashlib.sha1() 166 if os.path.islink(file_path): 167 h.update(os.readlink(file_path).encode('utf-8')) 168 else: 169 with open(file_path, 'rb') as f: 170 h.update(f.read()) 171 return f'SHA1: {h.hexdigest()}' 172 173 174def is_soong_prebuilt_module(file_metadata): 175 return (file_metadata['soong_module_type'] and 176 file_metadata['soong_module_type'] in SOONG_PREBUILT_MODULE_TYPES) 177 178 179def is_source_package(file_metadata): 180 module_path = file_metadata['module_path'] 181 return module_path.startswith('external/') and not is_prebuilt_package(file_metadata) 182 183 184def is_prebuilt_package(file_metadata): 185 module_path = file_metadata['module_path'] 186 if module_path: 187 return (module_path.startswith('prebuilts/') or 188 is_soong_prebuilt_module(file_metadata) or 189 file_metadata['is_prebuilt_make_module']) 190 191 kernel_module_copy_files = file_metadata['kernel_module_copy_files'] 192 if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'): 193 return True 194 195 return False 196 197 198def get_source_package_info(file_metadata, metadata_file_path): 199 """Return source package info exists in its METADATA file, currently including name, security tag 200 and external SBOM reference. 201 202 See go/android-spdx and go/android-sbom-gen for more details. 203 """ 204 if not metadata_file_path: 205 return file_metadata['module_path'], [] 206 207 metadata_proto = metadata_file_protos[metadata_file_path] 208 external_refs = [] 209 for tag in metadata_proto.third_party.security.tag: 210 if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()): 211 external_refs.append( 212 sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY, 213 type=sbom_data.PackageExternalRefType.cpe23Type, 214 locator=tag.removeprefix(NVD_CPE23))) 215 elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()): 216 external_refs.append( 217 sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY, 218 type=sbom_data.PackageExternalRefType.cpe22Type, 219 locator=tag.removeprefix(NVD_CPE23))) 220 221 if metadata_proto.name: 222 return metadata_proto.name, external_refs 223 else: 224 return os.path.basename(metadata_file_path), external_refs # return the directory name only as package name 225 226 227def get_prebuilt_package_name(file_metadata, metadata_file_path): 228 """Return name of a prebuilt package, which can be from the METADATA file, metadata file path, 229 module path or kernel module's source path if the installed file is a kernel module. 230 231 See go/android-spdx and go/android-sbom-gen for more details. 232 """ 233 name = None 234 if metadata_file_path: 235 metadata_proto = metadata_file_protos[metadata_file_path] 236 if metadata_proto.name: 237 name = metadata_proto.name 238 else: 239 name = metadata_file_path 240 elif file_metadata['module_path']: 241 name = file_metadata['module_path'] 242 elif file_metadata['kernel_module_copy_files']: 243 src_path = file_metadata['kernel_module_copy_files'].split(':')[0] 244 name = os.path.dirname(src_path) 245 246 return name.removeprefix('prebuilts/').replace('/', '-') 247 248 249def get_metadata_file_path(file_metadata): 250 """Search for METADATA file of a package and return its path.""" 251 metadata_path = '' 252 if file_metadata['module_path']: 253 metadata_path = file_metadata['module_path'] 254 elif file_metadata['kernel_module_copy_files']: 255 metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0]) 256 257 while metadata_path and not os.path.exists(metadata_path + '/METADATA'): 258 metadata_path = os.path.dirname(metadata_path) 259 260 return metadata_path 261 262 263def get_package_version(metadata_file_path): 264 """Return a package's version in its METADATA file.""" 265 if not metadata_file_path: 266 return None 267 metadata_proto = metadata_file_protos[metadata_file_path] 268 return metadata_proto.third_party.version 269 270 271def get_package_homepage(metadata_file_path): 272 """Return a package's homepage URL in its METADATA file.""" 273 if not metadata_file_path: 274 return None 275 metadata_proto = metadata_file_protos[metadata_file_path] 276 if metadata_proto.third_party.homepage: 277 return metadata_proto.third_party.homepage 278 for url in metadata_proto.third_party.url: 279 if url.type == metadata_file_pb2.URL.Type.HOMEPAGE: 280 return url.value 281 282 return None 283 284 285def get_package_download_location(metadata_file_path): 286 """Return a package's code repository URL in its METADATA file.""" 287 if not metadata_file_path: 288 return None 289 metadata_proto = metadata_file_protos[metadata_file_path] 290 if metadata_proto.third_party.url: 291 urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type) 292 if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE: 293 return urls[0].value 294 elif len(urls) > 1: 295 return urls[1].value 296 297 return None 298 299 300def get_license_text(license_files): 301 license_text = '' 302 for license_file in license_files: 303 if args.debug: 304 license_text += '#### Content from ' + license_file + '\n' 305 else: 306 license_text += pathlib.Path(license_file).read_text(errors='replace') + '\n\n' 307 return license_text 308 309 310def get_sbom_fragments(installed_file_metadata, metadata_file_path): 311 """Return SPDX fragment of source/prebuilt packages, which usually contains a SOURCE/PREBUILT 312 package, a UPSTREAM package and an external SBOM document reference if sbom_ref defined in its 313 METADATA file. 314 315 See go/android-spdx and go/android-sbom-gen for more details. 316 """ 317 external_doc_ref = None 318 packages = [] 319 relationships = [] 320 licenses = [] 321 322 # Info from METADATA file 323 homepage = get_package_homepage(metadata_file_path) 324 version = get_package_version(metadata_file_path) 325 download_location = get_package_download_location(metadata_file_path) 326 327 lics = db.get_package_licenses(installed_file_metadata['module_path']) 328 if not lics: 329 lics = db.get_package_licenses(metadata_file_path) 330 331 if lics: 332 for license_name, license_files in lics.items(): 333 if not license_files: 334 continue 335 license_id = new_license_id(license_name) 336 if license_name not in licenses_text: 337 licenses_text[license_name] = get_license_text(license_files.split(' ')) 338 licenses.append(sbom_data.License(id=license_id, name=license_name, text=licenses_text[license_name])) 339 340 if is_source_package(installed_file_metadata): 341 # Source fork packages 342 name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path) 343 source_package_id = new_package_id(name, PKG_SOURCE) 344 source_package = sbom_data.Package(id=source_package_id, name=name, version=args.build_version, 345 download_location=sbom_data.VALUE_NONE, 346 supplier='Organization: ' + args.product_mfr, 347 external_refs=external_refs) 348 349 upstream_package_id = new_package_id(name, PKG_UPSTREAM) 350 upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version, 351 supplier=( 352 'Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION, 353 download_location=download_location) 354 packages += [source_package, upstream_package] 355 relationships.append(sbom_data.Relationship(id1=source_package_id, 356 relationship=sbom_data.RelationshipType.VARIANT_OF, 357 id2=upstream_package_id)) 358 359 for license in licenses: 360 source_package.declared_license_ids.append(license.id) 361 upstream_package.declared_license_ids.append(license.id) 362 363 elif is_prebuilt_package(installed_file_metadata): 364 # Prebuilt fork packages 365 name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path) 366 prebuilt_package_id = new_package_id(name, PKG_PREBUILT) 367 prebuilt_package = sbom_data.Package(id=prebuilt_package_id, 368 name=name, 369 download_location=sbom_data.VALUE_NONE, 370 version=version if version else args.build_version, 371 supplier='Organization: ' + args.product_mfr) 372 373 upstream_package_id = new_package_id(name, PKG_UPSTREAM) 374 upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version, 375 supplier=( 376 'Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION, 377 download_location=download_location) 378 packages += [prebuilt_package, upstream_package] 379 relationships.append(sbom_data.Relationship(id1=prebuilt_package_id, 380 relationship=sbom_data.RelationshipType.VARIANT_OF, 381 id2=upstream_package_id)) 382 for license in licenses: 383 prebuilt_package.declared_license_ids.append(license.id) 384 upstream_package.declared_license_ids.append(license.id) 385 386 if metadata_file_path: 387 metadata_proto = metadata_file_protos[metadata_file_path] 388 if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref': 389 sbom_url = metadata_proto.third_party.sbom_ref.url 390 sbom_checksum = metadata_proto.third_party.sbom_ref.checksum 391 upstream_element_id = metadata_proto.third_party.sbom_ref.element_id 392 if sbom_url and sbom_checksum and upstream_element_id: 393 doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{sbom_data.encode_for_spdxid(name)}' 394 external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id, 395 uri=sbom_url, 396 checksum=sbom_checksum) 397 relationships.append( 398 sbom_data.Relationship(id1=upstream_package_id, 399 relationship=sbom_data.RelationshipType.VARIANT_OF, 400 id2=doc_ref_id + ':' + upstream_element_id)) 401 402 return external_doc_ref, packages, relationships, licenses 403 404 405def save_report(report_file_path, report): 406 with open(report_file_path, 'w', encoding='utf-8') as report_file: 407 for type, issues in report.items(): 408 report_file.write(type + '\n') 409 for issue in issues: 410 report_file.write('\t' + issue + '\n') 411 report_file.write('\n') 412 413 414# Validate the metadata generated by Make for installed files and report if there is no metadata. 415def installed_file_has_metadata(installed_file_metadata, report): 416 installed_file = installed_file_metadata['installed_file'] 417 module_path = installed_file_metadata['module_path'] 418 is_soong_module = installed_file_metadata['is_soong_module'] 419 product_copy_files = installed_file_metadata['product_copy_files'] 420 kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] 421 is_platform_generated = installed_file_metadata['is_platform_generated'] 422 423 if (not module_path and 424 not is_soong_module and 425 not product_copy_files and 426 not kernel_module_copy_files and 427 not is_platform_generated and 428 not installed_file.endswith('.fsv_meta')): 429 report[ISSUE_NO_METADATA].append(installed_file) 430 return False 431 432 return True 433 434 435# Validate identifiers in a package's METADATA. 436# 1) Only known identifier type is allowed 437# 2) Only one identifier's primary_source can be true 438def validate_package_metadata(metadata_file_path, package_metadata): 439 primary_source_found = False 440 for identifier in package_metadata.third_party.identifier: 441 if identifier.type not in THIRD_PARTY_IDENTIFIER_TYPES: 442 sys.exit(f'Unknown value of third_party.identifier.type in {metadata_file_path}/METADATA: {identifier.type}.') 443 if primary_source_found and identifier.primary_source: 444 sys.exit( 445 f'Field "primary_source" is set to true in multiple third_party.identifier in {metadata_file_path}/METADATA.') 446 primary_source_found = identifier.primary_source 447 448 449def report_metadata_file(metadata_file_path, installed_file_metadata, report): 450 if metadata_file_path: 451 report[INFO_METADATA_FOUND_FOR_PACKAGE].append( 452 'installed_file: {}, module_path: {}, METADATA file: {}'.format( 453 installed_file_metadata['installed_file'], 454 installed_file_metadata['module_path'], 455 metadata_file_path + '/METADATA')) 456 457 package_metadata = metadata_file_pb2.Metadata() 458 with open(metadata_file_path + '/METADATA', 'rt') as f: 459 text_format.Parse(f.read(), package_metadata) 460 461 validate_package_metadata(metadata_file_path, package_metadata) 462 463 if not metadata_file_path in metadata_file_protos: 464 metadata_file_protos[metadata_file_path] = package_metadata 465 if not package_metadata.name: 466 report[ISSUE_METADATA_FILE_INCOMPLETE].append(f'{metadata_file_path}/METADATA does not has "name"') 467 468 if not package_metadata.third_party.version: 469 report[ISSUE_METADATA_FILE_INCOMPLETE].append( 470 f'{metadata_file_path}/METADATA does not has "third_party.version"') 471 472 for tag in package_metadata.third_party.security.tag: 473 if not tag.startswith(NVD_CPE23): 474 report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append( 475 f'Unknown security tag type: {tag} in {metadata_file_path}/METADATA') 476 else: 477 report[ISSUE_NO_METADATA_FILE].append( 478 "installed_file: {}, module_path: {}".format( 479 installed_file_metadata['installed_file'], installed_file_metadata['module_path'])) 480 481 482# If a file is from a source fork or prebuilt fork package, add its package information to SBOM 483def add_package_of_file(file_id, file_metadata, doc, report): 484 metadata_file_path = get_metadata_file_path(file_metadata) 485 report_metadata_file(metadata_file_path, file_metadata, report) 486 487 external_doc_ref, pkgs, rels, licenses = get_sbom_fragments(file_metadata, metadata_file_path) 488 if len(pkgs) > 0: 489 if external_doc_ref: 490 doc.add_external_ref(external_doc_ref) 491 for p in pkgs: 492 doc.add_package(p) 493 for rel in rels: 494 doc.add_relationship(rel) 495 fork_package_id = pkgs[0].id # The first package should be the source/prebuilt fork package 496 doc.add_relationship(sbom_data.Relationship(id1=file_id, 497 relationship=sbom_data.RelationshipType.GENERATED_FROM, 498 id2=fork_package_id)) 499 for license in licenses: 500 doc.add_license(license) 501 502 503# Add STATIC_LINK relationship for static dependencies of a file 504def add_static_deps_of_file(file_id, file_metadata, doc): 505 if not file_metadata['static_dep_files'] and not file_metadata['whole_static_dep_files']: 506 return 507 static_dep_files = [] 508 if file_metadata['static_dep_files']: 509 static_dep_files += file_metadata['static_dep_files'].split(' ') 510 if file_metadata['whole_static_dep_files']: 511 static_dep_files += file_metadata['whole_static_dep_files'].split(' ') 512 513 for dep_file in static_dep_files: 514 # Static libs are not shipped on devices, so names are derived from .intermediates paths. 515 doc.add_relationship(sbom_data.Relationship(id1=file_id, 516 relationship=sbom_data.RelationshipType.STATIC_LINK, 517 id2=new_file_id( 518 dep_file.removeprefix(args.soong_out + '/.intermediates/')))) 519 520 521def add_licenses_of_file(file_id, file_metadata, doc): 522 lics = db.get_module_licenses(file_metadata.get('name', ''), file_metadata['module_path']) 523 if lics: 524 file = next(f for f in doc.files if file_id == f.id) 525 for license_name, license_files in lics.items(): 526 if not license_files: 527 continue 528 license_id = new_license_id(license_name) 529 file.concluded_license_ids.append(license_id) 530 if license_name not in licenses_text: 531 license_text = get_license_text(license_files.split(' ')) 532 licenses_text[license_name] = license_text 533 534 doc.add_license(sbom_data.License(id=license_id, name=license_name, text=licenses_text[license_name])) 535 536 537def get_all_transitive_static_dep_files_of_installed_files(installed_files_metadata, db, report): 538 # Find all transitive static dep files of all installed files 539 q = queue.Queue() 540 for installed_file_metadata in installed_files_metadata: 541 if installed_file_metadata['static_dep_files']: 542 for f in installed_file_metadata['static_dep_files'].split(' '): 543 q.put(f) 544 if installed_file_metadata['whole_static_dep_files']: 545 for f in installed_file_metadata['whole_static_dep_files'].split(' '): 546 q.put(f) 547 548 all_static_dep_files = {} 549 while not q.empty(): 550 dep_file = q.get() 551 if dep_file in all_static_dep_files: 552 # It has been processed 553 continue 554 555 all_static_dep_files[dep_file] = True 556 soong_module = db.get_soong_module_of_built_file(dep_file) 557 if not soong_module: 558 # This should not happen, add to report[ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP] 559 report[ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP].append(f) 560 continue 561 562 if soong_module['static_dep_files']: 563 for f in soong_module['static_dep_files'].split(' '): 564 if f not in all_static_dep_files: 565 q.put(f) 566 if soong_module['whole_static_dep_files']: 567 for f in soong_module['whole_static_dep_files'].split(' '): 568 if f not in all_static_dep_files: 569 q.put(f) 570 571 return sorted(all_static_dep_files.keys()) 572 573 574def main(): 575 global args 576 args = get_args() 577 log('Args:', vars(args)) 578 579 global db 580 db = compliance_metadata.MetadataDb(args.metadata) 581 if args.debug: 582 db.dump_debug_db(os.path.dirname(args.output_file) + '/compliance-metadata-debug.db') 583 584 global metadata_file_protos 585 metadata_file_protos = {} 586 global licenses_text 587 licenses_text = {} 588 589 product_package_id = sbom_data.SPDXID_PRODUCT 590 product_package_name = sbom_data.PACKAGE_NAME_PRODUCT 591 product_package = sbom_data.Package(id=product_package_id, 592 name=product_package_name, 593 download_location=sbom_data.VALUE_NONE, 594 version=args.build_version, 595 supplier='Organization: ' + args.product_mfr, 596 files_analyzed=True) 597 doc_name = args.build_version 598 doc = sbom_data.Document(name=doc_name, 599 namespace=f'https://www.google.com/sbom/spdx/android/{doc_name}', 600 creators=['Organization: ' + args.product_mfr], 601 describes=product_package_id) 602 603 doc.packages.append(product_package) 604 doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM, 605 name=sbom_data.PACKAGE_NAME_PLATFORM, 606 download_location=sbom_data.VALUE_NONE, 607 version=args.build_version, 608 supplier='Organization: ' + args.product_mfr, 609 declared_license_ids=[sbom_data.SPDXID_LICENSE_APACHE])) 610 611 # Report on some issues and information 612 report = { 613 ISSUE_NO_METADATA: [], 614 ISSUE_NO_METADATA_FILE: [], 615 ISSUE_METADATA_FILE_INCOMPLETE: [], 616 ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [], 617 ISSUE_INSTALLED_FILE_NOT_EXIST: [], 618 ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP: [], 619 INFO_METADATA_FOUND_FOR_PACKAGE: [], 620 } 621 622 # Get installed files and corresponding make modules' metadata if an installed file is from a make module. 623 installed_files_metadata = db.get_installed_files() 624 625 # Find which Soong module an installed file is from and merge metadata from Make and Soong 626 for installed_file_metadata in installed_files_metadata: 627 soong_module = db.get_soong_module_of_installed_file(installed_file_metadata['installed_file']) 628 if soong_module: 629 # Merge soong metadata to make metadata 630 installed_file_metadata.update(soong_module) 631 else: 632 # For make modules soong_module_type should be empty 633 installed_file_metadata['soong_module_type'] = '' 634 installed_file_metadata['static_dep_files'] = '' 635 installed_file_metadata['whole_static_dep_files'] = '' 636 637 # Scan the metadata and create the corresponding package and file records in SPDX 638 for installed_file_metadata in installed_files_metadata: 639 installed_file = installed_file_metadata['installed_file'] 640 module_path = installed_file_metadata['module_path'] 641 product_copy_files = installed_file_metadata['product_copy_files'] 642 kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files'] 643 build_output_path = installed_file 644 installed_file = installed_file.removeprefix(args.product_out) 645 646 if not installed_file_has_metadata(installed_file_metadata, report): 647 continue 648 if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)): 649 report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file) 650 continue 651 652 file_id = new_file_id(installed_file) 653 sha1 = checksum(build_output_path) 654 f = sbom_data.File(id=file_id, name=installed_file, checksum=sha1) 655 doc.files.append(f) 656 product_package.file_ids.append(file_id) 657 658 if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata): 659 add_package_of_file(file_id, installed_file_metadata, doc, report) 660 661 elif module_path or installed_file_metadata['is_platform_generated']: 662 # File from PLATFORM package 663 doc.add_relationship(sbom_data.Relationship(id1=file_id, 664 relationship=sbom_data.RelationshipType.GENERATED_FROM, 665 id2=sbom_data.SPDXID_PLATFORM)) 666 if installed_file_metadata['is_platform_generated']: 667 f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE] 668 669 elif product_copy_files: 670 # Format of product_copy_files: <source path>:<dest path> 671 src_path = product_copy_files.split(':')[0] 672 # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device, 673 # so process them as files from PLATFORM package 674 doc.add_relationship(sbom_data.Relationship(id1=file_id, 675 relationship=sbom_data.RelationshipType.GENERATED_FROM, 676 id2=sbom_data.SPDXID_PLATFORM)) 677 if installed_file_metadata['license_text']: 678 if installed_file_metadata['license_text'] == 'build/soong/licenses/LICENSE': 679 f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE] 680 681 elif installed_file.endswith('.fsv_meta'): 682 doc.add_relationship(sbom_data.Relationship(id1=file_id, 683 relationship=sbom_data.RelationshipType.GENERATED_FROM, 684 id2=sbom_data.SPDXID_PLATFORM)) 685 f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE] 686 687 elif kernel_module_copy_files.startswith('ANDROID-GEN'): 688 # For the four files generated for _dlkm, _ramdisk partitions 689 doc.add_relationship(sbom_data.Relationship(id1=file_id, 690 relationship=sbom_data.RelationshipType.GENERATED_FROM, 691 id2=sbom_data.SPDXID_PLATFORM)) 692 693 # Process static dependencies of the installed file 694 add_static_deps_of_file(file_id, installed_file_metadata, doc) 695 696 # Add licenses of the installed file 697 add_licenses_of_file(file_id, installed_file_metadata, doc) 698 699 # Add all static library files to SBOM 700 for dep_file in get_all_transitive_static_dep_files_of_installed_files(installed_files_metadata, db, report): 701 filepath = dep_file.removeprefix(args.soong_out + '/.intermediates/') 702 file_id = new_file_id(filepath) 703 # SHA1 of empty string. Sometimes .a files might not be built. 704 sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709' 705 if os.path.islink(dep_file) or os.path.isfile(dep_file): 706 sha1 = checksum(dep_file) 707 doc.files.append(sbom_data.File(id=file_id, 708 name=filepath, 709 checksum=sha1)) 710 file_metadata = { 711 'installed_file': dep_file, 712 'is_prebuilt_make_module': False 713 } 714 soong_module = db.get_soong_module_of_built_file(dep_file) 715 if not soong_module: 716 continue 717 file_metadata.update(soong_module) 718 if is_source_package(file_metadata) or is_prebuilt_package(file_metadata): 719 add_package_of_file(file_id, file_metadata, doc, report) 720 else: 721 # Other static lib files are generated from the platform 722 doc.add_relationship(sbom_data.Relationship(id1=file_id, 723 relationship=sbom_data.RelationshipType.GENERATED_FROM, 724 id2=sbom_data.SPDXID_PLATFORM)) 725 726 # Add relationships for static deps of static libraries 727 add_static_deps_of_file(file_id, file_metadata, doc) 728 729 # Add licenses of the static lib 730 add_licenses_of_file(file_id, file_metadata, doc) 731 732 # Save SBOM records to output file 733 doc.generate_packages_verification_code() 734 doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') 735 prefix = args.output_file 736 if prefix.endswith('.spdx'): 737 prefix = prefix.removesuffix('.spdx') 738 elif prefix.endswith('.spdx.json'): 739 prefix = prefix.removesuffix('.spdx.json') 740 741 output_file = prefix + '.spdx' 742 with open(output_file, 'w', encoding="utf-8") as file: 743 sbom_writers.TagValueWriter.write(doc, file) 744 if args.json: 745 with open(prefix + '.spdx.json', 'w', encoding="utf-8") as file: 746 sbom_writers.JSONWriter.write(doc, file) 747 748 save_report(prefix + '-gen-report.txt', report) 749 750 751if __name__ == '__main__': 752 main() 753