1// Copyright 2022 Google LLC 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package main 16 17import ( 18 "bytes" 19 "crypto/sha1" 20 "encoding/hex" 21 "flag" 22 "fmt" 23 "io" 24 "io/fs" 25 "os" 26 "path/filepath" 27 "sort" 28 "strings" 29 "time" 30 31 "android/soong/response" 32 "android/soong/tools/compliance" 33 "android/soong/tools/compliance/projectmetadata" 34 35 "github.com/google/blueprint/deptools" 36 37 "github.com/spdx/tools-golang/builder/builder2v2" 38 "github.com/spdx/tools-golang/json" 39 "github.com/spdx/tools-golang/spdx/common" 40 spdx "github.com/spdx/tools-golang/spdx/v2_2" 41 "github.com/spdx/tools-golang/spdxlib" 42) 43 44var ( 45 failNoneRequested = fmt.Errorf("\nNo license metadata files requested") 46 failNoLicenses = fmt.Errorf("No licenses found") 47) 48 49const NOASSERTION = "NOASSERTION" 50 51type context struct { 52 stdout io.Writer 53 stderr io.Writer 54 rootFS fs.FS 55 product string 56 stripPrefix []string 57 creationTime creationTimeGetter 58} 59 60func (ctx context) strip(installPath string) string { 61 for _, prefix := range ctx.stripPrefix { 62 if strings.HasPrefix(installPath, prefix) { 63 p := strings.TrimPrefix(installPath, prefix) 64 if 0 == len(p) { 65 p = ctx.product 66 } 67 if 0 == len(p) { 68 continue 69 } 70 return p 71 } 72 } 73 return installPath 74} 75 76// newMultiString creates a flag that allows multiple values in an array. 77func newMultiString(flags *flag.FlagSet, name, usage string) *multiString { 78 var f multiString 79 flags.Var(&f, name, usage) 80 return &f 81} 82 83// multiString implements the flag `Value` interface for multiple strings. 84type multiString []string 85 86func (ms *multiString) String() string { return strings.Join(*ms, ", ") } 87func (ms *multiString) Set(s string) error { *ms = append(*ms, s); return nil } 88 89func main() { 90 var expandedArgs []string 91 for _, arg := range os.Args[1:] { 92 if strings.HasPrefix(arg, "@") { 93 f, err := os.Open(strings.TrimPrefix(arg, "@")) 94 if err != nil { 95 fmt.Fprintln(os.Stderr, err.Error()) 96 os.Exit(1) 97 } 98 99 respArgs, err := response.ReadRspFile(f) 100 f.Close() 101 if err != nil { 102 fmt.Fprintln(os.Stderr, err.Error()) 103 os.Exit(1) 104 } 105 expandedArgs = append(expandedArgs, respArgs...) 106 } else { 107 expandedArgs = append(expandedArgs, arg) 108 } 109 } 110 111 flags := flag.NewFlagSet("flags", flag.ExitOnError) 112 113 flags.Usage = func() { 114 fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...} 115 116Outputs an SBOM.spdx. 117 118Options: 119`, filepath.Base(os.Args[0])) 120 flags.PrintDefaults() 121 } 122 123 outputFile := flags.String("o", "-", "Where to write the SBOM spdx file. (default stdout)") 124 depsFile := flags.String("d", "", "Where to write the deps file") 125 product := flags.String("product", "", "The name of the product for which the notice is generated.") 126 stripPrefix := newMultiString(flags, "strip_prefix", "Prefix to remove from paths. i.e. path to root (multiple allowed)") 127 128 flags.Parse(expandedArgs) 129 130 // Must specify at least one root target. 131 if flags.NArg() == 0 { 132 flags.Usage() 133 os.Exit(2) 134 } 135 136 if len(*outputFile) == 0 { 137 flags.Usage() 138 fmt.Fprintf(os.Stderr, "must specify file for -o; use - for stdout\n") 139 os.Exit(2) 140 } else { 141 dir, err := filepath.Abs(filepath.Dir(*outputFile)) 142 if err != nil { 143 fmt.Fprintf(os.Stderr, "cannot determine path to %q: %s\n", *outputFile, err) 144 os.Exit(1) 145 } 146 fi, err := os.Stat(dir) 147 if err != nil { 148 fmt.Fprintf(os.Stderr, "cannot read directory %q of %q: %s\n", dir, *outputFile, err) 149 os.Exit(1) 150 } 151 if !fi.IsDir() { 152 fmt.Fprintf(os.Stderr, "parent %q of %q is not a directory\n", dir, *outputFile) 153 os.Exit(1) 154 } 155 } 156 157 var ofile io.Writer 158 ofile = os.Stdout 159 var obuf *bytes.Buffer 160 if *outputFile != "-" { 161 obuf = &bytes.Buffer{} 162 ofile = obuf 163 } 164 165 ctx := &context{ofile, os.Stderr, compliance.FS, *product, *stripPrefix, actualTime} 166 167 spdxDoc, deps, err := sbomGenerator(ctx, flags.Args()...) 168 169 if err != nil { 170 if err == failNoneRequested { 171 flags.Usage() 172 } 173 fmt.Fprintf(os.Stderr, "%s\n", err.Error()) 174 os.Exit(1) 175 } 176 177 // writing the spdx Doc created 178 if err := spdx_json.Save2_2(spdxDoc, ofile); err != nil { 179 fmt.Fprintf(os.Stderr, "failed to write document to %v: %v", *outputFile, err) 180 os.Exit(1) 181 } 182 183 if *outputFile != "-" { 184 err := os.WriteFile(*outputFile, obuf.Bytes(), 0666) 185 if err != nil { 186 fmt.Fprintf(os.Stderr, "could not write output to %q: %s\n", *outputFile, err) 187 os.Exit(1) 188 } 189 } 190 191 if *depsFile != "" { 192 err := deptools.WriteDepFile(*depsFile, *outputFile, deps) 193 if err != nil { 194 fmt.Fprintf(os.Stderr, "could not write deps to %q: %s\n", *depsFile, err) 195 os.Exit(1) 196 } 197 } 198 os.Exit(0) 199} 200 201type creationTimeGetter func() string 202 203// actualTime returns current time in UTC 204func actualTime() string { 205 t := time.Now().UTC() 206 return t.UTC().Format("2006-01-02T15:04:05Z") 207} 208 209// replaceSlashes replaces "/" by "-" for the library path to be used for packages & files SPDXID 210func replaceSlashes(x string) string { 211 return strings.ReplaceAll(x, "/", "-") 212} 213 214// stripDocName removes the outdir prefix and meta_lic suffix from a target Name 215func stripDocName(name string) string { 216 // remove outdir prefix 217 if strings.HasPrefix(name, "out/") { 218 name = name[4:] 219 } 220 221 // remove suffix 222 if strings.HasSuffix(name, ".meta_lic") { 223 name = name[:len(name)-9] 224 } else if strings.HasSuffix(name, "/meta_lic") { 225 name = name[:len(name)-9] + "/" 226 } 227 228 return name 229} 230 231// getPackageName returns a package name of a target Node 232func getPackageName(_ *context, tn *compliance.TargetNode) string { 233 return replaceSlashes(tn.Name()) 234} 235 236// getDocumentName returns a package name of a target Node 237func getDocumentName(ctx *context, tn *compliance.TargetNode, pm *projectmetadata.ProjectMetadata) string { 238 if len(ctx.product) > 0 { 239 return replaceSlashes(ctx.product) 240 } 241 if len(tn.ModuleName()) > 0 { 242 if pm != nil { 243 return replaceSlashes(pm.Name() + ":" + tn.ModuleName()) 244 } 245 return replaceSlashes(tn.ModuleName()) 246 } 247 248 return stripDocName(replaceSlashes(tn.Name())) 249} 250 251// getDownloadUrl returns the download URL if available (GIT, SVN, etc..), 252// or NOASSERTION if not available, none determined or ambiguous 253func getDownloadUrl(_ *context, pm *projectmetadata.ProjectMetadata) string { 254 if pm == nil { 255 return NOASSERTION 256 } 257 258 urlsByTypeName := pm.UrlsByTypeName() 259 if urlsByTypeName == nil { 260 return NOASSERTION 261 } 262 263 url := urlsByTypeName.DownloadUrl() 264 if url == "" { 265 return NOASSERTION 266 } 267 return url 268} 269 270// getProjectMetadata returns the optimal project metadata for the target node 271func getProjectMetadata(_ *context, pmix *projectmetadata.Index, 272 tn *compliance.TargetNode) (*projectmetadata.ProjectMetadata, error) { 273 pms, err := pmix.MetadataForProjects(tn.Projects()...) 274 if err != nil { 275 return nil, fmt.Errorf("Unable to read projects for %q: %w\n", tn, err) 276 } 277 if len(pms) == 0 { 278 return nil, nil 279 } 280 281 // Getting the project metadata that contains most of the info needed for sbomGenerator 282 score := -1 283 index := -1 284 for i := 0; i < len(pms); i++ { 285 tempScore := 0 286 if pms[i].Name() != "" { 287 tempScore += 1 288 } 289 if pms[i].Version() != "" { 290 tempScore += 1 291 } 292 if pms[i].UrlsByTypeName().DownloadUrl() != "" { 293 tempScore += 1 294 } 295 296 if tempScore == score { 297 if pms[i].Project() < pms[index].Project() { 298 index = i 299 } 300 } else if tempScore > score { 301 score = tempScore 302 index = i 303 } 304 } 305 return pms[index], nil 306} 307 308// inputFiles returns the complete list of files read 309func inputFiles(lg *compliance.LicenseGraph, pmix *projectmetadata.Index, licenseTexts []string) []string { 310 projectMeta := pmix.AllMetadataFiles() 311 targets := lg.TargetNames() 312 files := make([]string, 0, len(licenseTexts)+len(targets)+len(projectMeta)) 313 files = append(files, licenseTexts...) 314 files = append(files, targets...) 315 files = append(files, projectMeta...) 316 return files 317} 318 319// generateSPDXNamespace generates a unique SPDX Document Namespace using a SHA1 checksum 320// and the CreationInfo.Created field as the date. 321func generateSPDXNamespace(created string) string { 322 // Compute a SHA1 checksum of the CreationInfo.Created field. 323 hash := sha1.Sum([]byte(created)) 324 checksum := hex.EncodeToString(hash[:]) 325 326 // Combine the checksum and timestamp to generate the SPDX Namespace. 327 namespace := fmt.Sprintf("SPDXRef-DOCUMENT-%s-%s", created, checksum) 328 329 return namespace 330} 331 332// sbomGenerator implements the spdx bom utility 333 334// SBOM is part of the new government regulation issued to improve national cyber security 335// and enhance software supply chain and transparency, see https://www.cisa.gov/sbom 336 337// sbomGenerator uses the SPDX standard, see the SPDX specification (https://spdx.github.io/spdx-spec/) 338// sbomGenerator is also following the internal google SBOM styleguide (http://goto.google.com/spdx-style-guide) 339func sbomGenerator(ctx *context, files ...string) (*spdx.Document, []string, error) { 340 // Must be at least one root file. 341 if len(files) < 1 { 342 return nil, nil, failNoneRequested 343 } 344 345 pmix := projectmetadata.NewIndex(ctx.rootFS) 346 347 lg, err := compliance.ReadLicenseGraph(ctx.rootFS, ctx.stderr, files) 348 349 if err != nil { 350 return nil, nil, fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err) 351 } 352 353 // creating the packages section 354 pkgs := []*spdx.Package{} 355 356 // creating the relationship section 357 relationships := []*spdx.Relationship{} 358 359 // creating the license section 360 otherLicenses := []*spdx.OtherLicense{} 361 362 // spdx document name 363 var docName string 364 365 // main package name 366 var mainPkgName string 367 368 // implementing the licenses references for the packages 369 licenses := make(map[string]string) 370 concludedLicenses := func(licenseTexts []string) string { 371 licenseRefs := make([]string, 0, len(licenseTexts)) 372 for _, licenseText := range licenseTexts { 373 license := strings.SplitN(licenseText, ":", 2)[0] 374 if _, ok := licenses[license]; !ok { 375 licenseRef := "LicenseRef-" + replaceSlashes(license) 376 licenses[license] = licenseRef 377 } 378 379 licenseRefs = append(licenseRefs, licenses[license]) 380 } 381 if len(licenseRefs) > 1 { 382 return "(" + strings.Join(licenseRefs, " AND ") + ")" 383 } else if len(licenseRefs) == 1 { 384 return licenseRefs[0] 385 } 386 return "NONE" 387 } 388 389 isMainPackage := true 390 visitedNodes := make(map[*compliance.TargetNode]struct{}) 391 392 // performing a Breadth-first top down walk of licensegraph and building package information 393 compliance.WalkTopDownBreadthFirst(nil, lg, 394 func(lg *compliance.LicenseGraph, tn *compliance.TargetNode, path compliance.TargetEdgePath) bool { 395 if err != nil { 396 return false 397 } 398 var pm *projectmetadata.ProjectMetadata 399 pm, err = getProjectMetadata(ctx, pmix, tn) 400 if err != nil { 401 return false 402 } 403 404 if isMainPackage { 405 docName = getDocumentName(ctx, tn, pm) 406 mainPkgName = replaceSlashes(getPackageName(ctx, tn)) 407 isMainPackage = false 408 } 409 410 if len(path) == 0 { 411 // Add the describe relationship for the main package 412 rln := &spdx.Relationship{ 413 RefA: common.MakeDocElementID("" /* this document */, "DOCUMENT"), 414 RefB: common.MakeDocElementID("", mainPkgName), 415 Relationship: "DESCRIBES", 416 } 417 relationships = append(relationships, rln) 418 419 } else { 420 // Check parent and identify annotation 421 parent := path[len(path)-1] 422 targetEdge := parent.Edge() 423 if targetEdge.IsRuntimeDependency() { 424 // Adding the dynamic link annotation RUNTIME_DEPENDENCY_OF relationship 425 rln := &spdx.Relationship{ 426 RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))), 427 RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))), 428 Relationship: "RUNTIME_DEPENDENCY_OF", 429 } 430 relationships = append(relationships, rln) 431 432 } else if targetEdge.IsDerivation() { 433 // Adding the derivation annotation as a CONTAINS relationship 434 rln := &spdx.Relationship{ 435 RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))), 436 RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))), 437 Relationship: "CONTAINS", 438 } 439 relationships = append(relationships, rln) 440 441 } else if targetEdge.IsBuildTool() { 442 // Adding the toolchain annotation as a BUILD_TOOL_OF relationship 443 rln := &spdx.Relationship{ 444 RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))), 445 RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))), 446 Relationship: "BUILD_TOOL_OF", 447 } 448 relationships = append(relationships, rln) 449 450 } else { 451 panic(fmt.Errorf("Unknown dependency type: %v", targetEdge.Annotations())) 452 } 453 } 454 455 if _, alreadyVisited := visitedNodes[tn]; alreadyVisited { 456 return false 457 } 458 visitedNodes[tn] = struct{}{} 459 pkgName := getPackageName(ctx, tn) 460 461 // Making an spdx package and adding it to pkgs 462 pkg := &spdx.Package{ 463 PackageName: replaceSlashes(pkgName), 464 PackageDownloadLocation: getDownloadUrl(ctx, pm), 465 PackageSPDXIdentifier: common.ElementID(replaceSlashes(pkgName)), 466 PackageLicenseConcluded: concludedLicenses(tn.LicenseTexts()), 467 } 468 469 if pm != nil && pm.Version() != "" { 470 pkg.PackageVersion = pm.Version() 471 } else { 472 pkg.PackageVersion = NOASSERTION 473 } 474 475 pkgs = append(pkgs, pkg) 476 477 return true 478 }) 479 480 // Adding Non-standard licenses 481 482 licenseTexts := make([]string, 0, len(licenses)) 483 484 for licenseText := range licenses { 485 licenseTexts = append(licenseTexts, licenseText) 486 } 487 488 sort.Strings(licenseTexts) 489 490 for _, licenseText := range licenseTexts { 491 // open the file 492 f, err := ctx.rootFS.Open(filepath.Clean(licenseText)) 493 if err != nil { 494 return nil, nil, fmt.Errorf("error opening license text file %q: %w", licenseText, err) 495 } 496 497 // read the file 498 text, err := io.ReadAll(f) 499 if err != nil { 500 return nil, nil, fmt.Errorf("error reading license text file %q: %w", licenseText, err) 501 } 502 // Making an spdx License and adding it to otherLicenses 503 otherLicenses = append(otherLicenses, &spdx.OtherLicense{ 504 LicenseName: strings.Replace(licenses[licenseText], "LicenseRef-", "", -1), 505 LicenseIdentifier: string(licenses[licenseText]), 506 ExtractedText: string(text), 507 }) 508 } 509 510 deps := inputFiles(lg, pmix, licenseTexts) 511 sort.Strings(deps) 512 513 // Making the SPDX doc 514 ci, err := builder2v2.BuildCreationInfoSection2_2("Organization", "Google LLC", nil) 515 if err != nil { 516 return nil, nil, fmt.Errorf("Unable to build creation info section for SPDX doc: %v\n", err) 517 } 518 519 ci.Created = ctx.creationTime() 520 521 doc := &spdx.Document{ 522 SPDXVersion: "SPDX-2.2", 523 DataLicense: "CC0-1.0", 524 SPDXIdentifier: "DOCUMENT", 525 DocumentName: docName, 526 DocumentNamespace: generateSPDXNamespace(ci.Created), 527 CreationInfo: ci, 528 Packages: pkgs, 529 Relationships: relationships, 530 OtherLicenses: otherLicenses, 531 } 532 533 if err := spdxlib.ValidateDocument2_2(doc); err != nil { 534 return nil, nil, fmt.Errorf("Unable to validate the SPDX doc: %v\n", err) 535 } 536 537 return doc, deps, nil 538} 539