• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2024 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//	http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package android
16
17import (
18	"bytes"
19	"encoding/csv"
20	"fmt"
21	"path/filepath"
22	"slices"
23	"sort"
24	"strconv"
25	"strings"
26
27	"github.com/google/blueprint"
28	"github.com/google/blueprint/gobtools"
29)
30
31var (
32	// Constants of property names used in compliance metadata of modules
33	ComplianceMetadataProp = struct {
34		NAME                   string
35		PACKAGE                string
36		MODULE_TYPE            string
37		OS                     string
38		ARCH                   string
39		IS_PRIMARY_ARCH        string
40		VARIANT                string
41		IS_STATIC_LIB          string
42		INSTALLED_FILES        string
43		BUILT_FILES            string
44		STATIC_DEPS            string
45		STATIC_DEP_FILES       string
46		WHOLE_STATIC_DEPS      string
47		WHOLE_STATIC_DEP_FILES string
48		HEADER_LIBS            string
49		LICENSES               string
50
51		// module_type=package
52		PKG_DEFAULT_APPLICABLE_LICENSES string
53
54		// module_type=license
55		LIC_LICENSE_KINDS string
56		LIC_LICENSE_TEXT  string
57		LIC_PACKAGE_NAME  string
58
59		// module_type=license_kind
60		LK_CONDITIONS string
61		LK_URL        string
62	}{
63		"name",
64		"package",
65		"module_type",
66		"os",
67		"arch",
68		"is_primary_arch",
69		"variant",
70		"is_static_lib",
71		"installed_files",
72		"built_files",
73		"static_deps",
74		"static_dep_files",
75		"whole_static_deps",
76		"whole_static_dep_files",
77		"header_libs",
78		"licenses",
79
80		"pkg_default_applicable_licenses",
81
82		"lic_license_kinds",
83		"lic_license_text",
84		"lic_package_name",
85
86		"lk_conditions",
87		"lk_url",
88	}
89
90	// A constant list of all property names in compliance metadata
91	// Order of properties here is the order of columns in the exported CSV file.
92	COMPLIANCE_METADATA_PROPS = []string{
93		ComplianceMetadataProp.NAME,
94		ComplianceMetadataProp.PACKAGE,
95		ComplianceMetadataProp.MODULE_TYPE,
96		ComplianceMetadataProp.OS,
97		ComplianceMetadataProp.ARCH,
98		ComplianceMetadataProp.VARIANT,
99		ComplianceMetadataProp.IS_STATIC_LIB,
100		ComplianceMetadataProp.IS_PRIMARY_ARCH,
101		// Space separated installed files
102		ComplianceMetadataProp.INSTALLED_FILES,
103		// Space separated built files
104		ComplianceMetadataProp.BUILT_FILES,
105		// Space separated module names of static dependencies
106		ComplianceMetadataProp.STATIC_DEPS,
107		// Space separated file paths of static dependencies
108		ComplianceMetadataProp.STATIC_DEP_FILES,
109		// Space separated module names of whole static dependencies
110		ComplianceMetadataProp.WHOLE_STATIC_DEPS,
111		// Space separated file paths of whole static dependencies
112		ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES,
113		// Space separated modules name of header libs
114		ComplianceMetadataProp.HEADER_LIBS,
115		ComplianceMetadataProp.LICENSES,
116		// module_type=package
117		ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES,
118		// module_type=license
119		ComplianceMetadataProp.LIC_LICENSE_KINDS,
120		ComplianceMetadataProp.LIC_LICENSE_TEXT, // resolve to file paths
121		ComplianceMetadataProp.LIC_PACKAGE_NAME,
122		// module_type=license_kind
123		ComplianceMetadataProp.LK_CONDITIONS,
124		ComplianceMetadataProp.LK_URL,
125	}
126)
127
128// ComplianceMetadataInfo provides all metadata of a module, e.g. name, module type, package, license,
129// dependencies, built/installed files, etc. It is a wrapper on a map[string]string with some utility
130// methods to get/set properties' values.
131type ComplianceMetadataInfo struct {
132	properties          map[string]string
133	filesContained      []string
134	prebuiltFilesCopied []string
135}
136
137type complianceMetadataInfoGob struct {
138	Properties          map[string]string
139	FilesContained      []string
140	PrebuiltFilesCopied []string
141}
142
143func NewComplianceMetadataInfo() *ComplianceMetadataInfo {
144	return &ComplianceMetadataInfo{
145		properties:          map[string]string{},
146		filesContained:      make([]string, 0),
147		prebuiltFilesCopied: make([]string, 0),
148	}
149}
150
151func (m *ComplianceMetadataInfo) ToGob() *complianceMetadataInfoGob {
152	return &complianceMetadataInfoGob{
153		Properties:          m.properties,
154		FilesContained:      m.filesContained,
155		PrebuiltFilesCopied: m.prebuiltFilesCopied,
156	}
157}
158
159func (m *ComplianceMetadataInfo) FromGob(data *complianceMetadataInfoGob) {
160	m.properties = data.Properties
161	m.filesContained = data.FilesContained
162	m.prebuiltFilesCopied = data.PrebuiltFilesCopied
163}
164
165func (c *ComplianceMetadataInfo) GobEncode() ([]byte, error) {
166	return gobtools.CustomGobEncode[complianceMetadataInfoGob](c)
167}
168
169func (c *ComplianceMetadataInfo) GobDecode(data []byte) error {
170	return gobtools.CustomGobDecode[complianceMetadataInfoGob](data, c)
171}
172
173func (c *ComplianceMetadataInfo) SetStringValue(propertyName string, value string) {
174	if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) {
175		panic(fmt.Errorf("Unknown metadata property: %s.", propertyName))
176	}
177	c.properties[propertyName] = value
178}
179
180func (c *ComplianceMetadataInfo) SetListValue(propertyName string, value []string) {
181	c.SetStringValue(propertyName, strings.TrimSpace(strings.Join(value, " ")))
182}
183
184func (c *ComplianceMetadataInfo) SetFilesContained(files []string) {
185	c.filesContained = files
186}
187
188func (c *ComplianceMetadataInfo) GetFilesContained() []string {
189	return c.filesContained
190}
191
192func (c *ComplianceMetadataInfo) SetPrebuiltFilesCopied(files []string) {
193	c.prebuiltFilesCopied = files
194}
195
196func (c *ComplianceMetadataInfo) GetPrebuiltFilesCopied() []string {
197	return c.prebuiltFilesCopied
198}
199
200func (c *ComplianceMetadataInfo) getStringValue(propertyName string) string {
201	if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) {
202		panic(fmt.Errorf("Unknown metadata property: %s.", propertyName))
203	}
204	return c.properties[propertyName]
205}
206
207func (c *ComplianceMetadataInfo) getAllValues() map[string]string {
208	return c.properties
209}
210
211var (
212	ComplianceMetadataProvider = blueprint.NewProvider[*ComplianceMetadataInfo]()
213)
214
215// buildComplianceMetadataProvider starts with the ModuleContext.ComplianceMetadataInfo() and fills in more common metadata
216// for different module types without accessing their private fields but through android.Module interface
217// and public/private fields of package android. The final metadata is stored to a module's ComplianceMetadataProvider.
218func buildComplianceMetadataProvider(ctx *moduleContext, m *ModuleBase) {
219	complianceMetadataInfo := ctx.ComplianceMetadataInfo()
220	complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.NAME, m.Name())
221	complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.PACKAGE, ctx.ModuleDir())
222	complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.MODULE_TYPE, ctx.ModuleType())
223
224	switch ctx.ModuleType() {
225	case "license":
226		licenseModule := m.module.(*licenseModule)
227		complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_KINDS, licenseModule.properties.License_kinds)
228		complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_TEXT, PathsForModuleSrc(ctx, licenseModule.properties.License_text).Strings())
229		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LIC_PACKAGE_NAME, String(licenseModule.properties.Package_name))
230	case "license_kind":
231		licenseKindModule := m.module.(*licenseKindModule)
232		complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LK_CONDITIONS, licenseKindModule.properties.Conditions)
233		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LK_URL, licenseKindModule.properties.Url)
234	default:
235		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.OS, ctx.Os().String())
236		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.ARCH, ctx.Arch().String())
237		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.IS_PRIMARY_ARCH, strconv.FormatBool(ctx.PrimaryArch()))
238		complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.VARIANT, ctx.ModuleSubDir())
239		if m.primaryLicensesProperty != nil && m.primaryLicensesProperty.getName() == "licenses" {
240			complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LICENSES, m.primaryLicensesProperty.getStrings())
241		}
242
243		var installed InstallPaths
244		installed = append(installed, ctx.installFiles...)
245		installed = append(installed, ctx.katiInstalls.InstallPaths()...)
246		installed = append(installed, ctx.katiSymlinks.InstallPaths()...)
247		installed = append(installed, ctx.katiInitRcInstalls.InstallPaths()...)
248		installed = append(installed, ctx.katiVintfInstalls.InstallPaths()...)
249		complianceMetadataInfo.SetListValue(ComplianceMetadataProp.INSTALLED_FILES, FirstUniqueStrings(installed.Strings()))
250	}
251	ctx.setProvider(ComplianceMetadataProvider, complianceMetadataInfo)
252}
253
254func init() {
255	RegisterComplianceMetadataSingleton(InitRegistrationContext)
256}
257
258func RegisterComplianceMetadataSingleton(ctx RegistrationContext) {
259	ctx.RegisterParallelSingletonType("compliance_metadata_singleton", complianceMetadataSingletonFactory)
260}
261
262var (
263	// sqlite3 command line tool
264	sqlite3 = pctx.HostBinToolVariable("sqlite3", "sqlite3")
265
266	// Command to import .csv files to sqlite3 database
267	importCsv = pctx.AndroidStaticRule("importCsv",
268		blueprint.RuleParams{
269			Command: `rm -rf $out && ` +
270				`${sqlite3} $out ".import --csv $in modules" && ` +
271				`${sqlite3} $out ".import --csv ${make_metadata} make_metadata" && ` +
272				`${sqlite3} $out ".import --csv ${make_modules} make_modules"`,
273			CommandDeps: []string{"${sqlite3}"},
274		}, "make_metadata", "make_modules")
275)
276
277func complianceMetadataSingletonFactory() Singleton {
278	return &complianceMetadataSingleton{}
279}
280
281type complianceMetadataSingleton struct {
282}
283
284func writerToCsv(csvWriter *csv.Writer, row []string) {
285	err := csvWriter.Write(row)
286	if err != nil {
287		panic(err)
288	}
289}
290
291// Collect compliance metadata from all Soong modules, write to a CSV file and
292// import compliance metadata from Make and Soong to a sqlite3 database.
293func (c *complianceMetadataSingleton) GenerateBuildActions(ctx SingletonContext) {
294	if !ctx.Config().HasDeviceProduct() {
295		return
296	}
297	var buffer bytes.Buffer
298	csvWriter := csv.NewWriter(&buffer)
299
300	// Collect compliance metadata of modules in Soong and write to out/soong/compliance-metadata/<product>/soong-modules.csv file.
301	columnNames := []string{"id"}
302	columnNames = append(columnNames, COMPLIANCE_METADATA_PROPS...)
303	writerToCsv(csvWriter, columnNames)
304
305	rowId := -1
306	ctx.VisitAllModuleProxies(func(module ModuleProxy) {
307		commonInfo := OtherModulePointerProviderOrDefault(ctx, module, CommonModuleInfoProvider)
308		if !commonInfo.Enabled {
309			return
310		}
311
312		moduleType := ctx.ModuleType(module)
313		if moduleType == "package" {
314			metadataMap := map[string]string{
315				ComplianceMetadataProp.NAME:                            ctx.ModuleName(module),
316				ComplianceMetadataProp.MODULE_TYPE:                     ctx.ModuleType(module),
317				ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES: strings.Join(commonInfo.PrimaryLicensesProperty.getStrings(), " "),
318			}
319			rowId = rowId + 1
320			metadata := []string{strconv.Itoa(rowId)}
321			for _, propertyName := range COMPLIANCE_METADATA_PROPS {
322				metadata = append(metadata, metadataMap[propertyName])
323			}
324			writerToCsv(csvWriter, metadata)
325			return
326		}
327		if metadataInfo, ok := OtherModuleProvider(ctx, module, ComplianceMetadataProvider); ok {
328			rowId = rowId + 1
329			metadata := []string{strconv.Itoa(rowId)}
330			for _, propertyName := range COMPLIANCE_METADATA_PROPS {
331				metadata = append(metadata, metadataInfo.getStringValue(propertyName))
332			}
333			writerToCsv(csvWriter, metadata)
334			return
335		}
336	})
337	csvWriter.Flush()
338
339	deviceProduct := ctx.Config().DeviceProduct()
340	modulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "soong-modules.csv")
341	WriteFileRuleVerbatim(ctx, modulesCsv, buffer.String())
342
343	// Metadata generated in Make
344	makeMetadataCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-metadata.csv")
345	makeModulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-modules.csv")
346
347	productOutPath := filepath.Join(ctx.Config().OutDir(), "target", "product", String(ctx.Config().productVariables.DeviceName))
348	if !ctx.Config().KatiEnabled() {
349		ctx.VisitAllModuleProxies(func(module ModuleProxy) {
350			// In soong-only build the installed file list is from android_device module
351			if androidDeviceInfo, ok := OtherModuleProvider(ctx, module, AndroidDeviceInfoProvider); ok && androidDeviceInfo.Main_device {
352				if metadataInfo, ok := OtherModuleProvider(ctx, module, ComplianceMetadataProvider); ok {
353					if len(metadataInfo.filesContained) > 0 || len(metadataInfo.prebuiltFilesCopied) > 0 {
354						allFiles := make([]string, 0, len(metadataInfo.filesContained)+len(metadataInfo.prebuiltFilesCopied))
355						allFiles = append(allFiles, metadataInfo.filesContained...)
356						prebuiltFilesSrcDest := make(map[string]string)
357						for _, srcDestPair := range metadataInfo.prebuiltFilesCopied {
358							prebuiltFilePath := filepath.Join(productOutPath, strings.Split(srcDestPair, ":")[1])
359							allFiles = append(allFiles, prebuiltFilePath)
360							prebuiltFilesSrcDest[prebuiltFilePath] = srcDestPair
361						}
362						sort.Strings(allFiles)
363
364						csvHeaders := "installed_file,module_path,is_soong_module,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,static_libs,whole_static_libs,license_text"
365						csvContent := make([]string, 0, len(allFiles)+1)
366						csvContent = append(csvContent, csvHeaders)
367						for _, file := range allFiles {
368							if _, ok := prebuiltFilesSrcDest[file]; ok {
369								srcDestPair := prebuiltFilesSrcDest[file]
370								csvContent = append(csvContent, file+",,,,"+srcDestPair+",,,,,")
371							} else {
372								csvContent = append(csvContent, file+",,Y,,,,,,,")
373							}
374						}
375
376						WriteFileRuleVerbatim(ctx, makeMetadataCsv, strings.Join(csvContent, "\n"))
377						WriteFileRuleVerbatim(ctx, makeModulesCsv, "name,module_path,module_class,module_type,static_libs,whole_static_libs,built_files,installed_files")
378					}
379					return
380				}
381			}
382		})
383	}
384
385	// Import metadata from Make and Soong to sqlite3 database
386	complianceMetadataDb := PathForOutput(ctx, "compliance-metadata", deviceProduct, "compliance-metadata.db")
387	ctx.Build(pctx, BuildParams{
388		Rule:  importCsv,
389		Input: modulesCsv,
390		Implicits: []Path{
391			makeMetadataCsv,
392			makeModulesCsv,
393		},
394		Output: complianceMetadataDb,
395		Args: map[string]string{
396			"make_metadata": makeMetadataCsv.String(),
397			"make_modules":  makeModulesCsv.String(),
398		},
399	})
400
401	// Phony rule "compliance-metadata.db". "m compliance-metadata.db" to create the compliance metadata database.
402	ctx.Build(pctx, BuildParams{
403		Rule:   blueprint.Phony,
404		Inputs: []Path{complianceMetadataDb},
405		Output: PathForPhony(ctx, "compliance-metadata.db"),
406	})
407
408}
409