// Copyright 2023 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package android import ( "crypto/sha1" "encoding/hex" "fmt" "github.com/google/blueprint" "io" "io/fs" "os" "path/filepath" "strings" "testing" "github.com/google/blueprint/proptools" ) // WriteFileRule creates a ninja rule to write contents to a file by immediately writing the // contents, plus a trailing newline, to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating // a ninja rule to copy the file into place. func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) { writeFileRule(ctx, outputFile, content, true, false) } // WriteFileRuleVerbatim creates a ninja rule to write contents to a file by immediately writing the // contents to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating a ninja rule to copy the file into place. func WriteFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) { writeFileRule(ctx, outputFile, content, false, false) } // WriteExecutableFileRuleVerbatim is the same as WriteFileRuleVerbatim, but runs chmod +x on the result func WriteExecutableFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) { writeFileRule(ctx, outputFile, content, false, true) } // tempFile provides a testable wrapper around a file in out/soong/.temp. It writes to a temporary file when // not in tests, but writes to a buffer in memory when used in tests. type tempFile struct { // tempFile contains wraps an io.Writer, which will be file if testMode is false, or testBuf if it is true. io.Writer file *os.File testBuf *strings.Builder } func newTempFile(ctx BuilderContext, pattern string, testMode bool) *tempFile { if testMode { testBuf := &strings.Builder{} return &tempFile{ Writer: testBuf, testBuf: testBuf, } } else { f, err := os.CreateTemp(absolutePath(ctx.Config().tempDir()), pattern) if err != nil { panic(fmt.Errorf("failed to open temporary raw file: %w", err)) } return &tempFile{ Writer: f, file: f, } } } func (t *tempFile) close() error { if t.file != nil { return t.file.Close() } return nil } func (t *tempFile) name() string { if t.file != nil { return t.file.Name() } return "temp_file_in_test" } func (t *tempFile) rename(to string) { if t.file != nil { os.MkdirAll(filepath.Dir(to), 0777) err := os.Rename(t.file.Name(), to) if err != nil { panic(fmt.Errorf("failed to rename %s to %s: %w", t.file.Name(), to, err)) } } } func (t *tempFile) remove() error { if t.file != nil { return os.Remove(t.file.Name()) } return nil } func writeContentToTempFileAndHash(ctx BuilderContext, content string, newline bool) (*tempFile, string) { tempFile := newTempFile(ctx, "raw", ctx.Config().captureBuild) defer tempFile.close() hash := sha1.New() w := io.MultiWriter(tempFile, hash) _, err := io.WriteString(w, content) if err == nil && newline { _, err = io.WriteString(w, "\n") } if err != nil { panic(fmt.Errorf("failed to write to temporary raw file %s: %w", tempFile.name(), err)) } return tempFile, hex.EncodeToString(hash.Sum(nil)) } func writeFileRule(ctx BuilderContext, outputFile WritablePath, content string, newline bool, executable bool) { // Write the contents to a temporary file while computing its hash. tempFile, hash := writeContentToTempFileAndHash(ctx, content, newline) // Shard the final location of the raw file into a subdirectory based on the first two characters of the // hash to avoid making the raw directory too large and slowing down accesses. relPath := filepath.Join(hash[0:2], hash) // These files are written during soong_build. If something outside the build deleted them there would be no // trigger to rerun soong_build, and the build would break with dependencies on missing files. Writing them // to their final locations would risk having them deleted when cleaning a module, and would also pollute the // output directory with files for modules that have never been built. // Instead, the files are written to a separate "raw" directory next to the build.ninja file, and a ninja // rule is created to copy the files into their final location as needed. // Obsolete files written by previous runs of soong_build must be cleaned up to avoid continually growing // disk usage as the hashes of the files change over time. The cleanup must not remove files that were // created by previous runs of soong_build for other products, as the build.ninja files for those products // may still exist and still reference those files. The raw files from different products are kept // separate by appending the Make_suffix to the directory name. rawPath := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix), relPath) rawFileInfo := rawFileInfo{ relPath: relPath, } if ctx.Config().captureBuild { // When running tests tempFile won't write to disk, instead store the contents for later retrieval by // ContentFromFileRuleForTests. rawFileInfo.contentForTests = tempFile.testBuf.String() } rawFileSet := getRawFileSet(ctx.Config()) if _, exists := rawFileSet.LoadOrStore(hash, rawFileInfo); exists { // If a raw file with this hash has already been created delete the temporary file. tempFile.remove() } else { // If this is the first time this hash has been seen then move it from the temporary directory // to the raw directory. If the file already exists in the raw directory assume it has the correct // contents. absRawPath := absolutePath(rawPath.String()) _, err := os.Stat(absRawPath) if os.IsNotExist(err) { tempFile.rename(absRawPath) } else if err != nil { panic(fmt.Errorf("failed to stat %q: %w", absRawPath, err)) } else { tempFile.remove() } } // Emit a rule to copy the file from raw directory to the final requested location in the output tree. // Restat is used to ensure that two different products that produce identical files copied from their // own raw directories they don't cause everything downstream to rebuild. rule := rawFileCopy if executable { rule = rawFileCopyExecutable } ctx.Build(pctx, BuildParams{ Rule: rule, Input: rawPath, Output: outputFile, Description: "raw " + outputFile.Base(), }) } var ( rawFileCopy = pctx.AndroidStaticRule("rawFileCopy", blueprint.RuleParams{ Command: "if ! cmp -s $in $out; then cp $in $out; fi", Description: "copy raw file $out", Restat: true, }) rawFileCopyExecutable = pctx.AndroidStaticRule("rawFileCopyExecutable", blueprint.RuleParams{ Command: "if ! cmp -s $in $out; then cp $in $out; fi && chmod +x $out", Description: "copy raw exectuable file $out", Restat: true, }) ) type rawFileInfo struct { relPath string contentForTests string } var rawFileSetKey OnceKey = NewOnceKey("raw file set") func getRawFileSet(config Config) *SyncMap[string, rawFileInfo] { return config.Once(rawFileSetKey, func() any { return &SyncMap[string, rawFileInfo]{} }).(*SyncMap[string, rawFileInfo]) } // ContentFromFileRuleForTests returns the content that was passed to a WriteFileRule for use // in tests. func ContentFromFileRuleForTests(t *testing.T, ctx *TestContext, params TestingBuildParams) string { t.Helper() if params.Rule != rawFileCopy && params.Rule != rawFileCopyExecutable { t.Errorf("expected params.Rule to be rawFileCopy or rawFileCopyExecutable, was %q", params.Rule) return "" } key := filepath.Base(params.Input.String()) rawFileSet := getRawFileSet(ctx.Config()) rawFileInfo, _ := rawFileSet.Load(key) return rawFileInfo.contentForTests } func rawFilesSingletonFactory() Singleton { return &rawFilesSingleton{} } type rawFilesSingleton struct{} func (rawFilesSingleton) GenerateBuildActions(ctx SingletonContext) { if ctx.Config().captureBuild { // Nothing to do when running in tests, no temporary files were created. return } rawFileSet := getRawFileSet(ctx.Config()) rawFilesDir := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix)).String() absRawFilesDir := absolutePath(rawFilesDir) err := filepath.WalkDir(absRawFilesDir, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { // Ignore obsolete directories for now. return nil } // Assume the basename of the file is a hash key := filepath.Base(path) relPath, err := filepath.Rel(absRawFilesDir, path) if err != nil { return err } // Check if a file with the same hash was written by this run of soong_build. If the file was not written, // or if a file with the same hash was written but to a different path in the raw directory, then delete it. // Checking that the path matches allows changing the structure of the raw directory, for example to increase // the sharding. rawFileInfo, written := rawFileSet.Load(key) if !written || rawFileInfo.relPath != relPath { os.Remove(path) } return nil }) if err != nil { panic(fmt.Errorf("failed to clean %q: %w", rawFilesDir, err)) } }