soong/android/raw_files.go

// Copyright 2023 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package android

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/google/blueprint"
	"github.com/google/blueprint/syncmap"

	"github.com/google/blueprint/proptools"
)

// WriteFileRule creates a ninja rule to write contents to a file by immediately writing the
// contents, plus a trailing newline, to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating
// a ninja rule to copy the file into place.
func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string, validations ...Path) {
	writeFileRule(ctx, outputFile, content, true, false, validations)
}

// WriteFileRuleVerbatim creates a ninja rule to write contents to a file by immediately writing the
// contents to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating a ninja rule to copy the file into place.
func WriteFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string, validations ...Path) {
	writeFileRule(ctx, outputFile, content, false, false, validations)
}

// WriteExecutableFileRuleVerbatim is the same as WriteFileRuleVerbatim, but runs chmod +x on the result
func WriteExecutableFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string, validations ...Path) {
	writeFileRule(ctx, outputFile, content, false, true, validations)
}

// tempFile provides a testable wrapper around a file in out/soong/.temp.  It writes to a temporary file when
// not in tests, but writes to a buffer in memory when used in tests.
type tempFile struct {
	// tempFile contains wraps an io.Writer, which will be file if testMode is false, or testBuf if it is true.
	io.Writer

	file    *os.File
	testBuf *strings.Builder
}

func newTempFile(ctx BuilderContext, pattern string, testMode bool) *tempFile {
	if testMode {
		testBuf := &strings.Builder{}
		return &tempFile{
			Writer:  testBuf,
			testBuf: testBuf,
		}
	} else {
		f, err := os.CreateTemp(absolutePath(ctx.Config().tempDir()), pattern)
		if err != nil {
			panic(fmt.Errorf("failed to open temporary raw file: %w", err))
		}
		return &tempFile{
			Writer: f,
			file:   f,
		}
	}
}

func (t *tempFile) close() error {
	if t.file != nil {
		return t.file.Close()
	}
	return nil
}

func (t *tempFile) name() string {
	if t.file != nil {
		return t.file.Name()
	}
	return "temp_file_in_test"
}

func (t *tempFile) rename(to string) {
	if t.file != nil {
		os.MkdirAll(filepath.Dir(to), 0777)
		err := os.Rename(t.file.Name(), to)
		if err != nil {
			panic(fmt.Errorf("failed to rename %s to %s: %w", t.file.Name(), to, err))
		}
	}
}

func (t *tempFile) remove() error {
	if t.file != nil {
		return os.Remove(t.file.Name())
	}
	return nil
}

func writeContentToTempFileAndHash(ctx BuilderContext, content string, newline bool) (*tempFile, string) {
	tempFile := newTempFile(ctx, "raw", ctx.Config().captureBuild)
	defer tempFile.close()

	hash := sha1.New()
	w := io.MultiWriter(tempFile, hash)

	_, err := io.WriteString(w, content)
	if err == nil && newline {
		_, err = io.WriteString(w, "\n")
	}
	if err != nil {
		panic(fmt.Errorf("failed to write to temporary raw file %s: %w", tempFile.name(), err))
	}
	return tempFile, hex.EncodeToString(hash.Sum(nil))
}

func writeFileRule(ctx BuilderContext, outputFile WritablePath, content string, newline bool, executable bool, validations Paths) {
	// Write the contents to a temporary file while computing its hash.
	tempFile, hash := writeContentToTempFileAndHash(ctx, content, newline)

	// Shard the final location of the raw file into a subdirectory based on the first two characters of the
	// hash to avoid making the raw directory too large and slowing down accesses.
	relPath := filepath.Join(hash[0:2], hash)

	// These files are written during soong_build.  If something outside the build deleted them there would be no
	// trigger to rerun soong_build, and the build would break with dependencies on missing files.  Writing them
	// to their final locations would risk having them deleted when cleaning a module, and would also pollute the
	// output directory with files for modules that have never been built.
	// Instead, the files are written to a separate "raw" directory next to the build.ninja file, and a ninja
	// rule is created to copy the files into their final location as needed.
	// Obsolete files written by previous runs of soong_build must be cleaned up to avoid continually growing
	// disk usage as the hashes of the files change over time.  The cleanup must not remove files that were
	// created by previous runs of soong_build for other products, as the build.ninja files for those products
	// may still exist and still reference those files.  The raw files from different products are kept
	// separate by appending the Make_suffix to the directory name.
	rawPath := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix), relPath)

	rawFileInfo := rawFileInfo{
		relPath: relPath,
	}

	if ctx.Config().captureBuild {
		// When running tests tempFile won't write to disk, instead store the contents for later retrieval by
		// ContentFromFileRuleForTests.
		rawFileInfo.contentForTests = tempFile.testBuf.String()
	}

	rawFileSet := getRawFileSet(ctx.Config())
	if _, exists := rawFileSet.LoadOrStore(hash, rawFileInfo); exists {
		// If a raw file with this hash has already been created delete the temporary file.
		tempFile.remove()
	} else {
		// If this is the first time this hash has been seen then move it from the temporary directory
		// to the raw directory.  If the file already exists in the raw directory assume it has the correct
		// contents.
		absRawPath := absolutePath(rawPath.String())
		_, err := os.Stat(absRawPath)
		if os.IsNotExist(err) {
			tempFile.rename(absRawPath)
		} else if err != nil {
			panic(fmt.Errorf("failed to stat %q: %w", absRawPath, err))
		} else {
			tempFile.remove()
		}
	}

	// Emit a rule to copy the file from raw directory to the final requested location in the output tree.
	// Restat is used to ensure that two different products that produce identical files copied from their
	// own raw directories they don't cause everything downstream to rebuild.
	rule := rawFileCopy
	if executable {
		rule = rawFileCopyExecutable
	}
	ctx.Build(pctx, BuildParams{
		Rule:        rule,
		Input:       rawPath,
		Output:      outputFile,
		Description: "raw " + outputFile.Base(),
		Validations: validations,
	})
}

var (
	rawFileCopy = pctx.AndroidStaticRule("rawFileCopy",
		blueprint.RuleParams{
			Command:     "if ! cmp -s $in $out; then cp $in $out; fi",
			Description: "copy raw file $out",
			Restat:      true,
		})
	rawFileCopyExecutable = pctx.AndroidStaticRule("rawFileCopyExecutable",
		blueprint.RuleParams{
			Command:     "if ! cmp -s $in $out; then cp $in $out; fi && chmod +x $out",
			Description: "copy raw exectuable file $out",
			Restat:      true,
		})
)

type rawFileInfo struct {
	relPath         string
	contentForTests string
}

var rawFileSetKey OnceKey = NewOnceKey("raw file set")

func getRawFileSet(config Config) *syncmap.SyncMap[string, rawFileInfo] {
	return config.Once(rawFileSetKey, func() any {
		return &syncmap.SyncMap[string, rawFileInfo]{}
	}).(*syncmap.SyncMap[string, rawFileInfo])
}

// ContentFromFileRuleForTests returns the content that was passed to a WriteFileRule for use
// in tests.
func ContentFromFileRuleForTests(t *testing.T, ctx *TestContext, params TestingBuildParams) string {
	t.Helper()
	if params.Rule != rawFileCopy && params.Rule != rawFileCopyExecutable {
		t.Errorf("expected params.Rule to be rawFileCopy or rawFileCopyExecutable, was %q", params.Rule)
		return ""
	}

	key := filepath.Base(params.Input.String())
	rawFileSet := getRawFileSet(ctx.Config())
	rawFileInfo, _ := rawFileSet.Load(key)

	return rawFileInfo.contentForTests
}

func rawFilesSingletonFactory() Singleton {
	return &rawFilesSingleton{}
}

type rawFilesSingleton struct{}

func (rawFilesSingleton) GenerateBuildActions(ctx SingletonContext) {
	if ctx.Config().captureBuild {
		// Nothing to do when running in tests, no temporary files were created.
		return
	}
	rawFileSet := getRawFileSet(ctx.Config())
	rawFilesDir := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix)).String()
	absRawFilesDir := absolutePath(rawFilesDir)
	err := filepath.WalkDir(absRawFilesDir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			// Ignore obsolete directories for now.
			return nil
		}

		// Assume the basename of the file is a hash
		key := filepath.Base(path)
		relPath, err := filepath.Rel(absRawFilesDir, path)
		if err != nil {
			return err
		}

		// Check if a file with the same hash was written by this run of soong_build.  If the file was not written,
		// or if a file with the same hash was written but to a different path in the raw directory, then delete it.
		// Checking that the path matches allows changing the structure of the raw directory, for example to increase
		// the sharding.
		rawFileInfo, written := rawFileSet.Load(key)
		if !written || rawFileInfo.relPath != relPath {
			os.Remove(path)
		}
		return nil
	})
	if err != nil {
		panic(fmt.Errorf("failed to clean %q: %w", rawFilesDir, err))
	}
}