// Copyright (C) 2024 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use std::{ cell::OnceCell, collections::BTreeSet, fs::{copy, read_dir, read_to_string, remove_dir_all, remove_file, rename, write}, os::unix::fs::symlink, path::{Path, PathBuf}, process::Command, }; use anyhow::{anyhow, bail, Context, Result}; use crate_config::CrateConfig; use glob::glob; use google_metadata::GoogleMetadata; use itertools::Itertools; use license_checker::{find_licenses, LicenseState}; use name_and_version::NamedAndVersioned; use rooted_path::RootedPath; use semver::Version; use test_mapping::TestMapping; use crate::{ android_bp::run_cargo_embargo, copy_dir, crate_type::Crate, ensure_exists_and_empty, license::{most_restrictive_type, update_module_license_files}, patch::Patch, pseudo_crate::{CargoVendorClean, PseudoCrate}, SuccessOrError, }; #[derive(Debug)] pub struct ManagedCrate { /// The crate with Android customizations, a subdirectory of crates/. android_crate: Crate, config: OnceCell, extra: State, } #[derive(Debug)] pub struct New {} /// Crate state indicating we have run `cargo vendor` on the pseudo-crate. #[derive(Debug)] pub struct Vendored { /// The vendored copy of the crate, from running `cargo vendor` vendored_crate: Crate, } /// Crate state indicating we have copied the vendored code to a temporary build /// directory, copied over Android customizations, and applied patches. #[derive(Debug)] pub struct CopiedAndPatched { /// The vendored copy of the crate, from running `cargo vendor` vendored_crate: Crate, /// The license terms and associated license files. licenses: LicenseState, } pub trait ManagedCrateState {} impl ManagedCrateState for New {} impl ManagedCrateState for Vendored {} impl ManagedCrateState for CopiedAndPatched {} static CUSTOMIZATIONS: &[&str] = &[ "*.bp", "*.bp.fragment", "*.mk", "android_config.toml", "android", "cargo_embargo.json", "patches", "METADATA", "TEST_MAPPING", ]; impl ManagedCrate { pub fn name(&self) -> &str { self.android_crate.name() } pub fn android_version(&self) -> &Version { self.android_crate.version() } fn android_crate_path(&self) -> &RootedPath { self.android_crate.path() } pub fn config(&self) -> &CrateConfig { self.config.get_or_init(|| { CrateConfig::read(self.android_crate_path().abs()).unwrap_or_else(|e| { panic!( "Failed to read crate config {}/{}: {}", self.android_crate_path(), crate_config::CONFIG_FILE_NAME, e ) }) }) } fn patch_dir(&self) -> RootedPath { self.android_crate_path().join("patches").unwrap() } fn temporary_build_directory(&self) -> RootedPath { self.android_crate .path() .with_same_root("out/rust-crate-temporary-build") .unwrap() .join(self.name()) .unwrap() } pub fn patches(&self) -> Result> { let mut patches = Vec::new(); let patch_dir = self.patch_dir(); if patch_dir.abs().exists() { for entry in read_dir(&patch_dir).context(format!("Failed to read_dir {}", patch_dir))? { let entry = entry?; if entry.file_name() == "Android.bp.patch" || entry.file_name() == "Android.bp.diff" || entry.file_name() == "rules.mk.diff" { continue; } patches.push(entry.path()); } } patches.sort(); Ok(patches) } pub fn recontextualize_patches(&self) -> Result<()> { let output = Command::new("git") .args(["status", "--porcelain", "."]) .current_dir(self.android_crate_path()) .output()? .success_or_error()?; if !output.stdout.is_empty() { return Err(anyhow!( "Crate directory {} has uncommitted changes", self.android_crate_path() )); } let mut new_patch_contents = Vec::new(); for patch in self.patches()? { println!("Recontextualizing {}", patch.display()); // Patch files can be in many different formats, and patch is very // forgiving. We might be able to use "git apply -R --directory=crates/foo" // once we have everything in the same format. Command::new("patch") .args(["-R", "-p1", "-l", "--reject-file=-", "--no-backup-if-mismatch", "-i"]) .arg(&patch) .current_dir(self.android_crate_path()) .spawn()? .wait()? .success_or_error()?; Command::new("git") .args(["add", "."]) .current_dir(self.android_crate_path()) .spawn()? .wait()? .success_or_error()?; let output = Command::new("git") .args([ "diff", format!("--relative=crates/{}", self.name()).as_str(), "-p", "--stat", "-R", "--staged", ".", ]) .current_dir(self.android_crate_path()) .output()? .success_or_error()?; Command::new("git") .args(["restore", "--staged", "."]) .current_dir(self.android_crate_path()) .spawn()? .wait()? .success_or_error()?; Command::new("git") .args(["restore", "."]) .current_dir(self.android_crate_path()) .spawn()? .wait()? .success_or_error()?; Command::new("git") .args(["clean", "-f", "."]) .current_dir(self.android_crate_path()) .spawn()? .wait()? .success_or_error()?; let patch_contents = read_to_string(&patch)?; let parsed = Patch::parse(&patch_contents); new_patch_contents.push((patch, parsed.reassemble(&output.stdout))); } for (path, contents) in new_patch_contents { write(path, contents)?; } Ok(()) } } impl ManagedCrate { pub fn new(android_crate: Crate) -> Self { ManagedCrate { android_crate, config: OnceCell::new(), extra: New {} } } fn into_vendored( self, pseudo_crate: &PseudoCrate, ) -> Result> { let vendored_crate = Crate::from(pseudo_crate.vendored_dir_for(self.android_crate.name())?.clone())?; Ok(ManagedCrate { android_crate: self.android_crate, config: self.config, extra: Vendored { vendored_crate }, }) } pub fn regenerate( self, pseudo_crate: &PseudoCrate, run_cargo_embargo: bool, ) -> Result> { let regenerated = self.into_vendored(pseudo_crate)?.regenerate(run_cargo_embargo)?; Ok(regenerated) } } impl ManagedCrate { fn into_copied_and_patched( self, licenses: LicenseState, ) -> Result> { Ok(ManagedCrate { android_crate: self.android_crate, config: self.config, extra: CopiedAndPatched { vendored_crate: self.extra.vendored_crate, licenses }, }) } /// Makes a clean copy of the vendored crates in a temporary build directory. fn copy_to_temporary_build_directory(&self) -> Result<()> { let build_dir = self.temporary_build_directory(); ensure_exists_and_empty(&build_dir)?; remove_dir_all(&build_dir).context(format!("Failed to remove {}", build_dir))?; copy_dir(self.extra.vendored_crate.path(), &build_dir).context(format!( "Failed to copy {} to {}", self.extra.vendored_crate.path(), self.temporary_build_directory() ))?; Ok(()) } /// Copies Android-specific customizations, such as Android.bp, METADATA, etc. from /// the managed crate directory to the temporary build directory. These are things that /// we have added and know need to be preserved. fn copy_customizations(&self) -> Result<()> { let dest_dir = self.temporary_build_directory(); for pattern in CUSTOMIZATIONS { let full_pattern = self.android_crate.path().join(pattern)?; for entry in glob( full_pattern .abs() .to_str() .ok_or(anyhow!("Failed to convert path {} to str", full_pattern))?, )? { let entry = entry?; let filename = entry .file_name() .context(format!("Failed to get file name for {}", entry.display()))? .to_os_string(); if entry.is_dir() { copy_dir(&entry, &dest_dir.join(filename)?).context(format!( "Failed to copy {} to {}", entry.display(), dest_dir ))?; } else { let dest_file = dest_dir.join(&filename)?; if dest_file.abs().exists() { return Err(anyhow!("Destination file {} exists", dest_file)); } copy(&entry, dest_dir.join(filename)?).context(format!( "Failed to copy {} to {}", entry.display(), dest_dir ))?; } } } Ok(()) } /// Applies patches from the patches/ directory in a deterministic order. /// /// Patches to the Android.bp file are excluded as they are applied /// later, by cargo_embargo fn apply_patches(&self) -> Result<()> { for patch in self.patches()? { Command::new("patch") .args(["-p1", "-l", "--no-backup-if-mismatch", "-i"]) .arg(&patch) .current_dir(self.temporary_build_directory()) .output()? .success_or_error() .context(format!( "Failed to apply patch file {}", patch.file_name().unwrap_or_default().to_string_lossy() ))?; } Ok(()) } pub fn regenerate(self, run_cargo_embargo: bool) -> Result> { self.copy_to_temporary_build_directory()?; self.copy_customizations()?; // Delete stuff that we don't want to keep around, as specified in the // android_config.toml for deletion in self.config().deletions() { let dir = self.temporary_build_directory().join(deletion)?; if dir.abs().is_dir() { remove_dir_all(dir)?; } else { remove_file(dir)?; } } self.apply_patches()?; let licenses = find_licenses( self.temporary_build_directory(), self.name(), self.android_crate.license(), )?; let regenerated = self.into_copied_and_patched(licenses)?; regenerated.regenerate(run_cargo_embargo)?; Ok(regenerated) } } impl ManagedCrate { pub fn regenerate(&self, run_cargo_embargo: bool) -> Result<()> { // License logic must happen AFTER applying patches, because we use patches // to add missing license files. It must also happen BEFORE cargo_embargo, // because cargo_embargo needs to put license information in the Android.bp. self.update_license_files()?; // If the crate has an out/ directory, that means we cannot skip running cargo_embargo. // The out/ directory is used by cargo_embargo to preserve files generated // by a build script. Soong can't run build scripts, so we rely on cargo_embargo // to run `cargo build` and save the intermediates. if run_cargo_embargo || self.android_crate_path().abs().join("out").exists() { self.run_cargo_embargo()?; } self.update_metadata()?; self.fix_test_mapping()?; // Fails on dangling symlinks, which happens when we run on the log crate. checksum::generate(self.temporary_build_directory())?; let android_crate_dir = self.android_crate.path(); remove_dir_all(android_crate_dir)?; rename(self.temporary_build_directory(), android_crate_dir)?; Ok(()) } fn update_license_files(&self) -> Result<()> { // For every chosen license, we must be able to find an associated // license file. if !self.extra.licenses.unsatisfied.is_empty() { bail!( "Could not find license files for some licenses: {:?}", self.extra.licenses.unsatisfied ); } // SOME license must apply to the code. If none apply, that's an error. if self.extra.licenses.satisfied.is_empty() { bail!("No license terms were found for this crate"); } // There should be no license files for terms not mentioned in Cargo.toml // license expression. For example, if Cargo.toml says "Apache-2.0" and we // find a file named "LICENSE-MIT", that suggests that something suspicious might // be going on. if !self.extra.licenses.unexpected.is_empty() { bail!( "Found unexpected license files that don't correspond to any terms of {}: {:?}", self.android_crate.license().unwrap_or(""), self.extra.licenses.unexpected ); } // Per go/thirdparty/licenses#multiple: // "Delete any LICENSE files or license texts that were not selected and are not in use." for unneeded_license_file in self.extra.licenses.unneeded.values() { remove_file(self.temporary_build_directory().abs().join(unneeded_license_file))?; } // Per http://go/thirdpartyreviewers#license: // "There must be a file called LICENSE containing an allowed third party license." let license_files = self .extra .licenses .satisfied .values() .map(|path| path.as_path()) .collect::>(); let canonical_license_file_name = Path::new("LICENSE"); let canonical_license_path = self.temporary_build_directory().abs().join("LICENSE"); if license_files.len() == 1 { // If there's a single applicable license file, it must either // be called LICENSE, or else we need to symlink LICENSE to it. let license_file = license_files.first().unwrap(); if *license_file != canonical_license_file_name { if canonical_license_path.exists() { // TODO: Maybe just blindly delete LICENSE and replace it with a symlink. // Currently, we have to use a deletions config in android_config.toml // to remove it. bail!("Found a single license file {}, but we can't create a symlink to it because a file named LICENSE already exists", license_file.display()); } else { symlink(license_file, canonical_license_path)?; } } } else { // We found multiple license files. Per go/thirdparty/licenses#multiple they should // be concatenated into a single LICENSE file, separated by dashed line dividers. let mut license_contents = Vec::new(); for file in license_files { license_contents .push(read_to_string(self.temporary_build_directory().abs().join(file))?); } // TODO: Maybe warn if LICENSE file exists. But there are cases where we can't just delete it // because it contains *some* of the necessary license texts. write( canonical_license_path, license_contents.iter().join("\n\n------------------\n\n"), )?; } update_module_license_files(&self.temporary_build_directory(), &self.extra.licenses)?; Ok(()) } /// Runs cargo_embargo on the crate in the temporary build directory. /// /// Because cargo_embargo can modify Cargo.lock files, we save them, if present. fn run_cargo_embargo(&self) -> Result<()> { let temporary_build_path = self.temporary_build_directory(); let cargo_lock = temporary_build_path.join("Cargo.lock")?; let saved_cargo_lock = temporary_build_path.join("Cargo.lock.saved")?; if cargo_lock.abs().exists() { rename(&cargo_lock, &saved_cargo_lock)?; } run_cargo_embargo(&temporary_build_path)? .success_or_error() .context(format!("cargo_embargo execution failed for {}", self.name()))?; if cargo_lock.abs().exists() { remove_file(&cargo_lock)?; } if saved_cargo_lock.abs().exists() { rename(saved_cargo_lock, cargo_lock)?; } Ok(()) } /// Updates the METADATA file in the temporary build directory. fn update_metadata(&self) -> Result<()> { let mut metadata = GoogleMetadata::try_from(self.temporary_build_directory().join("METADATA").unwrap())?; metadata.update( self.name(), self.extra.vendored_crate.version().to_string(), self.extra.vendored_crate.description(), most_restrictive_type(&self.extra.licenses), ); metadata.write()?; Ok(()) } /// Updates the TEST_MAPPING file in the temporary build directory, by /// removing deleted tests and adding new tests as post-submits. fn fix_test_mapping(&self) -> Result<()> { let mut tm = TestMapping::read(self.temporary_build_directory())?; let mut changed = tm.fix_import_paths(); changed |= tm.add_new_tests_to_postsubmit()?; changed |= tm.remove_unknown_tests()?; // TODO: Add an option to fix up the reverse dependencies. if changed { println!("Updating TEST_MAPPING for {}", self.name()); tm.write()?; } Ok(()) } }