1#!/bin/bash 2 3# Copyright (C) 2023 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17set -uo pipefail 18 19# Integration test for verifying generated SBOM for cuttlefish device. 20 21if [ ! -e "build/make/core/Makefile" ]; then 22 echo "$0 must be run from the top of the Android source tree." 23 exit 1 24fi 25 26function setup { 27 tmp_dir="$(mktemp -d tmp.XXXXXX)" 28 trap 'cleanup "${tmp_dir}"' EXIT 29 echo "${tmp_dir}" 30} 31 32function cleanup { 33 tmp_dir="$1"; shift 34 rm -rf "${tmp_dir}" 35} 36 37function run_soong { 38 local out_dir="$1"; shift 39 local targets="$1"; shift 40 if [ "$#" -ge 1 ]; then 41 local apps=$1; shift 42 TARGET_PRODUCT="${target_product}" TARGET_RELEASE="${target_release}" TARGET_BUILD_VARIANT="${target_build_variant}" OUT_DIR="${out_dir}" TARGET_BUILD_UNBUNDLED=true TARGET_BUILD_APPS=$apps \ 43 build/soong/soong_ui.bash --make-mode ${targets} 44 else 45 TARGET_PRODUCT="${target_product}" TARGET_RELEASE="${target_release}" TARGET_BUILD_VARIANT="${target_build_variant}" OUT_DIR="${out_dir}" \ 46 build/soong/soong_ui.bash --make-mode ${targets} 47 fi 48} 49 50function diff_files { 51 local file_list_file="$1"; shift 52 local files_in_spdx_file="$1"; shift 53 local partition_name="$1"; shift 54 local exclude="$1"; shift 55 56 diff "$file_list_file" "$files_in_spdx_file" $exclude 57 if [ $? != "0" ]; then 58 echo Found diffs in $f and SBOM. 59 exit 1 60 else 61 echo No diffs. 62 fi 63} 64 65function test_sbom_aosp_cf_x86_64_phone { 66 # Setup 67 out_dir="$(setup)" 68 69 # Test 70 # m droid, build sbom later in case additional dependencies might be built and included in partition images. 71 run_soong "${out_dir}" "droid dump.erofs lz4" 72 73 soong_sbom_out=$out_dir/soong/sbom/$target_product 74 product_out=$out_dir/target/product/vsoc_x86_64 75 sbom_test=$product_out/sbom_test 76 mkdir -p $sbom_test 77 cp $product_out/*.img $sbom_test 78 79 # m sbom 80 run_soong "${out_dir}" "sbom" 81 82 # Generate installed file list from .img files in PRODUCT_OUT 83 dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs 84 lz4=$out_dir/host/linux-x86/bin/lz4 85 86 declare -A diff_excludes 87 diff_excludes[system]="\ 88 -I /etc/NOTICE.xml.gz \ 89 -I /odm_dlkm/etc \ 90 -I /vendor_dlkm/etc" 91 92 # Example output of dump.erofs is as below, and the data used in the test start 93 # at line 11. Column 1 is inode id, column 2 is inode type and column 3 is name. 94 # Each line is captured in variable "entry", awk is used to get type and name. 95 # Output of dump.erofs: 96 # File : / 97 # Size: 160 On-disk size: 160 directory 98 # NID: 39 Links: 10 Layout: 2 Compression ratio: 100.00% 99 # Inode size: 64 Extent size: 0 Xattr size: 16 100 # Uid: 0 Gid: 0 Access: 0755/rwxr-xr-x 101 # Timestamp: 2023-02-14 01:15:54.000000000 102 # 103 # NID TYPE FILENAME 104 # 39 2 . 105 # 39 2 .. 106 # 47 2 app 107 # 1286748 2 bin 108 # 1286754 2 etc 109 # 5304814 2 lib 110 # 5309056 2 lib64 111 # 5309130 2 media 112 # 5388910 2 overlay 113 # 5479537 2 priv-app 114 EROFS_IMAGES="\ 115 $sbom_test/product.img \ 116 $sbom_test/system.img \ 117 $sbom_test/system_ext.img \ 118 $sbom_test/system_dlkm.img \ 119 $sbom_test/system_other.img \ 120 $sbom_test/odm.img \ 121 $sbom_test/odm_dlkm.img \ 122 $sbom_test/vendor.img \ 123 $sbom_test/vendor_dlkm.img" 124 for f in $EROFS_IMAGES; do 125 partition_name=$(basename $f | cut -d. -f1) 126 file_list_file="${sbom_test}/sbom-${partition_name}-files.txt" 127 files_in_soong_spdx_file="${sbom_test}/soong-sbom-${partition_name}-files-in-spdx.txt" 128 rm "$file_list_file" > /dev/null 2>&1 || true 129 all_dirs="/" 130 while [ ! -z "$all_dirs" ]; do 131 dir=$(echo "$all_dirs" | cut -d ' ' -f1) 132 all_dirs=$(echo "$all_dirs" | cut -d ' ' -f1 --complement -s) 133 entries=$($dump_erofs --ls --path "$dir" $f | tail -n +11) 134 while read -r entry; do 135 inode_type=$(echo $entry | awk -F ' ' '{print $2}') 136 name=$(echo $entry | awk -F ' ' '{print $3}') 137 case $inode_type in 138 "2") # directory 139 all_dirs=$(echo "$all_dirs $dir/$name" | sed 's/^\s*//') 140 ;; 141 "1"|"7") # 1: file, 7: symlink 142 ( 143 if [ "$partition_name" != "system" ]; then 144 # system partition is mounted to /, not to prepend partition name. 145 printf %s "/$partition_name" 146 fi 147 echo "$dir/$name" | sed 's#^//#/#' 148 ) >> "$file_list_file" 149 ;; 150 esac 151 done <<< "$entries" 152 done 153 sort -n -o "$file_list_file" "$file_list_file" 154 155 # Diff the file list from image and file list in SBOM created by Soong 156 grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_soong_spdx_file" 157 if [ "$partition_name" = "system" ]; then 158 # system partition is mounted to /, so include FileName starts with /root/ too. 159 grep "FileName: /root/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_soong_spdx_file" 160 fi 161 sort -n -o "$files_in_soong_spdx_file" "$files_in_soong_spdx_file" 162 163 echo ============ Diffing files in $f and SBOM created by Soong 164 exclude= 165 if [ -v 'diff_excludes[$partition_name]' ]; then 166 exclude=${diff_excludes[$partition_name]} 167 fi 168 diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" "$exclude" 169 done 170 171 RAMDISK_IMAGES="$product_out/ramdisk.img" 172 for f in $RAMDISK_IMAGES; do 173 partition_name=$(basename $f | cut -d. -f1) 174 file_list_file="${sbom_test}/sbom-${partition_name}-files.txt" 175 files_in_soong_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-soong-spdx.txt" 176 # lz4 decompress $f to stdout 177 # cpio list all entries like ls -l 178 # grep filter normal files and symlinks 179 # awk get entry names 180 # sed remove partition name from entry names 181 $lz4 -c -d $f | cpio -tv 2>/dev/null | grep '^[-l]' | awk -F ' ' '{print $9}' | sed "s:^:/$partition_name/:" | sort -n > "$file_list_file" 182 183 grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_soong_spdx_file" 184 185 echo ============ Diffing files in $f and SBOM created by Soong 186 diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" "" 187 done 188 189 verify_package_verification_code "$soong_sbom_out/sbom.spdx" 190 191 verify_packages_licenses "$soong_sbom_out/sbom.spdx" 192 193 # Teardown 194 cleanup "${out_dir}" 195} 196 197function verify_package_verification_code { 198 local sbom_file="$1"; shift 199 200 local -a file_checksums 201 local package_product_found= 202 while read -r line; 203 do 204 if grep -q 'PackageVerificationCode' <<<"$line" 205 then 206 package_product_found=true 207 fi 208 if [ -n "$package_product_found" ] 209 then 210 if grep -q 'FileChecksum' <<< "$line" 211 then 212 checksum=$(echo $line | sed 's/^.*: //') 213 file_checksums+=("$checksum") 214 fi 215 fi 216 done <<< "$(grep -E 'PackageVerificationCode|FileChecksum' $sbom_file)" 217 IFS=$'\n' file_checksums=($(sort <<<"${file_checksums[*]}")); unset IFS 218 IFS= expected_package_verification_code=$(printf "${file_checksums[*]}" | sha1sum | sed 's/[[:space:]]*-//'); unset IFS 219 220 actual_package_verification_code=$(grep PackageVerificationCode $sbom_file | sed 's/PackageVerificationCode: //g') 221 if [ $actual_package_verification_code = $expected_package_verification_code ] 222 then 223 echo "Package verification code is correct." 224 else 225 echo "Unexpected package verification code." 226 exit 1 227 fi 228} 229 230function verify_packages_licenses { 231 local sbom_file="$1"; shift 232 233 num_of_packages=$(grep 'PackageName:' $sbom_file | wc -l) 234 num_of_declared_licenses=$(grep 'PackageLicenseDeclared:' $sbom_file | wc -l) 235 if [ "$num_of_packages" = "$num_of_declared_licenses" ] 236 then 237 echo "Number of packages with declared license is correct." 238 else 239 echo "Number of packages with declared license is WRONG." 240 exit 1 241 fi 242 243 # PRODUCT and 6 prebuilt packages have "PackageLicenseDeclared: NOASSERTION" 244 # All other packages have declared licenses 245 num_of_packages_with_noassertion_license=$(grep 'PackageLicenseDeclared: NOASSERTION' $sbom_file | wc -l) 246 if [ $num_of_packages_with_noassertion_license = 13 ] 247 then 248 echo "Number of packages with NOASSERTION license is correct." 249 else 250 echo "Number of packages with NOASSERTION license is WRONG." 251 exit 1 252 fi 253 254 num_of_files=$(grep 'FileName:' $sbom_file | wc -l) 255 num_of_concluded_licenses=$(grep 'LicenseConcluded:' $sbom_file | wc -l) 256 if [ "$num_of_files" = "$num_of_concluded_licenses" ] 257 then 258 echo "Number of files with concluded license is correct." 259 else 260 echo "Number of files with concluded license is WRONG." 261 exit 1 262 fi 263} 264 265function test_sbom_unbundled_apex { 266 # Setup 267 out_dir="$(setup)" 268 269 # run_soong to build com.android.adbd.apex 270 run_soong "${out_dir}" "sbom deapexer" "com.android.adbd" 271 272 deapexer=${out_dir}/host/linux-x86/bin/deapexer 273 debugfs=${out_dir}/host/linux-x86/bin/debugfs_static 274 apex_file=${out_dir}/target/product/module_arm64/system/apex/com.android.adbd.apex 275 echo "============ Diffing files in $apex_file and SBOM" 276 set +e 277 # deapexer prints the list of all files and directories 278 # sed extracts the file/directory names 279 # grep removes directories 280 # sed removes leading ./ in file names 281 diff -I /system/apex/com.android.adbd.apex -I apex_manifest.pb \ 282 <($deapexer --debugfs_path=$debugfs list --extents ${apex_file} | sed -E 's#(.*) \[.*\]$#\1#' | grep -v "/$" | sed -E 's#^\./(.*)#\1#' | sort -n) \ 283 <(grep '"fileName": ' ${apex_file}.spdx.json | sed -E 's/.*"fileName": "(.*)",/\1/' | sort -n ) 284 285 if [ $? != "0" ]; then 286 echo "Diffs found in $apex_file and SBOM" 287 exit 1 288 else 289 echo "No diffs." 290 fi 291 set -e 292 293 # Teardown 294 cleanup "${out_dir}" 295} 296 297function test_sbom_unbundled_apk { 298 # Setup 299 out_dir="$(setup)" 300 301 # run_soong to build Browser2.apk 302 run_soong "${out_dir}" "sbom" "Browser2" 303 304 sbom_file=${out_dir}/target/product/module_arm64/system/product/app/Browser2/Browser2.apk.spdx.json 305 echo "============ Diffing files in Browser2.apk and SBOM" 306 set +e 307 # There is only one file in SBOM of APKs 308 diff \ 309 <(echo "/system/product/app/Browser2/Browser2.apk" ) \ 310 <(grep '"fileName": ' ${sbom_file} | sed -E 's/.*"fileName": "(.*)",/\1/' ) 311 312 if [ $? != "0" ]; then 313 echo "Diffs found in $sbom_file" 314 exit 1 315 else 316 echo "No diffs." 317 fi 318 set -e 319 320 # Teardown 321 cleanup "${out_dir}" 322} 323 324target_product=aosp_cf_x86_64_phone 325target_release=trunk_staging 326target_build_variant=eng 327for i in "$@"; do 328 case $i in 329 TARGET_PRODUCT=*) 330 target_product=${i#*=} 331 shift 332 ;; 333 TARGET_RELEASE=*) 334 target_release=${i#*=} 335 shift 336 ;; 337 TARGET_BUILD_VARIANT=*) 338 target_build_variant=${i#*=} 339 shift 340 ;; 341 *) 342 echo "Unknown command line arguments: $i" 343 exit 1 344 ;; 345 esac 346done 347 348echo "target product: $target_product, target_release: $target_release, target build variant: $target_build_variant" 349case $target_product in 350 aosp_cf_x86_64_phone) 351 test_sbom_aosp_cf_x86_64_phone 352 ;; 353 module_arm64) 354 test_sbom_unbundled_apex 355 test_sbom_unbundled_apk 356 ;; 357 *) 358 echo "Unknown TARGET_PRODUCT: $target_product" 359 exit 1 360 ;; 361esac