1#!/bin/bash 2 3# Copyright (C) 2023 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17set -uo pipefail 18 19# Integration test for verifying generated SBOM for cuttlefish device. 20 21if [ ! -e "build/make/core/Makefile" ]; then 22 echo "$0 must be run from the top of the Android source tree." 23 exit 1 24fi 25 26function setup { 27 tmp_dir="$(mktemp -d tmp.XXXXXX)" 28 trap 'cleanup "${tmp_dir}"' EXIT 29 echo "${tmp_dir}" 30} 31 32function cleanup { 33 tmp_dir="$1"; shift 34 rm -rf "${tmp_dir}" 35} 36 37function run_soong { 38 target_product="$1";shift 39 out_dir="$1"; shift 40 targets="$1"; shift 41 if [ "$#" -ge 1 ]; then 42 apps=$1; shift 43 TARGET_PRODUCT="${target_product}" TARGET_BUILD_VARIANT=userdebug OUT_DIR="${out_dir}" TARGET_BUILD_UNBUNDLED=true TARGET_BUILD_APPS=$apps build/soong/soong_ui.bash --make-mode ${targets} 44 else 45 TARGET_PRODUCT="${target_product}" TARGET_BUILD_VARIANT=userdebug OUT_DIR="${out_dir}" build/soong/soong_ui.bash --make-mode ${targets} 46 fi 47} 48 49function diff_files { 50 file_list_file="$1"; shift 51 files_in_spdx_file="$1"; shift 52 partition_name="$1"; shift 53 exclude= 54 if [ -v 'diff_excludes[$partition_name]' ]; then 55 exclude=${diff_excludes[$partition_name]} 56 fi 57 58 diff "$file_list_file" "$files_in_spdx_file" $exclude 59 if [ $? != "0" ]; then 60 echo Found diffs in $f and SBOM. 61 exit 1 62 else 63 echo No diffs. 64 fi 65} 66 67function test_sbom_aosp_cf_x86_64_phone { 68 # Setup 69 out_dir="$(setup)" 70 71 # Test 72 # m droid, build sbom later in case additional dependencies might be built and included in partition images. 73 run_soong "aosp_cf_x86_64_phone" "${out_dir}" "droid dump.erofs lz4" 74 75 product_out=$out_dir/target/product/vsoc_x86_64 76 sbom_test=$product_out/sbom_test 77 mkdir -p $sbom_test 78 cp $product_out/*.img $sbom_test 79 80 # m sbom 81 run_soong "aosp_cf_x86_64_phone" "${out_dir}" sbom 82 83 # Generate installed file list from .img files in PRODUCT_OUT 84 dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs 85 lz4=$out_dir/host/linux-x86/bin/lz4 86 87 declare -A diff_excludes 88 diff_excludes[vendor]="-I /vendor/lib64/libkeystore2_crypto.so" 89 diff_excludes[system]="\ 90 -I /bin \ 91 -I /bugreports \ 92 -I /cache \ 93 -I /d \ 94 -I /etc \ 95 -I /init \ 96 -I /odm/app \ 97 -I /odm/bin \ 98 -I /odm_dlkm/etc \ 99 -I /odm/etc \ 100 -I /odm/firmware \ 101 -I /odm/framework \ 102 -I /odm/lib \ 103 -I /odm/lib64 \ 104 -I /odm/overlay \ 105 -I /odm/priv-app \ 106 -I /odm/usr \ 107 -I /sdcard \ 108 -I /system/lib64/android.hardware.confirmationui@1.0.so \ 109 -I /system/lib64/android.hardware.confirmationui-V1-ndk.so \ 110 -I /system/lib64/android.hardware.keymaster@4.1.so \ 111 -I /system/lib64/android.hardware.security.rkp-V3-ndk.so \ 112 -I /system/lib64/android.hardware.security.sharedsecret-V1-ndk.so \ 113 -I /system/lib64/android.security.compat-ndk.so \ 114 -I /system/lib64/libkeymaster4_1support.so \ 115 -I /system/lib64/libkeymaster4support.so \ 116 -I /system/lib64/libkeymint.so \ 117 -I /system/lib64/libkeystore2_aaid.so \ 118 -I /system/lib64/libkeystore2_apc_compat.so \ 119 -I /system/lib64/libkeystore2_crypto.so \ 120 -I /system/lib64/libkeystore-attestation-application-id.so \ 121 -I /system/lib64/libkm_compat_service.so \ 122 -I /system/lib64/libkm_compat.so \ 123 -I /system/lib64/vndk-29 \ 124 -I /system/lib64/vndk-sp-29 \ 125 -I /system/lib/vndk-29 \ 126 -I /system/lib/vndk-sp-29 \ 127 -I /system/usr/icu \ 128 -I /vendor_dlkm/etc" 129 130 # Example output of dump.erofs is as below, and the data used in the test start 131 # at line 11. Column 1 is inode id, column 2 is inode type and column 3 is name. 132 # Each line is captured in variable "entry", awk is used to get type and name. 133 # Output of dump.erofs: 134 # File : / 135 # Size: 160 On-disk size: 160 directory 136 # NID: 39 Links: 10 Layout: 2 Compression ratio: 100.00% 137 # Inode size: 64 Extent size: 0 Xattr size: 16 138 # Uid: 0 Gid: 0 Access: 0755/rwxr-xr-x 139 # Timestamp: 2023-02-14 01:15:54.000000000 140 # 141 # NID TYPE FILENAME 142 # 39 2 . 143 # 39 2 .. 144 # 47 2 app 145 # 1286748 2 bin 146 # 1286754 2 etc 147 # 5304814 2 lib 148 # 5309056 2 lib64 149 # 5309130 2 media 150 # 5388910 2 overlay 151 # 5479537 2 priv-app 152 EROFS_IMAGES="\ 153 $sbom_test/product.img \ 154 $sbom_test/system.img \ 155 $sbom_test/system_ext.img \ 156 $sbom_test/system_dlkm.img \ 157 $sbom_test/system_other.img \ 158 $sbom_test/odm.img \ 159 $sbom_test/odm_dlkm.img \ 160 $sbom_test/vendor.img \ 161 $sbom_test/vendor_dlkm.img" 162 for f in $EROFS_IMAGES; do 163 partition_name=$(basename $f | cut -d. -f1) 164 file_list_file="${sbom_test}/sbom-${partition_name}-files.txt" 165 files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt" 166 rm "$file_list_file" > /dev/null 2>&1 || true 167 all_dirs="/" 168 while [ ! -z "$all_dirs" ]; do 169 dir=$(echo "$all_dirs" | cut -d ' ' -f1) 170 all_dirs=$(echo "$all_dirs" | cut -d ' ' -f1 --complement -s) 171 entries=$($dump_erofs --ls --path "$dir" $f | tail -n +11) 172 while read -r entry; do 173 inode_type=$(echo $entry | awk -F ' ' '{print $2}') 174 name=$(echo $entry | awk -F ' ' '{print $3}') 175 case $inode_type in 176 "2") # directory 177 all_dirs=$(echo "$all_dirs $dir/$name" | sed 's/^\s*//') 178 ;; 179 "1"|"7") # 1: file, 7: symlink 180 ( 181 if [ "$partition_name" != "system" ]; then 182 # system partition is mounted to /, not to prepend partition name. 183 printf %s "/$partition_name" 184 fi 185 echo "$dir/$name" | sed 's#^//#/#' 186 ) >> "$file_list_file" 187 ;; 188 esac 189 done <<< "$entries" 190 done 191 sort -n -o "$file_list_file" "$file_list_file" 192 193 grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file" 194 if [ "$partition_name" = "system" ]; then 195 # system partition is mounted to /, so include FileName starts with /root/ too. 196 grep "FileName: /root/" $product_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_spdx_file" 197 fi 198 sort -n -o "$files_in_spdx_file" "$files_in_spdx_file" 199 200 echo ============ Diffing files in $f and SBOM 201 diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" 202 done 203 204 RAMDISK_IMAGES="$product_out/ramdisk.img" 205 for f in $RAMDISK_IMAGES; do 206 partition_name=$(basename $f | cut -d. -f1) 207 file_list_file="${sbom_test}/sbom-${partition_name}-files.txt" 208 files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt" 209 # lz4 decompress $f to stdout 210 # cpio list all entries like ls -l 211 # grep filter normal files and symlinks 212 # awk get entry names 213 # sed remove partition name from entry names 214 $lz4 -c -d $f | cpio -tv 2>/dev/null | grep '^[-l]' | awk -F ' ' '{print $9}' | sed "s:^:/$partition_name/:" | sort -n > "$file_list_file" 215 216 grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file" 217 218 echo ============ Diffing files in $f and SBOM 219 diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" 220 done 221 222 # Teardown 223 cleanup "${out_dir}" 224} 225 226function test_sbom_unbundled_apex { 227 # Setup 228 out_dir="$(setup)" 229 230 # run_soong to build com.android.adbd.apex 231 run_soong "module_arm64" "${out_dir}" "sbom deapexer" "com.android.adbd" 232 233 deapexer=${out_dir}/host/linux-x86/bin/deapexer 234 debugfs=${out_dir}/host/linux-x86/bin/debugfs_static 235 apex_file=${out_dir}/target/product/module_arm64/system/apex/com.android.adbd.apex 236 echo "============ Diffing files in $apex_file and SBOM" 237 set +e 238 # deapexer prints the list of all files and directories 239 # sed extracts the file/directory names 240 # grep removes directories 241 # sed removes leading ./ in file names 242 diff -I /system/apex/com.android.adbd.apex -I apex_manifest.pb \ 243 <($deapexer --debugfs_path=$debugfs list --extents ${apex_file} | sed -E 's#(.*) \[.*\]$#\1#' | grep -v "/$" | sed -E 's#^\./(.*)#\1#' | sort -n) \ 244 <(grep '"fileName": ' ${apex_file}.spdx.json | sed -E 's/.*"fileName": "(.*)",/\1/' | sort -n ) 245 246 if [ $? != "0" ]; then 247 echo "Diffs found in $apex_file and SBOM" 248 exit 1 249 else 250 echo "No diffs." 251 fi 252 set -e 253 254 # Teardown 255 cleanup "${out_dir}" 256} 257 258function test_sbom_unbundled_apk { 259 # Setup 260 out_dir="$(setup)" 261 262 # run_soong to build Browser2.apk 263 run_soong "module_arm64" "${out_dir}" "sbom" "Browser2" 264 265 sbom_file=${out_dir}/target/product/module_arm64/system/product/app/Browser2/Browser2.apk.spdx.json 266 echo "============ Diffing files in Browser2.apk and SBOM" 267 set +e 268 # There is only one file in SBOM of APKs 269 diff \ 270 <(echo "/system/product/app/Browser2/Browser2.apk" ) \ 271 <(grep '"fileName": ' ${sbom_file} | sed -E 's/.*"fileName": "(.*)",/\1/' ) 272 273 if [ $? != "0" ]; then 274 echo "Diffs found in $sbom_file" 275 exit 1 276 else 277 echo "No diffs." 278 fi 279 set -e 280 281 # Teardown 282 cleanup "${out_dir}" 283} 284 285test_sbom_aosp_cf_x86_64_phone 286test_sbom_unbundled_apex 287test_sbom_unbundled_apk