1#!/bin/bash 2# Copyright 2020 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# Due to crbug.com/1081332, we need to update AFDO metadata 7# manually. This script performs a few checks and generates a 8# new kernel_afdo.json file, which can then be submitted. 9# 10 11USAGE=" 12Usage: $(basename "$0") [--noupload|-upload] [main|beta|stable|all] [--help] 13 14Description: 15 The script takes one optional argument which is the channel where we want 16to update the kernel afdo and creates a commit (or commits with \"all\" 17channels) in the corresponding branch. 18 No arguments defaults to \"all\". 19 Follow the prompt to upload the changes. 20 NO CLEAN-UP NEEDED. The script ignores any local changes and keeps 21the current branch unchanged. 22 23 Args: 24 --help Show this help. 25 --upload Upload CLs when the update succeeded (default). 26 --noupload Do not upload CLs. Instead, print the upload commands. 27 main|beta|stable Update metadata only on the specified channel. 28" 29 30set -eu 31set -o pipefail 32 33AMD_GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel 34ARM_GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel/arm 35AMD_KVERS="4.14 4.19 5.4 5.10" 36ARM_KVERS="5.15" 37failed_channels="" 38# Add skipped chrome branches in ascending order here. 39SKIPPED_BRANCHES="95" 40 41# NOTE: We enable/disable kernel AFDO starting from a particular branch. 42# For example if we want to enable kernel AFDO in 5.15, first, we do it 43# in main. In this case we want to disable it in beta and stable branches. 44# The second scenario is when we want to disable kernel AFDO (when all devices 45# move to kernelnext and there are no new profiles from the field). In this 46# case we disable AFDO in main but still keep it live in beta and stable. 47declare -A SKIPPED_KVERS_IN_BRANCHES 48# In SKIPPED_KVERS_IN_BRANCHES 49# - key is a branch number string; 50# - value is the list of kernels separated by space. 51# Example: SKIPPED_KVERS_IN_BRANCHES["105"]="4.4 4.14" 52 53# b/223115767. In M-100 there are no new profiles in 5.10. And AFDO is not 54# enabled on any 5.10 board in M-100 either. 55SKIPPED_KVERS_IN_BRANCHES["100"]="5.10" 56 57script_dir=$(dirname "$0") 58tc_utils_dir="${script_dir}/.." 59metadata_dir="${tc_utils_dir}/afdo_metadata" 60amd_outfile="$(realpath --relative-to="${tc_utils_dir}" \ 61 "${metadata_dir}"/kernel_afdo.json)" 62arm_outfile="$(realpath --relative-to="${tc_utils_dir}" \ 63 "${metadata_dir}"/kernel_arm_afdo.json)" 64# Convert toolchain_utils into the absolute path. 65abs_tc_utils_dir="$(realpath "${tc_utils_dir}")" 66 67# Check profiles uploaded within the last week. 68expected_time=$(date +%s -d "week ago") 69# Upload CLs on success. 70upload_cl=true 71 72ARCHS="amd arm" 73declare -A arch_gsbase arch_kvers arch_outfile 74arch_gsbase["amd"]="${AMD_GS_BASE}" 75arch_gsbase["arm"]="${ARM_GS_BASE}" 76arch_kvers["amd"]="${AMD_KVERS}" 77arch_kvers["arm"]="${ARM_KVERS}" 78arch_outfile["amd"]="${amd_outfile}" 79arch_outfile["arm"]="${arm_outfile}" 80 81declare -A branch branch_number commit 82remote_repo=$(git -C "${tc_utils_dir}" remote) 83canary_ref="refs/heads/main" 84# Read the last two release-Rxx from remote branches 85# and assign them to stable_ref and beta_ref. 86# sort -V is the version sort which puts R100 after R99. 87# We need `echo` to convert newlines into spaces for read. 88read -r stable_ref beta_ref <<< "$(git -C "${tc_utils_dir}" ls-remote -h \ 89 "${remote_repo}" release-R\* | cut -f2 | sort -V | tail -n 2 | paste -s)" 90# Branch names which start from release-R. 91branch["beta"]=${beta_ref##*/} 92branch["stable"]=${stable_ref##*/} 93branch["canary"]=${canary_ref##*/} 94 95# Get current branch numbers (number which goes after R). 96branch_number["stable"]=$(echo "${branch["stable"]}" | \ 97 sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") 98branch_number["beta"]=$(echo "${branch["beta"]}" | \ 99 sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") 100branch_number["canary"]="$((branch_number[beta] + 1))" 101for skipped_branch in ${SKIPPED_BRANCHES} ; do 102 if [[ ${branch_number["canary"]} == "${skipped_branch}" ]] ; then 103 ((branch_number[canary]++)) 104 fi 105done 106 107# Without arguments the script updates all branches. 108channels="" 109for arg in "$@" 110do 111 case "${arg}" in 112 stable | canary | beta ) 113 channels="${channels} ${arg}" 114 ;; 115 main ) 116 channels="${channels} canary" 117 ;; 118 all ) 119 channels="canary beta stable" 120 ;; 121 --noupload | --no-upload) 122 upload_cl=false 123 ;; 124 --upload) 125 upload_cl=true 126 ;; 127 --help | help | -h ) 128 echo "${USAGE}" 129 exit 0 130 ;; 131 -*) 132 echo "Option \"${arg}\" is not supported." >&2 133 echo "${USAGE}" 134 exit 1 135 ;; 136 *) 137 echo "Channel \"${arg}\" is not supported. 138Must be main (or canary), beta, stable or all." >&2 139 echo "${USAGE}" 140 exit 1 141 esac 142done 143 144if [[ -z "${channels}" ]] 145then 146 channels="canary beta stable" 147fi 148 149# Fetch latest branches. 150git -C "${tc_utils_dir}" fetch "${remote_repo}" 151 152worktree_dir=$(mktemp -d) 153echo "-> Working in ${worktree_dir}" 154# Create a worktree and make changes there. 155# This way we don't need to clean-up and sync toolchain_utils before the 156# change. Neither we should care about clean-up after the submit. 157git -C "${tc_utils_dir}" worktree add --detach "${worktree_dir}" 158trap 'git -C "${abs_tc_utils_dir}" worktree remove -f "${worktree_dir}"' EXIT 159pushd "${worktree_dir}" 160 161for channel in ${channels} 162do 163 set +u 164 if [[ -n "${commit[${channel}]}" ]] 165 then 166 echo "Skipping channel ${channel} which already has commit\ 167 ${commit[${channel}]}." 168 continue 169 fi 170 set -u 171 172 errs="" 173 successes=0 174 curr_branch_number=${branch_number[${channel}]} 175 curr_branch=${branch[${channel}]} 176 echo 177 echo "Checking \"${channel}\" channel..." 178 echo "branch_number=${curr_branch_number} branch=${curr_branch}" 179 180 git reset --hard HEAD 181 git checkout "${remote_repo}/${curr_branch}" 182 183 for arch in ${ARCHS} 184 do 185 json="{" 186 sep="" 187 for kver in ${arch_kvers[${arch}]} 188 do 189 # Skip kernels disabled in this branch. 190 skipped=false 191 for skipped_branch in "${!SKIPPED_KVERS_IN_BRANCHES[@]}" 192 do 193 if [[ ${curr_branch_number} == "${skipped_branch}" ]] 194 then 195 # Current branch is in the keys of SKIPPED_KVERS_IN_BRANCHES. 196 # Now lets check if $kver is in the list. 197 for skipped_kver in ${SKIPPED_KVERS_IN_BRANCHES[${skipped_branch}]} 198 do 199 if [[ ${kver} == "${skipped_kver}" ]] 200 then 201 skipped=true 202 break 203 fi 204 done 205 fi 206 done 207 if ${skipped} 208 then 209 echo "${kver} is skipped in branch ${curr_branch_number}. Skip it." 210 continue 211 fi 212 # Sort the gs output by timestamp, default ordering is by name. So 213 # R86-13310.3-1594633089.gcov.xz goes after 214 # R86-13310.18-1595237847.gcov.xz. 215 latest=$(gsutil.py ls -l "${arch_gsbase[${arch}]}/${kver}/" | sort -k2 | \ 216 grep "R${curr_branch_number}" | tail -1 || true) 217 if [[ -z "${latest}" && "${channel}" != "stable" ]] 218 then 219 # if no profiles exist for the current branch, try the previous branch 220 latest=$(gsutil.py ls -l "${arch_gsbase[${arch}]}/${kver}/" | \ 221 sort -k2 | grep "R$((curr_branch_number - 1))" | tail -1) 222 fi 223 224 # Verify that the file has the expected date. 225 file_time=$(echo "${latest}" | awk '{print $2}') 226 file_time_unix=$(date +%s -d "${file_time}") 227 if [ "${file_time_unix}" -lt "${expected_time}" ] 228 then 229 expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @"${expected_time}") 230 echo "Wrong date for ${kver}: ${file_time} is before ${expected}" >&2 231 errs="${errs} ${kver}" 232 continue 233 fi 234 235 # Generate JSON. 236 json_kver=$(echo "${kver}" | tr . _) 237 # b/147370213 (migrating profiles from gcov format) may result in the 238 # pattern below no longer doing the right thing. 239 name="$(basename "${latest%.gcov.*}")" 240 # Skip kernels with no AFDO support in the current channel. 241 if [[ "${name}" == "" ]] 242 then 243 continue 244 fi 245 json=$(cat <<EOT 246${json}${sep} 247 "chromeos-kernel-${json_kver}": { 248 "name": "${name}" 249 } 250EOT 251 ) 252 sep="," 253 successes=$((successes + 1)) 254 done # kvers loop 255 256 # If we did not succeed for any kvers, exit now. 257 if [[ ${successes} -eq 0 ]] 258 then 259 echo "error: AFDO profiles out of date for all kernel versions" >&2 260 failed_channels="${failed_channels} ${channel}" 261 continue 262 fi 263 264 # Write new JSON file. 265 # Don't use `echo` since `json` might have esc characters in it. 266 printf "%s\n}\n" "${json}" > "${arch_outfile[${arch}]}" 267 268 # If no changes were made, say so. 269 outdir=$(dirname "${arch_outfile[${arch}]}") 270 shortstat=$(cd "${outdir}" &&\ 271 git status --short "$(basename "${arch_outfile[${arch}]}")") 272 [ -z "${shortstat}" ] &&\ 273 echo "$(basename "${arch_outfile[${arch}]}") is up to date." \ 274 && continue 275 276 # If we had any errors, warn about them. 277 if [[ -n "${errs}" ]] 278 then 279 echo "warning: failed to update ${errs} in ${channel}" >&2 280 failed_channels="${failed_channels} ${channel}" 281 continue 282 fi 283 284 git add "${arch_outfile[${arch}]}" 285 done # ARCHS loop 286 287 case "${channel}" in 288 canary ) 289 commit_contents=$'afdo_metadata: Publish the new kernel profiles\n\n' 290 for arch in ${ARCHS} ; do 291 for kver in ${arch_kvers[${arch}]} ; do 292 commit_contents="${commit_contents}Update ${arch} profile on\ 293 chromeos-kernel-${kver}"$'\n' 294 done 295 done 296 commit_contents="${commit_contents} 297 298BUG=None 299TEST=Verified in kernel-release-afdo-verify-orchestrator" 300 ;; 301 beta | stable ) 302 commit_contents="afdo_metadata: Publish the new kernel profiles\ 303 in ${curr_branch} 304 305Have PM pre-approval because this shouldn't break the release branch. 306 307BUG=None 308TEST=Verified in kernel-release-afdo-verify-orchestrator" 309 ;; 310 * ) 311 echo "internal error: unhandled channel \"${channel}\"" >&2 312 exit 2 313 esac 314 315 git commit -v -e -m "${commit_contents}" 316 317 commit[${channel}]=$(git -C "${worktree_dir}" rev-parse HEAD) 318done 319 320popd 321echo 322# Array size check doesn't play well with the unbound variable option. 323set +u 324if [[ ${#commit[@]} -gt 0 ]] 325then 326 set -u 327 echo "The change is applied in ${!commit[*]}." 328 if ${upload_cl} 329 then 330 for channel in "${!commit[@]}" 331 do 332 git -C "${tc_utils_dir}" push "${remote_repo}" \ 333 "${commit[${channel}]}:refs/for/${branch[${channel}]}" 334 done 335 else 336 echo "Run these commands to upload the change:" 337 echo 338 for channel in "${!commit[@]}" 339 do 340 echo -e "\tgit -C ${tc_utils_dir} push ${remote_repo} \ 341 ${commit[${channel}]}:refs/for/${branch[${channel}]}" 342 done 343 fi 344 345 # Report failed channels. 346 if [[ -n "${failed_channels}" ]] 347 then 348 echo 349 echo "error: failed to update kernel afdo in ${failed_channels}" >&2 350 exit 3 351 fi 352else 353 # No commits. Check if it is due to failures. 354 if [[ -z "${failed_channels}" ]] 355 then 356 echo "No changes are applied. It looks like AFDO versions are up to date." 357 else 358 echo "error: update in ${failed_channels} failed" >&2 359 exit 3 360 fi 361fi 362