1#!/bin/bash 2# Copyright 2020 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# Due to crbug.com/1081332, we need to update AFDO metadata 7# manually. This script performs a few checks and generates a 8# new kernel_afdo.json file, which can then be submitted. 9# 10 11USAGE=" 12Usage: $(basename $0) [main|beta|stable|all] [--help] 13 14Description: 15 The script takes one optional argument which is the channel where we want 16to update the kernel afdo and creates a commit (or commits with \"all\" 17channels) in the corresponding branch. 18 No arguments defaults to \"all\". 19 Follow the prompt to submit the changes. 20 NO CLEAN-UP NEEDED. The script ignores any local changes and keeps 21the current branch unchanged. 22" 23 24set -eu 25set -o pipefail 26 27GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel 28KVERS="4.4 4.14 4.19 5.4" 29failed_channels="" 30# Add skipped chrome branches in ascending order here. 31SKIPPED_BRANCHES="95" 32 33script_dir=$(dirname "$0") 34tc_utils_dir="${script_dir}/.." 35metadata_dir="${tc_utils_dir}/afdo_metadata" 36outfile="$(realpath --relative-to="${tc_utils_dir}" \ 37 "${metadata_dir}"/kernel_afdo.json)" 38# Convert toolchain_utils into the absolute path. 39abs_tc_utils_dir="$(realpath ${tc_utils_dir})" 40 41# Check profiles uploaded within the last week. 42expected_time=$(date +%s -d "week ago") 43 44declare -A branch branch_number commit 45remote_repo=$(git -C "${tc_utils_dir}" remote) 46canary_ref="refs/heads/main" 47# Read the last two release-Rxx from remote branches 48# and assign them to stable_ref and beta_ref. 49# sort -V is the version sort which puts R100 after R99. 50last_branches=$(git -C "${tc_utils_dir}" ls-remote -h "${remote_repo}" \ 51 release-R\* | cut -f2 | sort -V | tail -n 2) 52# We need `echo` to convert newlines into spaces for read. 53read stable_ref beta_ref <<< $(echo ${last_branches}) 54# Branch names which start from release-R. 55branch["beta"]=${beta_ref##*/} 56branch["stable"]=${stable_ref##*/} 57branch["canary"]=${canary_ref##*/} 58 59# Get current branch numbers (number which goes after R). 60branch_number["stable"]=$(echo "${branch["stable"]}" | \ 61 sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") 62branch_number["beta"]=$(echo "${branch["beta"]}" | \ 63 sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p") 64branch_number["canary"]="$((branch_number[beta] + 1))" 65for skipped_branch in $SKIPPED_BRANCHES ; do 66 if [[ ${branch_number["canary"]} == $skipped_branch ]] ; then 67 ((branch_number[canary]++)) 68 fi 69done 70 71# Without arguments the script updates all branches. 72channels=${1:-"all"} 73case "${channels}" in 74 stable | canary | beta ) 75 ;; 76 main ) 77 channels="canary" 78 ;; 79 all ) 80 channels="canary beta stable" 81 ;; 82 --help | help | -h ) 83 echo "$USAGE" 84 exit 0 85 ;; 86 * ) 87 echo "Channel \"${channels}\" is not supported. 88Must be main (or canary), beta, stable or all." >&2 89 echo "$USAGE" 90 exit 1 91esac 92 93# Fetch latest branches. 94git -C "${tc_utils_dir}" fetch "${remote_repo}" 95 96worktree_dir=$(mktemp -d) 97echo "-> Working in ${worktree_dir}" 98# Create a worktree and make changes there. 99# This way we don't need to clean-up and sync toolchain_utils before the 100# change. Neither we should care about clean-up after the submit. 101git -C "${tc_utils_dir}" worktree add --detach "${worktree_dir}" 102trap "git -C ${abs_tc_utils_dir} worktree remove ${worktree_dir}" EXIT 103cd "${worktree_dir}" 104 105for channel in ${channels} 106do 107 errs="" 108 successes=0 109 curr_branch_number=${branch_number[${channel}]} 110 curr_branch=${branch[${channel}]} 111 echo 112 echo "Checking \"${channel}\" channel..." 113 echo "branch_number=${curr_branch_number} branch=${curr_branch}" 114 json="{" 115 sep="" 116 for kver in $KVERS 117 do 118 # Sort the gs output by timestamp (default ordering is by name, so 119 # R86-13310.3-1594633089.gcov.xz goes after R86-13310.18-1595237847.gcov.xz) 120 latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \ 121 grep "R${curr_branch_number}" | tail -1 || true) 122 if [[ -z "$latest" && "${channel}" != "stable" ]] 123 then 124 # if no profiles exist for the current branch, try the previous branch 125 latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \ 126 grep "R$((curr_branch_number - 1))" | tail -1) 127 fi 128 129 # Verify that the file has the expected date. 130 file_time=$(echo "$latest" | awk '{print $2}') 131 file_time_unix=$(date +%s -d "$file_time") 132 if [ $file_time_unix -lt $expected_time ] 133 then 134 expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @$expected_time) 135 echo "Wrong date for $kver: $file_time is before $expected" >&2 136 errs="$errs $kver" 137 continue 138 fi 139 140 # Generate JSON. 141 json_kver=$(echo "$kver" | tr . _) 142 # b/147370213 (migrating profiles from gcov format) may result in the 143 # pattern below no longer doing the right thing. 144 name=$(echo "$latest" | sed 's%.*/\(.*\)\.gcov.*%\1%') 145 json=$(cat <<EOT 146$json$sep 147 "chromeos-kernel-$json_kver": { 148 "name": "$name" 149 } 150EOT 151 ) 152 sep="," 153 successes=$((successes + 1)) 154 done 155 156 # If we did not succeed for any kvers, exit now. 157 if [[ $successes -eq 0 ]] 158 then 159 echo "error: AFDO profiles out of date for all kernel versions" >&2 160 failed_channels="${failed_channels} ${channel}" 161 continue 162 fi 163 164 git reset --hard HEAD 165 echo git checkout "${remote_repo}/${curr_branch}" 166 git checkout "${remote_repo}/${curr_branch}" 167 168 # Write new JSON file. 169 # Don't use `echo` since `json` might have esc characters in it. 170 printf "%s\n}\n" "$json" > "$outfile" 171 172 # If no changes were made, say so. 173 outdir=$(dirname "$outfile") 174 shortstat=$(cd "$outdir" && git status --short $(basename "$outfile")) 175 [ -z "$shortstat" ] && echo $(basename "$outfile")" is up to date." \ 176 && continue 177 178 # If we had any errors, warn about them. 179 if [[ -n "$errs" ]] 180 then 181 echo "warning: failed to update $errs in ${channel}" >&2 182 failed_channels="${failed_channels} ${channel}" 183 continue 184 fi 185 186 git add afdo_metadata/kernel_afdo.json 187 case "${channel}" in 188 canary ) 189 commit_contents="afdo_metadata: Publish the new kernel profiles 190 191Update chromeos-kernel-4_4 192Update chromeos-kernel-4_14 193Update chromeos-kernel-4_19 194Update chromeos-kernel-5_4 195 196BUG=None 197TEST=Verified in kernel-release-afdo-verify-orchestrator" 198 ;; 199 beta | stable ) 200 commit_contents="afdo_metadata: Publish the new kernel profiles\ 201 in ${curr_branch} 202 203Have PM pre-approval because this shouldn't break the release branch. 204 205BUG=None 206TEST=Verified in kernel-release-afdo-verify-orchestrator" 207 ;; 208 * ) 209 echo "internal error: unhandled channel \"${channel}\"" >&2 210 exit 2 211 esac 212 213 git commit -v -e -m "${commit_contents}" 214 215 commit[${channel}]=$(git -C "${worktree_dir}" rev-parse HEAD) 216done 217 218echo 219# Array size check doesn't play well with the unbound variable option. 220set +u 221if [[ ${#commit[@]} -gt 0 ]] 222then 223 set -u 224 echo "The change is applied in ${!commit[@]}." 225 echo "Run these commands to submit the change:" 226 echo 227 for channel in ${!commit[@]} 228 do 229 echo -e "\tgit -C ${tc_utils_dir} push ${remote_repo} \ 230${commit[${channel}]}:refs/for/${branch[${channel}]}" 231 done 232 233 # Report failed channels. 234 if [[ -n "${failed_channels}" ]] 235 then 236 echo 237 echo "error: failed to update kernel afdo in ${failed_channels}" >&2 238 exit 3 239 fi 240else 241 # No commits. Check if it is due to failures. 242 if [[ -z "${failed_channels}" ]] 243 then 244 echo "No changes are applied. It looks like AFDO versions are up to date." 245 else 246 echo "error: update in ${failed_channels} failed" >&2 247 exit 3 248 fi 249fi 250