1#!/usr/bin/env bash 2 3usage () { 4 echo "USAGE: ./sync-kernel.sh <bpftool-repo> <kernel-repo>" 5 echo "" 6 echo "This script synchronizes the mirror with upstream bpftool sources from the kernel repository." 7 echo "It performs the following steps:" 8 echo " - Update the libbpf submodule, commit, and use its new checkpoints as target commits for bpftool." 9 echo " - Cherry-pick commits from the bpf-next branch, up to the bpf-next target commit." 10 echo " - Cherry-pick commits from the bpf branch, up to the bpf target commit." 11 echo " - Update bpftool's version number based on bpf-next's kernel version and target commit." 12 echo " - Create a new commit with the updated version and checkpoints." 13 echo " - Check consistency." 14 echo "" 15 echo "Set BPF_NEXT_BASELINE to override bpf-next tree commit, otherwise read from <bpftool-repo>/CHECKPOINT-COMMIT." 16 echo "Set BPF_BASELINE to override bpf tree commit, otherwise read from <bpftool-repo>/BPF-CHECKPOINT-COMMIT." 17 echo "Set BPF_NEXT_TIP_COMMIT to override bpf-next tree target commit, otherwise read from <bpftool-repo>/libbpf/CHECKPOINT-COMMIT, after libbpf update." 18 echo "Set BPF_TIP_COMMIT to override bpf tree target commit, otherwise read from <bpftool-repo>/libbpf/BPF-CHECKPOINT-COMMIT, after libbpf update." 19 echo "Set SKIP_LIBBPF_UPDATE to 1 to avoid updating libbpf automatically." 20 echo "Set MANUAL_MODE to 1 to manually control every cherry-picked commit." 21 exit 1 22} 23 24set -eu 25 26BPFTOOL_REPO=${1-""} 27LINUX_REPO=${2-""} 28 29if [ -z "${BPFTOOL_REPO}" ] || [ -z "${LINUX_REPO}" ]; then 30 echo "Error: bpftool or linux repos are not specified" 31 usage 32fi 33 34BASELINE_COMMIT=${BPF_NEXT_BASELINE:-$(cat ${BPFTOOL_REPO}/CHECKPOINT-COMMIT)} 35BPF_BASELINE_COMMIT=${BPF_BASELINE:-$(cat ${BPFTOOL_REPO}/BPF-CHECKPOINT-COMMIT)} 36 37if [ -z "${BASELINE_COMMIT}" ] || [ -z "${BPF_BASELINE_COMMIT}" ]; then 38 echo "Error: bpf or bpf-next baseline commits are not provided" 39 usage 40fi 41 42SUFFIX=$(date --utc +%Y-%m-%dT%H-%M-%S.%3NZ) 43WORKDIR=$(pwd) 44TMP_DIR=$(mktemp -d) 45 46trap "cd ${WORKDIR}; exit" INT TERM EXIT 47 48BPFTOOL_SRC_DIR="tools/bpf/bpftool" 49 50declare -A PATH_MAP 51PATH_MAP=( \ 52 [${BPFTOOL_SRC_DIR}]=src \ 53 [${BPFTOOL_SRC_DIR}/bash-completion]=bash-completion \ 54 [${BPFTOOL_SRC_DIR}/Documentation]=docs \ 55 [kernel/bpf/disasm.c]=src/kernel/bpf/disasm.c \ 56 [kernel/bpf/disasm.h]=src/kernel/bpf/disasm.h \ 57 [tools/include/uapi/asm-generic/bitsperlong.h]=include/uapi/asm-generic/bitsperlong.h \ 58 [tools/include/uapi/linux/bpf_common.h]=include/uapi/linux/bpf_common.h \ 59 [tools/include/uapi/linux/bpf.h]=include/uapi/linux/bpf.h \ 60 [tools/include/uapi/linux/btf.h]=include/uapi/linux/btf.h \ 61 [tools/include/uapi/linux/const.h]=include/uapi/linux/const.h \ 62 [tools/include/uapi/linux/if_link.h]=include/uapi/linux/if_link.h \ 63 [tools/include/uapi/linux/perf_event.h]=include/uapi/linux/perf_event.h \ 64 [tools/include/uapi/linux/pkt_cls.h]=include/uapi/linux/pkt_cls.h \ 65 [tools/include/uapi/linux/pkt_sched.h]=include/uapi/linux/pkt_sched.h \ 66 [tools/include/uapi/linux/tc_act/tc_bpf.h]=include/uapi/linux/tc_act/tc_bpf.h \ 67) 68 69BPFTOOL_PATHS="${!PATH_MAP[@]}" 70BPFTOOL_VIEW_PATHS="${PATH_MAP[@]}" 71BPFTOOL_VIEW_EXCLUDE_REGEX='^(docs/\.gitignore|src/Makefile\.(feature|include))$' 72LINUX_VIEW_EXCLUDE_REGEX='^$' 73 74# Deal with tools/bpf/bpftool first, because once we've mkdir-ed src/, command 75# "git mv" doesn't move bpftool _as_ src but _into_ src/. 76BPFTOOL_TREE_FILTER="mkdir __bpftool && "$'\\\n' 77BPFTOOL_TREE_FILTER+="git mv -kf ${BPFTOOL_SRC_DIR} __bpftool/${PATH_MAP[${BPFTOOL_SRC_DIR}]} && "$'\\\n' 78 79# Extract bash-completion and Documentation from src/. 80BPFTOOL_TREE_FILTER+="git mv -kf __bpftool/src/bash-completion __bpftool/bash-completion && "$'\\\n' 81BPFTOOL_TREE_FILTER+="git mv -kf __bpftool/src/Documentation __bpftool/docs && "$'\\\n' 82 83BPFTOOL_TREE_FILTER+="mkdir -p __bpftool/include/uapi/asm-generic __bpftool/include/uapi/linux/tc_act __bpftool/src/kernel/bpf && "$'\\\n' 84for p in "${!PATH_MAP[@]}"; do 85 case ${p} in 86 ${BPFTOOL_SRC_DIR}*) 87 continue;; 88 esac 89 BPFTOOL_TREE_FILTER+="git mv -kf ${p} __bpftool/${PATH_MAP[${p}]} && "$'\\\n' 90done 91BPFTOOL_TREE_FILTER+="true >/dev/null" 92 93cd_to() 94{ 95 cd ${WORKDIR} && cd "$1" 96} 97 98# Output brief single-line commit description 99# $1 - commit ref 100commit_desc() 101{ 102 git log -n1 --pretty='%h ("%s")' $1 103} 104 105# Create commit single-line signature, which consists of: 106# - full commit subject 107# - author date in ISO8601 format 108# - full commit body with newlines replaced with vertical bars (|) 109# - shortstat appended at the end 110# The idea is that this single-line signature is good enough to make final 111# decision about whether two commits are the same, across different repos. 112# $1 - commit ref 113# $2 - paths filter 114commit_signature() 115{ 116 git show --pretty='("%s")|%aI|%b' --shortstat $1 -- ${2-.} | tr '\n' '|' 117} 118 119# Cherry-pick commits touching bpftool-related files 120# $1 - baseline_tag 121# $2 - tip_tag 122cherry_pick_commits() 123{ 124 local manual_mode=${MANUAL_MODE:-0} 125 local baseline_tag=$1 126 local tip_tag=$2 127 local new_commits 128 local signature 129 local should_skip 130 local synced_cnt 131 local manual_check 132 local bpftool_conflict_cnt 133 local desc 134 135 new_commits=$(git rev-list --no-merges --topo-order --reverse ${baseline_tag}..${tip_tag} ${BPFTOOL_PATHS[@]}) 136 for new_commit in ${new_commits}; do 137 if [[ "${baseline_tag}" == "${BPF_BASELINE_TAG}" ]]; then 138 if git merge-base --is-ancestor "${new_commit}" "${BASELINE_COMMIT}"; then 139 echo "Commit ${new_commit::12} from bpf is already in bpf-next branch, skipping." 140 continue 141 fi 142 fi 143 desc="$(commit_desc ${new_commit})" 144 signature="$(commit_signature ${new_commit} "${BPFTOOL_PATHS[@]}")" 145 synced_cnt=$(grep -F "${signature}" ${TMP_DIR}/bpftool_commits.txt | wc -l) 146 manual_check=0 147 if ((${synced_cnt} > 0)); then 148 # commit with the same subject is already in bpftool, but it's 149 # not 100% the same commit, so check with user 150 echo "Commit '${desc}' is synced into bpftool as:" 151 grep -F "${signature}" ${TMP_DIR}/bpftool_commits.txt | \ 152 cut -d'|' -f1 | sed -e 's/^/- /' 153 if ((${manual_mode} != 1 && ${synced_cnt} == 1)); then 154 echo "Skipping '${desc}' due to unique match..." 155 continue 156 fi 157 if ((${synced_cnt} > 1)); then 158 echo "'${desc} matches multiple commits, please, double-check!" 159 manual_check=1 160 fi 161 fi 162 if ((${manual_mode} == 1 || ${manual_check} == 1)); then 163 read -p "Do you want to skip '${desc}'? [y/N]: " should_skip 164 case "${should_skip}" in 165 "y" | "Y") 166 echo "Skipping '${desc}'..." 167 continue 168 ;; 169 esac 170 fi 171 # commit hasn't been synced into bpftool yet 172 echo "Picking '${desc}'..." 173 if ! git cherry-pick ${new_commit} &>/dev/null; then 174 echo "Warning! Cherry-picking '${desc} failed, checking if it's non-bpftool files causing problems..." 175 bpftool_conflict_cnt=$(git diff --name-only --diff-filter=U -- ${BPFTOOL_PATHS[@]} | wc -l) 176 conflict_cnt=$(git diff --name-only | wc -l) 177 prompt_resolution=1 178 179 if ((${bpftool_conflict_cnt} == 0)); then 180 echo "Looks like only non-bpftool files have conflicts, ignoring..." 181 if ((${conflict_cnt} == 0)); then 182 echo "Empty cherry-pick, skipping it..." 183 git cherry-pick --abort 184 continue 185 fi 186 187 git add . 188 # GIT_EDITOR=true to avoid editor popping up to edit commit message 189 if ! GIT_EDITOR=true git cherry-pick --continue &>/dev/null; then 190 echo "Error! That still failed! Please resolve manually." 191 else 192 echo "Success! All cherry-pick conflicts were resolved for '${desc}'!" 193 prompt_resolution=0 194 fi 195 fi 196 197 if ((${prompt_resolution} == 1)); then 198 read -p "Error! Cherry-picking '${desc}' failed, please fix manually and press <return> to proceed..." 199 fi 200 fi 201 # Append signature of just cherry-picked commit to avoid 202 # potentially cherry-picking the same commit twice later when 203 # processing bpf tree commits. At this point we don't know yet 204 # the final commit sha in bpftool repo, so we record Linux SHA 205 # instead as LINUX_<sha>. 206 echo LINUX_$(git log --pretty='%h' -n1) "${signature}" >> ${TMP_DIR}/bpftool_commits.txt 207 done 208} 209 210cleanup() 211{ 212 echo "Cleaning up..." 213 rm -r ${TMP_DIR} 214 cd_to ${LINUX_REPO} 215 git checkout ${TIP_SYM_REF} 216 git branch -D ${BASELINE_TAG} ${TIP_TAG} ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} \ 217 ${SQUASH_BASE_TAG} ${SQUASH_TIP_TAG} ${VIEW_TAG} || true 218 219 cd_to . 220 echo "DONE." 221} 222 223cd_to ${BPFTOOL_REPO} 224BPFTOOL_SYNC_TAG=bpftool-sync-${SUFFIX} 225git checkout -b ${BPFTOOL_SYNC_TAG} 226 227# Update libbpf 228if [[ "${SKIP_LIBBPF_UPDATE:-0}" -ne 1 ]]; then 229 cd_to ${BPFTOOL_REPO}/libbpf 230 git pull origin master 231 LIBBPF_VERSION=$(grep -oE '^LIBBPF_([0-9.]+)' src/libbpf.map | sort -rV | head -n1 | cut -d'_' -f2) 232 LIBBPF_COMMIT=$(git rev-parse HEAD) 233 cd_to ${BPFTOOL_REPO} 234 if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then 235 git add libbpf 236 git commit -m 'sync: Update libbpf submodule' \ 237 -m "\ 238Pull latest libbpf from mirror. 239Libbpf version: ${LIBBPF_VERSION} 240Libbpf commit: ${LIBBPF_COMMIT}" \ 241 -- libbpf 242 fi 243fi 244 245# Use libbpf's new checkpoints as tips 246TIP_COMMIT=${BPF_NEXT_TIP_COMMIT:-$(cat ${BPFTOOL_REPO}/libbpf/CHECKPOINT-COMMIT)} 247BPF_TIP_COMMIT=${BPF_TIP_COMMIT:-$(cat ${BPFTOOL_REPO}/libbpf/BPF-CHECKPOINT-COMMIT)} 248if [ -z "${TIP_COMMIT}" ] || [ -z "${BPF_TIP_COMMIT}" ]; then 249 echo "Error: bpf or bpf-next tip commits are not provided" 250 usage 251fi 252 253cd_to ${BPFTOOL_REPO} 254GITHUB_ABS_DIR=$(pwd) 255echo "Dumping existing bpftool commit signatures..." 256for h in $(git log --pretty='%h' -n500); do 257 echo $h "$(commit_signature $h)" >> ${TMP_DIR}/bpftool_commits.txt 258done 259 260# Use current kernel repo HEAD as a source of patches 261cd_to ${LINUX_REPO} 262LINUX_ABS_DIR=$(pwd) 263TIP_SYM_REF=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD) 264BASELINE_TAG=bpftool-baseline-${SUFFIX} 265TIP_TAG=bpftool-tip-${SUFFIX} 266BPF_BASELINE_TAG=bpftool-bpf-baseline-${SUFFIX} 267BPF_TIP_TAG=bpftool-bpf-tip-${SUFFIX} 268VIEW_TAG=bpftool-view-${SUFFIX} 269 270# Squash state of kernel repo at baseline into single commit 271SQUASH_BASE_TAG=bpftool-squash-base-${SUFFIX} 272SQUASH_TIP_TAG=bpftool-squash-tip-${SUFFIX} 273SQUASH_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE SQUASH ${BASELINE_COMMIT}") 274 275echo "WORKDIR: ${WORKDIR}" 276echo "LINUX REPO: ${LINUX_REPO}" 277echo "BPFTOOL REPO: ${BPFTOOL_REPO}" 278echo "TEMP DIR: ${TMP_DIR}" 279echo "SUFFIX: ${SUFFIX}" 280echo "BASE COMMIT: '$(commit_desc ${BASELINE_COMMIT})'" 281echo "TIP COMMIT: '$(commit_desc ${TIP_COMMIT})'" 282echo "BPF BASE COMMIT: '$(commit_desc ${BPF_BASELINE_COMMIT})'" 283echo "BPF TIP COMMIT: '$(commit_desc ${BPF_TIP_COMMIT})'" 284echo "SQUASH COMMIT: ${SQUASH_COMMIT}" 285echo "BASELINE TAG: ${BASELINE_TAG}" 286echo "TIP TAG: ${TIP_TAG}" 287echo "BPF BASELINE TAG: ${BPF_BASELINE_TAG}" 288echo "BPF TIP TAG: ${BPF_TIP_TAG}" 289echo "SQUASH BASE TAG: ${SQUASH_BASE_TAG}" 290echo "SQUASH TIP TAG: ${SQUASH_TIP_TAG}" 291echo "VIEW TAG: ${VIEW_TAG}" 292echo "BPFTOOL SYNC TAG: ${BPFTOOL_SYNC_TAG}" 293echo "PATCHES: ${TMP_DIR}/patches" 294 295git branch ${BASELINE_TAG} ${BASELINE_COMMIT} 296git branch ${TIP_TAG} ${TIP_COMMIT} 297git branch ${BPF_BASELINE_TAG} ${BPF_BASELINE_COMMIT} 298git branch ${BPF_TIP_TAG} ${BPF_TIP_COMMIT} 299git branch ${SQUASH_BASE_TAG} ${SQUASH_COMMIT} 300git checkout -b ${SQUASH_TIP_TAG} ${SQUASH_COMMIT} 301 302# Cherry-pick new commits onto squashed baseline commit 303echo "Cherry-pick for bpf-next..." 304cherry_pick_commits ${BASELINE_TAG} ${TIP_TAG} 305echo "Cherry-pick for bpf..." 306cherry_pick_commits ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} 307 308# Move all bpftool files into __bpftool directory. 309FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --tree-filter "${BPFTOOL_TREE_FILTER}" ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG} 310# Make __bpftool a new root directory 311FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --subdirectory-filter __bpftool ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG} 312 313# If there are no new commits with bpftool-related changes, bail out 314COMMIT_CNT=$(git rev-list --count ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG}) 315if ((${COMMIT_CNT} <= 0)); then 316 echo "No new changes to apply, we are done!" 317 cleanup 318 exit 2 319fi 320 321# Exclude baseline commit and generate nice cover letter with summary 322git format-patch ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --cover-letter -o ${TMP_DIR}/patches 323 324# Compute version by concatenating kernel tip's version and hash 325git -c advice.detachedHead=false checkout ${TIP_COMMIT} 326BPFTOOL_VERSION="$(make kernelversion)+${TIP_COMMIT::12}" 327 328# Now is time to re-apply bpftool-related linux patches to bpftool repo 329cd_to ${BPFTOOL_REPO} 330 331for patch in $(ls -1 ${TMP_DIR}/patches | tail -n +2); do 332 if ! git am --3way --committer-date-is-author-date "${TMP_DIR}/patches/${patch}"; then 333 read -p "Applying ${TMP_DIR}/patches/${patch} failed, please resolve manually and press <return> to proceed..." 334 fi 335done 336 337# Bump bpftool version number if necessary 338sed -i "s/^\(BPFTOOL_VERSION := \).*/\1${BPFTOOL_VERSION}/" src/Makefile 339git add src/Makefile 340 341# Use generated cover-letter as a template for "sync commit" with 342# baseline and checkpoint commits from kernel repo (and leave summary 343# from cover letter intact, of course) 344echo ${TIP_COMMIT} > CHECKPOINT-COMMIT && \ 345echo ${BPF_TIP_COMMIT} > BPF-CHECKPOINT-COMMIT && \ 346git add CHECKPOINT-COMMIT && \ 347git add BPF-CHECKPOINT-COMMIT && \ 348awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' ${TMP_DIR}/patches/0000-cover-letter.patch | \ 349sed "s/\*\*\* BLURB HERE \*\*\*/\ 350sync: Pull latest bpftool changes from kernel\n\ 351\n\ 352Syncing latest bpftool commits from kernel repository.\n\ 353Baseline bpf-next commit: ${BASELINE_COMMIT}\n\ 354Checkpoint bpf-next commit: ${TIP_COMMIT}\n\ 355Baseline bpf commit: ${BPF_BASELINE_COMMIT}\n\ 356Checkpoint bpf commit: ${BPF_TIP_COMMIT}\n\ 357Latest bpftool version: ${BPFTOOL_VERSION}/" | \ 358git commit --file=- 359 360echo "SUCCESS! ${COMMIT_CNT} commits synced." 361 362echo "Verifying Linux's and Github's bpftool state" 363 364cd_to ${LINUX_REPO} 365git checkout -b ${VIEW_TAG} ${TIP_COMMIT} 366FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --tree-filter "${BPFTOOL_TREE_FILTER}" ${VIEW_TAG}^..${VIEW_TAG} 367FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --subdirectory-filter __bpftool ${VIEW_TAG}^..${VIEW_TAG} 368git ls-files -- ${BPFTOOL_VIEW_PATHS[@]} | grep -v -E "${LINUX_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/linux-view.ls 369 370cd_to ${BPFTOOL_REPO} 371git ls-files -- ${BPFTOOL_VIEW_PATHS[@]} | grep -v -E "${BPFTOOL_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/github-view.ls 372 373echo "Comparing list of files..." 374diff -u ${TMP_DIR}/linux-view.ls ${TMP_DIR}/github-view.ls 375echo "Comparing file contents..." 376CONSISTENT=1 377for F in $(cat ${TMP_DIR}/linux-view.ls); do 378 if ! diff -u --color "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"; then 379 echo "${LINUX_ABS_DIR}/${F} and ${GITHUB_ABS_DIR}/${F} are different!" 380 CONSISTENT=0 381 fi 382done 383if ((${CONSISTENT} == 1)); then 384 echo "Great! Content is identical!" 385else 386 ignore_inconsistency=n 387 echo "Unfortunately, there are some inconsistencies, please double check." 388 read -p "Does everything look good? [y/N]: " ignore_inconsistency 389 case "${ignore_inconsistency}" in 390 "y" | "Y") 391 echo "Ok, proceeding..." 392 ;; 393 *) 394 echo "Oops, exiting with error..." 395 exit 4 396 esac 397fi 398 399cleanup 400