1#!/bin/bash 2 3usage () { 4 echo "USAGE: ./sync-kernel.sh <libbpf-repo> <kernel-repo> <bpf-branch>" 5 echo "" 6 echo "Set BPF_NEXT_BASELINE to override bpf-next tree commit, otherwise read from <libbpf-repo>/CHECKPOINT-COMMIT." 7 echo "Set BPF_BASELINE to override bpf tree commit, otherwise read from <libbpf-repo>/BPF-CHECKPOINT-COMMIT." 8 echo "Set MANUAL_MODE to 1 to manually control every cherry-picked commits." 9 exit 1 10} 11 12set -eu 13 14LIBBPF_REPO=${1-""} 15LINUX_REPO=${2-""} 16BPF_BRANCH=${3-""} 17BASELINE_COMMIT=${BPF_NEXT_BASELINE:-$(cat ${LIBBPF_REPO}/CHECKPOINT-COMMIT)} 18BPF_BASELINE_COMMIT=${BPF_BASELINE:-$(cat ${LIBBPF_REPO}/BPF-CHECKPOINT-COMMIT)} 19 20if [ -z "${LIBBPF_REPO}" ] || [ -z "${LINUX_REPO}" ]; then 21 echo "Error: libbpf or linux repos are not specified" 22 usage 23fi 24if [ -z "${BPF_BRANCH}" ]; then 25 echo "Error: linux's bpf tree branch is not specified" 26 usage 27fi 28if [ -z "${BASELINE_COMMIT}" ] || [ -z "${BPF_BASELINE_COMMIT}" ]; then 29 echo "Error: bpf or bpf-next baseline commits are not provided" 30 usage 31fi 32 33SUFFIX=$(date --utc +%Y-%m-%dT%H-%M-%S.%3NZ) 34WORKDIR=$(pwd) 35TMP_DIR=$(mktemp -d) 36 37trap "cd ${WORKDIR}; exit" INT TERM EXIT 38 39declare -A PATH_MAP 40PATH_MAP=( \ 41 [tools/lib/bpf]=src \ 42 [tools/include/uapi/linux/bpf_common.h]=include/uapi/linux/bpf_common.h \ 43 [tools/include/uapi/linux/bpf.h]=include/uapi/linux/bpf.h \ 44 [tools/include/uapi/linux/btf.h]=include/uapi/linux/btf.h \ 45 [tools/include/uapi/linux/fcntl.h]=include/uapi/linux/fcntl.h \ 46 [tools/include/uapi/linux/if_link.h]=include/uapi/linux/if_link.h \ 47 [tools/include/uapi/linux/if_xdp.h]=include/uapi/linux/if_xdp.h \ 48 [tools/include/uapi/linux/netlink.h]=include/uapi/linux/netlink.h \ 49 [tools/include/uapi/linux/pkt_cls.h]=include/uapi/linux/pkt_cls.h \ 50 [tools/include/uapi/linux/pkt_sched.h]=include/uapi/linux/pkt_sched.h \ 51 [include/uapi/linux/perf_event.h]=include/uapi/linux/perf_event.h \ 52 [Documentation/bpf/libbpf]=docs \ 53) 54 55LIBBPF_PATHS=("${!PATH_MAP[@]}" ":^tools/lib/bpf/Makefile" ":^tools/lib/bpf/Build" ":^tools/lib/bpf/.gitignore" ":^tools/include/tools/libc_compat.h") 56LIBBPF_VIEW_PATHS=("${PATH_MAP[@]}") 57LIBBPF_VIEW_EXCLUDE_REGEX='^src/(Makefile|Build|test_libbpf\.c|bpf_helper_defs\.h|\.gitignore)$|^docs/(\.gitignore|api\.rst|conf\.py)$|^docs/sphinx/.*' 58LINUX_VIEW_EXCLUDE_REGEX='^include/tools/libc_compat.h$' 59 60LIBBPF_TREE_FILTER="mkdir -p __libbpf/include/uapi/linux __libbpf/include/tools && "$'\\\n' 61for p in "${!PATH_MAP[@]}"; do 62 LIBBPF_TREE_FILTER+="git mv -kf ${p} __libbpf/${PATH_MAP[${p}]} && "$'\\\n' 63done 64LIBBPF_TREE_FILTER+="git rm --ignore-unmatch -f __libbpf/src/{Makefile,Build,test_libbpf.c,.gitignore} >/dev/null" 65 66cd_to() 67{ 68 cd ${WORKDIR} && cd "$1" 69} 70 71# Output brief single-line commit description 72# $1 - commit ref 73commit_desc() 74{ 75 git log -n1 --pretty='%h ("%s")' $1 76} 77 78# Create commit single-line signature, which consists of: 79# - full commit subject 80# - author date in ISO8601 format 81# - full commit body with newlines replaced with vertical bars (|) 82# - shortstat appended at the end 83# The idea is that this single-line signature is good enough to make final 84# decision about whether two commits are the same, across different repos. 85# $1 - commit ref 86# $2 - paths filter 87commit_signature() 88{ 89 local ref=$1 90 shift 91 git show --pretty='("%s")|%aI|%b' --shortstat $ref -- "${@-.}" | tr '\n' '|' 92} 93 94# Cherry-pick commits touching libbpf-related files 95# $1 - baseline_tag 96# $2 - tip_tag 97cherry_pick_commits() 98{ 99 local manual_mode=${MANUAL_MODE:-0} 100 local baseline_tag=$1 101 local tip_tag=$2 102 local new_commits 103 local signature 104 local should_skip 105 local synced_cnt 106 local manual_check 107 local libbpf_conflict_cnt 108 local desc 109 110 new_commits=$(git rev-list --no-merges --topo-order --reverse ${baseline_tag}..${tip_tag} -- "${LIBBPF_PATHS[@]}") 111 for new_commit in ${new_commits}; do 112 desc="$(commit_desc ${new_commit})" 113 signature="$(commit_signature ${new_commit} "${LIBBPF_PATHS[@]}")" 114 synced_cnt=$(grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | wc -l) 115 manual_check=0 116 if ((${synced_cnt} > 0)); then 117 # commit with the same subject is already in libbpf, but it's 118 # not 100% the same commit, so check with user 119 echo "Commit '${desc}' is synced into libbpf as:" 120 grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | \ 121 cut -d'|' -f1 | sed -e 's/^/- /' 122 if ((${manual_mode} != 1 && ${synced_cnt} == 1)); then 123 echo "Skipping '${desc}' due to unique match..." 124 continue 125 fi 126 if ((${synced_cnt} > 1)); then 127 echo "'${desc} matches multiple commits, please, double-check!" 128 manual_check=1 129 fi 130 fi 131 if ((${manual_mode} == 1 || ${manual_check} == 1)); then 132 read -p "Do you want to skip '${desc}'? [y/N]: " should_skip 133 case "${should_skip}" in 134 "y" | "Y") 135 echo "Skipping '${desc}'..." 136 continue 137 ;; 138 esac 139 fi 140 # commit hasn't been synced into libbpf yet 141 echo "Picking '${desc}'..." 142 if ! git cherry-pick ${new_commit} &>/dev/null; then 143 echo "Warning! Cherry-picking '${desc} failed, checking if it's non-libbpf files causing problems..." 144 libbpf_conflict_cnt=$(git diff --name-only --diff-filter=U -- "${LIBBPF_PATHS[@]}" | wc -l) 145 conflict_cnt=$(git diff --name-only | wc -l) 146 prompt_resolution=1 147 148 if ((${libbpf_conflict_cnt} == 0)); then 149 echo "Looks like only non-libbpf files have conflicts, ignoring..." 150 if ((${conflict_cnt} == 0)); then 151 echo "Empty cherry-pick, skipping it..." 152 git cherry-pick --abort 153 continue 154 fi 155 156 git add . 157 # GIT_EDITOR=true to avoid editor popping up to edit commit message 158 if ! GIT_EDITOR=true git cherry-pick --continue &>/dev/null; then 159 echo "Error! That still failed! Please resolve manually." 160 else 161 echo "Success! All cherry-pick conflicts were resolved for '${desc}'!" 162 prompt_resolution=0 163 fi 164 fi 165 166 if ((${prompt_resolution} == 1)); then 167 read -p "Error! Cherry-picking '${desc}' failed, please fix manually and press <return> to proceed..." 168 fi 169 fi 170 # Append signature of just cherry-picked commit to avoid 171 # potentially cherry-picking the same commit twice later when 172 # processing bpf tree commits. At this point we don't know yet 173 # the final commit sha in libbpf repo, so we record Linux SHA 174 # instead as LINUX_<sha>. 175 echo LINUX_$(git log --pretty='%h' -n1) "${signature}" >> ${TMP_DIR}/libbpf_commits.txt 176 done 177} 178 179cleanup() 180{ 181 echo "Cleaning up..." 182 rm -r ${TMP_DIR} 183 cd_to ${LINUX_REPO} 184 git checkout ${TIP_SYM_REF} 185 git branch -D ${BASELINE_TAG} ${TIP_TAG} ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} \ 186 ${SQUASH_BASE_TAG} ${SQUASH_TIP_TAG} ${VIEW_TAG} || true 187 188 cd_to . 189 echo "DONE." 190} 191 192 193cd_to ${LIBBPF_REPO} 194GITHUB_ABS_DIR=$(pwd) 195echo "Dumping existing libbpf commit signatures..." 196for h in $(git log --pretty='%h' -n500); do 197 echo $h "$(commit_signature $h)" >> ${TMP_DIR}/libbpf_commits.txt 198done 199 200# Use current kernel repo HEAD as a source of patches 201cd_to ${LINUX_REPO} 202LINUX_ABS_DIR=$(pwd) 203TIP_SYM_REF=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD) 204TIP_COMMIT=$(git rev-parse HEAD) 205BPF_TIP_COMMIT=$(git rev-parse ${BPF_BRANCH}) 206BASELINE_TAG=libbpf-baseline-${SUFFIX} 207TIP_TAG=libbpf-tip-${SUFFIX} 208BPF_BASELINE_TAG=libbpf-bpf-baseline-${SUFFIX} 209BPF_TIP_TAG=libbpf-bpf-tip-${SUFFIX} 210VIEW_TAG=libbpf-view-${SUFFIX} 211LIBBPF_SYNC_TAG=libbpf-sync-${SUFFIX} 212 213# Squash state of kernel repo at baseline into single commit 214SQUASH_BASE_TAG=libbpf-squash-base-${SUFFIX} 215SQUASH_TIP_TAG=libbpf-squash-tip-${SUFFIX} 216SQUASH_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE SQUASH ${BASELINE_COMMIT}") 217 218echo "WORKDIR: ${WORKDIR}" 219echo "LINUX REPO: ${LINUX_REPO}" 220echo "LIBBPF REPO: ${LIBBPF_REPO}" 221echo "TEMP DIR: ${TMP_DIR}" 222echo "SUFFIX: ${SUFFIX}" 223echo "BASE COMMIT: '$(commit_desc ${BASELINE_COMMIT})'" 224echo "TIP COMMIT: '$(commit_desc ${TIP_COMMIT})'" 225echo "BPF BASE COMMIT: '$(commit_desc ${BPF_BASELINE_COMMIT})'" 226echo "BPF TIP COMMIT: '$(commit_desc ${BPF_TIP_COMMIT})'" 227echo "SQUASH COMMIT: ${SQUASH_COMMIT}" 228echo "BASELINE TAG: ${BASELINE_TAG}" 229echo "TIP TAG: ${TIP_TAG}" 230echo "BPF BASELINE TAG: ${BPF_BASELINE_TAG}" 231echo "BPF TIP TAG: ${BPF_TIP_TAG}" 232echo "SQUASH BASE TAG: ${SQUASH_BASE_TAG}" 233echo "SQUASH TIP TAG: ${SQUASH_TIP_TAG}" 234echo "VIEW TAG: ${VIEW_TAG}" 235echo "LIBBPF SYNC TAG: ${LIBBPF_SYNC_TAG}" 236echo "PATCHES: ${TMP_DIR}/patches" 237 238git branch ${BASELINE_TAG} ${BASELINE_COMMIT} 239git branch ${TIP_TAG} ${TIP_COMMIT} 240git branch ${BPF_BASELINE_TAG} ${BPF_BASELINE_COMMIT} 241git branch ${BPF_TIP_TAG} ${BPF_TIP_COMMIT} 242git branch ${SQUASH_BASE_TAG} ${SQUASH_COMMIT} 243git checkout -b ${SQUASH_TIP_TAG} ${SQUASH_COMMIT} 244 245# Cherry-pick new commits onto squashed baseline commit 246cherry_pick_commits ${BASELINE_TAG} ${TIP_TAG} 247cherry_pick_commits ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} 248 249# Move all libbpf files into __libbpf directory. 250FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --tree-filter "${LIBBPF_TREE_FILTER}" ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG} 251# Make __libbpf a new root directory 252FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --subdirectory-filter __libbpf ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG} 253 254# If there are no new commits with libbpf-related changes, bail out 255COMMIT_CNT=$(git rev-list --count ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG}) 256if ((${COMMIT_CNT} <= 0)); then 257 echo "No new changes to apply, we are done!" 258 cleanup 259 exit 2 260fi 261 262# Exclude baseline commit and generate nice cover letter with summary 263git format-patch ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --cover-letter -o ${TMP_DIR}/patches 264 265# Now is time to re-apply libbpf-related linux patches to libbpf repo 266cd_to ${LIBBPF_REPO} 267git checkout -b ${LIBBPF_SYNC_TAG} 268 269for patch in $(ls -1 ${TMP_DIR}/patches | tail -n +2); do 270 if ! git am -3 --committer-date-is-author-date "${TMP_DIR}/patches/${patch}"; then 271 if ! patch -p1 --merge < "${TMP_DIR}/patches/${patch}"; then 272 read -p "Applying ${TMP_DIR}/patches/${patch} failed, please resolve manually and press <return> to proceed..." 273 fi 274 git am --continue 275 fi 276done 277 278# Generate bpf_helper_defs.h and commit, if anything changed 279# restore Linux tip to use bpf_doc.py 280cd_to ${LINUX_REPO} 281git checkout ${TIP_TAG} 282# re-generate bpf_helper_defs.h 283cd_to ${LIBBPF_REPO} 284"${LINUX_ABS_DIR}/scripts/bpf_doc.py" --header \ 285 --file include/uapi/linux/bpf.h > src/bpf_helper_defs.h 286# if anything changed, commit it 287helpers_changes=$(git status --porcelain src/bpf_helper_defs.h | wc -l) 288if ((${helpers_changes} == 1)); then 289 git add src/bpf_helper_defs.h 290 git commit -s -m "sync: auto-generate latest BPF helpers 291 292Latest changes to BPF helper definitions. 293" -- src/bpf_helper_defs.h 294fi 295 296# Use generated cover-letter as a template for "sync commit" with 297# baseline and checkpoint commits from kernel repo (and leave summary 298# from cover letter intact, of course) 299echo ${TIP_COMMIT} > CHECKPOINT-COMMIT && \ 300echo ${BPF_TIP_COMMIT} > BPF-CHECKPOINT-COMMIT && \ 301git add CHECKPOINT-COMMIT && \ 302git add BPF-CHECKPOINT-COMMIT && \ 303awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' ${TMP_DIR}/patches/0000-cover-letter.patch | \ 304sed "s/\*\*\* BLURB HERE \*\*\*/\ 305sync: latest libbpf changes from kernel\n\ 306\n\ 307Syncing latest libbpf commits from kernel repository.\n\ 308Baseline bpf-next commit: ${BASELINE_COMMIT}\n\ 309Checkpoint bpf-next commit: ${TIP_COMMIT}\n\ 310Baseline bpf commit: ${BPF_BASELINE_COMMIT}\n\ 311Checkpoint bpf commit: ${BPF_TIP_COMMIT}/" | \ 312git commit -s --file=- 313 314echo "SUCCESS! ${COMMIT_CNT} commits synced." 315 316echo "Verifying Linux's and Github's libbpf state" 317 318cd_to ${LINUX_REPO} 319git checkout -b ${VIEW_TAG} ${TIP_COMMIT} 320FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --tree-filter "${LIBBPF_TREE_FILTER}" ${VIEW_TAG}^..${VIEW_TAG} 321FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --subdirectory-filter __libbpf ${VIEW_TAG}^..${VIEW_TAG} 322git ls-files -- "${LIBBPF_VIEW_PATHS[@]}" | grep -v -E "${LINUX_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/linux-view.ls 323 324cd_to ${LIBBPF_REPO} 325git ls-files -- "${LIBBPF_VIEW_PATHS[@]}" | grep -v -E "${LIBBPF_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/github-view.ls 326 327echo "Comparing list of files..." 328diff -u ${TMP_DIR}/linux-view.ls ${TMP_DIR}/github-view.ls 329echo "Comparing file contents..." 330CONSISTENT=1 331for F in $(cat ${TMP_DIR}/linux-view.ls); do 332 if ! diff -u "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"; then 333 echo "${LINUX_ABS_DIR}/${F} and ${GITHUB_ABS_DIR}/${F} are different!" 334 CONSISTENT=0 335 fi 336done 337if ((${CONSISTENT} == 1)); then 338 echo "Great! Content is identical!" 339else 340 ignore_inconsistency=n 341 echo "Unfortunately, there are some inconsistencies, please double check." 342 read -p "Does everything look good? [y/N]: " ignore_inconsistency 343 case "${ignore_inconsistency}" in 344 "y" | "Y") 345 echo "Ok, proceeding..." 346 ;; 347 *) 348 echo "Oops, exiting with error..." 349 exit 4 350 esac 351fi 352 353cleanup 354