1#!/bin/bash 2 3usage () { 4 echo "USAGE: ./sync-kernel.sh <libbpf-repo> <kernel-repo> <bpf-branch>" 5 echo "" 6 echo "Set BPF_NEXT_BASELINE to override bpf-next tree commit, otherwise read from <libbpf-repo>/CHECKPOINT-COMMIT." 7 echo "Set BPF_BASELINE to override bpf tree commit, otherwise read from <libbpf-repo>/BPF-CHECKPOINT-COMMIT." 8 echo "Set MANUAL_MODE to 1 to manually control every cherry-picked commits." 9 exit 1 10} 11 12set -eu 13 14LIBBPF_REPO=${1-""} 15LINUX_REPO=${2-""} 16BPF_BRANCH=${3-""} 17BASELINE_COMMIT=${BPF_NEXT_BASELINE:-$(cat ${LIBBPF_REPO}/CHECKPOINT-COMMIT)} 18BPF_BASELINE_COMMIT=${BPF_BASELINE:-$(cat ${LIBBPF_REPO}/BPF-CHECKPOINT-COMMIT)} 19 20if [ -z "${LIBBPF_REPO}" ] || [ -z "${LINUX_REPO}" ]; then 21 echo "Error: libbpf or linux repos are not specified" 22 usage 23fi 24if [ -z "${BPF_BRANCH}" ]; then 25 echo "Error: linux's bpf tree branch is not specified" 26 usage 27fi 28if [ -z "${BASELINE_COMMIT}" ] || [ -z "${BPF_BASELINE_COMMIT}" ]; then 29 echo "Error: bpf or bpf-next baseline commits are not provided" 30 usage 31fi 32 33SUFFIX=$(date --utc +%Y-%m-%dT%H-%M-%S.%3NZ) 34WORKDIR=$(pwd) 35TMP_DIR=$(mktemp -d) 36 37trap "cd ${WORKDIR}; exit" INT TERM EXIT 38 39declare -A PATH_MAP 40PATH_MAP=( \ 41 [tools/lib/bpf]=src \ 42 [tools/include/uapi/linux/bpf_common.h]=include/uapi/linux/bpf_common.h \ 43 [tools/include/uapi/linux/bpf.h]=include/uapi/linux/bpf.h \ 44 [tools/include/uapi/linux/btf.h]=include/uapi/linux/btf.h \ 45 [tools/include/uapi/linux/if_link.h]=include/uapi/linux/if_link.h \ 46 [tools/include/uapi/linux/if_xdp.h]=include/uapi/linux/if_xdp.h \ 47 [tools/include/uapi/linux/netlink.h]=include/uapi/linux/netlink.h \ 48 [tools/include/uapi/linux/pkt_cls.h]=include/uapi/linux/pkt_cls.h \ 49 [tools/include/uapi/linux/pkt_sched.h]=include/uapi/linux/pkt_sched.h \ 50 [include/uapi/linux/perf_event.h]=include/uapi/linux/perf_event.h \ 51 [Documentation/bpf/libbpf]=docs \ 52) 53 54LIBBPF_PATHS="${!PATH_MAP[@]} :^tools/lib/bpf/Makefile :^tools/lib/bpf/Build :^tools/lib/bpf/.gitignore :^tools/include/tools/libc_compat.h" 55LIBBPF_VIEW_PATHS="${PATH_MAP[@]}" 56LIBBPF_VIEW_EXCLUDE_REGEX='^src/(Makefile|Build|test_libbpf\.c|bpf_helper_defs\.h|\.gitignore)$|^docs/(\.gitignore|api\.rst|conf\.py)$|^docs/sphinx/.*' 57LINUX_VIEW_EXCLUDE_REGEX='^include/tools/libc_compat.h$' 58 59LIBBPF_TREE_FILTER="mkdir -p __libbpf/include/uapi/linux __libbpf/include/tools && "$'\\\n' 60for p in "${!PATH_MAP[@]}"; do 61 LIBBPF_TREE_FILTER+="git mv -kf ${p} __libbpf/${PATH_MAP[${p}]} && "$'\\\n' 62done 63LIBBPF_TREE_FILTER+="git rm --ignore-unmatch -f __libbpf/src/{Makefile,Build,test_libbpf.c,.gitignore} >/dev/null" 64 65cd_to() 66{ 67 cd ${WORKDIR} && cd "$1" 68} 69 70# Output brief single-line commit description 71# $1 - commit ref 72commit_desc() 73{ 74 git log -n1 --pretty='%h ("%s")' $1 75} 76 77# Create commit single-line signature, which consists of: 78# - full commit subject 79# - author date in ISO8601 format 80# - full commit body with newlines replaced with vertical bars (|) 81# - shortstat appended at the end 82# The idea is that this single-line signature is good enough to make final 83# decision about whether two commits are the same, across different repos. 84# $1 - commit ref 85# $2 - paths filter 86commit_signature() 87{ 88 git show --pretty='("%s")|%aI|%b' --shortstat $1 -- ${2-.} | tr '\n' '|' 89} 90 91# Cherry-pick commits touching libbpf-related files 92# $1 - baseline_tag 93# $2 - tip_tag 94cherry_pick_commits() 95{ 96 local manual_mode=${MANUAL_MODE:-0} 97 local baseline_tag=$1 98 local tip_tag=$2 99 local new_commits 100 local signature 101 local should_skip 102 local synced_cnt 103 local manual_check 104 local libbpf_conflict_cnt 105 local desc 106 107 new_commits=$(git rev-list --no-merges --topo-order --reverse ${baseline_tag}..${tip_tag} ${LIBBPF_PATHS[@]}) 108 for new_commit in ${new_commits}; do 109 desc="$(commit_desc ${new_commit})" 110 signature="$(commit_signature ${new_commit} "${LIBBPF_PATHS[@]}")" 111 synced_cnt=$(grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | wc -l) 112 manual_check=0 113 if ((${synced_cnt} > 0)); then 114 # commit with the same subject is already in libbpf, but it's 115 # not 100% the same commit, so check with user 116 echo "Commit '${desc}' is synced into libbpf as:" 117 grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | \ 118 cut -d'|' -f1 | sed -e 's/^/- /' 119 if ((${manual_mode} != 1 && ${synced_cnt} == 1)); then 120 echo "Skipping '${desc}' due to unique match..." 121 continue 122 fi 123 if ((${synced_cnt} > 1)); then 124 echo "'${desc} matches multiple commits, please, double-check!" 125 manual_check=1 126 fi 127 fi 128 if ((${manual_mode} == 1 || ${manual_check} == 1)); then 129 read -p "Do you want to skip '${desc}'? [y/N]: " should_skip 130 case "${should_skip}" in 131 "y" | "Y") 132 echo "Skipping '${desc}'..." 133 continue 134 ;; 135 esac 136 fi 137 # commit hasn't been synced into libbpf yet 138 echo "Picking '${desc}'..." 139 if ! git cherry-pick ${new_commit} &>/dev/null; then 140 echo "Warning! Cherry-picking '${desc} failed, checking if it's non-libbpf files causing problems..." 141 libbpf_conflict_cnt=$(git diff --name-only --diff-filter=U -- ${LIBBPF_PATHS[@]} | wc -l) 142 conflict_cnt=$(git diff --name-only | wc -l) 143 prompt_resolution=1 144 145 if ((${libbpf_conflict_cnt} == 0)); then 146 echo "Looks like only non-libbpf files have conflicts, ignoring..." 147 if ((${conflict_cnt} == 0)); then 148 echo "Empty cherry-pick, skipping it..." 149 git cherry-pick --abort 150 continue 151 fi 152 153 git add . 154 # GIT_EDITOR=true to avoid editor popping up to edit commit message 155 if ! GIT_EDITOR=true git cherry-pick --continue &>/dev/null; then 156 echo "Error! That still failed! Please resolve manually." 157 else 158 echo "Success! All cherry-pick conflicts were resolved for '${desc}'!" 159 prompt_resolution=0 160 fi 161 fi 162 163 if ((${prompt_resolution} == 1)); then 164 read -p "Error! Cherry-picking '${desc}' failed, please fix manually and press <return> to proceed..." 165 fi 166 fi 167 # Append signature of just cherry-picked commit to avoid 168 # potentially cherry-picking the same commit twice later when 169 # processing bpf tree commits. At this point we don't know yet 170 # the final commit sha in libbpf repo, so we record Linux SHA 171 # instead as LINUX_<sha>. 172 echo LINUX_$(git log --pretty='%h' -n1) "${signature}" >> ${TMP_DIR}/libbpf_commits.txt 173 done 174} 175 176cleanup() 177{ 178 echo "Cleaning up..." 179 rm -r ${TMP_DIR} 180 cd_to ${LINUX_REPO} 181 git checkout ${TIP_SYM_REF} 182 git branch -D ${BASELINE_TAG} ${TIP_TAG} ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} \ 183 ${SQUASH_BASE_TAG} ${SQUASH_TIP_TAG} ${VIEW_TAG} || true 184 185 cd_to . 186 echo "DONE." 187} 188 189 190cd_to ${LIBBPF_REPO} 191GITHUB_ABS_DIR=$(pwd) 192echo "Dumping existing libbpf commit signatures..." 193for h in $(git log --pretty='%h' -n500); do 194 echo $h "$(commit_signature $h)" >> ${TMP_DIR}/libbpf_commits.txt 195done 196 197# Use current kernel repo HEAD as a source of patches 198cd_to ${LINUX_REPO} 199LINUX_ABS_DIR=$(pwd) 200TIP_SYM_REF=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD) 201TIP_COMMIT=$(git rev-parse HEAD) 202BPF_TIP_COMMIT=$(git rev-parse ${BPF_BRANCH}) 203BASELINE_TAG=libbpf-baseline-${SUFFIX} 204TIP_TAG=libbpf-tip-${SUFFIX} 205BPF_BASELINE_TAG=libbpf-bpf-baseline-${SUFFIX} 206BPF_TIP_TAG=libbpf-bpf-tip-${SUFFIX} 207VIEW_TAG=libbpf-view-${SUFFIX} 208LIBBPF_SYNC_TAG=libbpf-sync-${SUFFIX} 209 210# Squash state of kernel repo at baseline into single commit 211SQUASH_BASE_TAG=libbpf-squash-base-${SUFFIX} 212SQUASH_TIP_TAG=libbpf-squash-tip-${SUFFIX} 213SQUASH_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE SQUASH ${BASELINE_COMMIT}") 214 215echo "WORKDIR: ${WORKDIR}" 216echo "LINUX REPO: ${LINUX_REPO}" 217echo "LIBBPF REPO: ${LIBBPF_REPO}" 218echo "TEMP DIR: ${TMP_DIR}" 219echo "SUFFIX: ${SUFFIX}" 220echo "BASE COMMIT: '$(commit_desc ${BASELINE_COMMIT})'" 221echo "TIP COMMIT: '$(commit_desc ${TIP_COMMIT})'" 222echo "BPF BASE COMMIT: '$(commit_desc ${BPF_BASELINE_COMMIT})'" 223echo "BPF TIP COMMIT: '$(commit_desc ${BPF_TIP_COMMIT})'" 224echo "SQUASH COMMIT: ${SQUASH_COMMIT}" 225echo "BASELINE TAG: ${BASELINE_TAG}" 226echo "TIP TAG: ${TIP_TAG}" 227echo "BPF BASELINE TAG: ${BPF_BASELINE_TAG}" 228echo "BPF TIP TAG: ${BPF_TIP_TAG}" 229echo "SQUASH BASE TAG: ${SQUASH_BASE_TAG}" 230echo "SQUASH TIP TAG: ${SQUASH_TIP_TAG}" 231echo "VIEW TAG: ${VIEW_TAG}" 232echo "LIBBPF SYNC TAG: ${LIBBPF_SYNC_TAG}" 233echo "PATCHES: ${TMP_DIR}/patches" 234 235git branch ${BASELINE_TAG} ${BASELINE_COMMIT} 236git branch ${TIP_TAG} ${TIP_COMMIT} 237git branch ${BPF_BASELINE_TAG} ${BPF_BASELINE_COMMIT} 238git branch ${BPF_TIP_TAG} ${BPF_TIP_COMMIT} 239git branch ${SQUASH_BASE_TAG} ${SQUASH_COMMIT} 240git checkout -b ${SQUASH_TIP_TAG} ${SQUASH_COMMIT} 241 242# Cherry-pick new commits onto squashed baseline commit 243cherry_pick_commits ${BASELINE_TAG} ${TIP_TAG} 244cherry_pick_commits ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} 245 246# Move all libbpf files into __libbpf directory. 247FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --tree-filter "${LIBBPF_TREE_FILTER}" ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG} 248# Make __libbpf a new root directory 249FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --subdirectory-filter __libbpf ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG} 250 251# If there are no new commits with libbpf-related changes, bail out 252COMMIT_CNT=$(git rev-list --count ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG}) 253if ((${COMMIT_CNT} <= 0)); then 254 echo "No new changes to apply, we are done!" 255 cleanup 256 exit 2 257fi 258 259# Exclude baseline commit and generate nice cover letter with summary 260git format-patch ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --cover-letter -o ${TMP_DIR}/patches 261 262# Now is time to re-apply libbpf-related linux patches to libbpf repo 263cd_to ${LIBBPF_REPO} 264git checkout -b ${LIBBPF_SYNC_TAG} 265 266for patch in $(ls -1 ${TMP_DIR}/patches | tail -n +2); do 267 if ! git am -3 --committer-date-is-author-date "${TMP_DIR}/patches/${patch}"; then 268 if ! patch -p1 --merge < "${TMP_DIR}/patches/${patch}"; then 269 read -p "Applying ${TMP_DIR}/patches/${patch} failed, please resolve manually and press <return> to proceed..." 270 fi 271 git am --continue 272 fi 273done 274 275# Generate bpf_helper_defs.h and commit, if anything changed 276# restore Linux tip to use bpf_doc.py 277cd_to ${LINUX_REPO} 278git checkout ${TIP_TAG} 279# re-generate bpf_helper_defs.h 280cd_to ${LIBBPF_REPO} 281"${LINUX_ABS_DIR}/scripts/bpf_doc.py" --header \ 282 --file include/uapi/linux/bpf.h > src/bpf_helper_defs.h 283# if anything changed, commit it 284helpers_changes=$(git status --porcelain src/bpf_helper_defs.h | wc -l) 285if ((${helpers_changes} == 1)); then 286 git add src/bpf_helper_defs.h 287 git commit -m "sync: auto-generate latest BPF helpers 288 289Latest changes to BPF helper definitions. 290" -- src/bpf_helper_defs.h 291fi 292 293# Use generated cover-letter as a template for "sync commit" with 294# baseline and checkpoint commits from kernel repo (and leave summary 295# from cover letter intact, of course) 296echo ${TIP_COMMIT} > CHECKPOINT-COMMIT && \ 297echo ${BPF_TIP_COMMIT} > BPF-CHECKPOINT-COMMIT && \ 298git add CHECKPOINT-COMMIT && \ 299git add BPF-CHECKPOINT-COMMIT && \ 300awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' ${TMP_DIR}/patches/0000-cover-letter.patch | \ 301sed "s/\*\*\* BLURB HERE \*\*\*/\ 302sync: latest libbpf changes from kernel\n\ 303\n\ 304Syncing latest libbpf commits from kernel repository.\n\ 305Baseline bpf-next commit: ${BASELINE_COMMIT}\n\ 306Checkpoint bpf-next commit: ${TIP_COMMIT}\n\ 307Baseline bpf commit: ${BPF_BASELINE_COMMIT}\n\ 308Checkpoint bpf commit: ${BPF_TIP_COMMIT}/" | \ 309git commit --file=- 310 311echo "SUCCESS! ${COMMIT_CNT} commits synced." 312 313echo "Verifying Linux's and Github's libbpf state" 314 315cd_to ${LINUX_REPO} 316git checkout -b ${VIEW_TAG} ${TIP_COMMIT} 317FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --tree-filter "${LIBBPF_TREE_FILTER}" ${VIEW_TAG}^..${VIEW_TAG} 318FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --subdirectory-filter __libbpf ${VIEW_TAG}^..${VIEW_TAG} 319git ls-files -- ${LIBBPF_VIEW_PATHS[@]} | grep -v -E "${LINUX_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/linux-view.ls 320 321cd_to ${LIBBPF_REPO} 322git ls-files -- ${LIBBPF_VIEW_PATHS[@]} | grep -v -E "${LIBBPF_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/github-view.ls 323 324echo "Comparing list of files..." 325diff -u ${TMP_DIR}/linux-view.ls ${TMP_DIR}/github-view.ls 326echo "Comparing file contents..." 327CONSISTENT=1 328for F in $(cat ${TMP_DIR}/linux-view.ls); do 329 if ! diff -u "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"; then 330 echo "${LINUX_ABS_DIR}/${F} and ${GITHUB_ABS_DIR}/${F} are different!" 331 CONSISTENT=0 332 fi 333done 334if ((${CONSISTENT} == 1)); then 335 echo "Great! Content is identical!" 336else 337 ignore_inconsistency=n 338 echo "Unfortunately, there are some inconsistencies, please double check." 339 read -p "Does everything look good? [y/N]: " ignore_inconsistency 340 case "${ignore_inconsistency}" in 341 "y" | "Y") 342 echo "Ok, proceeding..." 343 ;; 344 *) 345 echo "Oops, exiting with error..." 346 exit 4 347 esac 348fi 349 350cleanup 351