1#!/usr/bin/python3 2# 3# Protocol Buffers - Google's data interchange format 4# Copyright 2023 Google LLC. All rights reserved. 5# https://developers.google.com/protocol-buffers/ 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions are 9# met: 10# 11# * Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# * Redistributions in binary form must reproduce the above 14# copyright notice, this list of conditions and the following disclaimer 15# in the documentation and/or other materials provided with the 16# distribution. 17# * Neither the name of Google LLC nor the names of its 18# contributors may be used to endorse or promote products derived from 19# this software without specific prior written permission. 20# 21# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 33"""Benchmarks the current working directory against a given baseline. 34 35This script benchmarks both size and speed. Sample output: 36""" 37 38import contextlib 39import json 40import os 41import re 42import subprocess 43import sys 44import tempfile 45 46@contextlib.contextmanager 47def GitWorktree(commit): 48 tmpdir = tempfile.mkdtemp() 49 subprocess.run(['git', 'worktree', 'add', '-q', '-d', tmpdir, commit], check=True) 50 cwd = os.getcwd() 51 os.chdir(tmpdir) 52 try: 53 yield tmpdir 54 finally: 55 os.chdir(cwd) 56 subprocess.run(['git', 'worktree', 'remove', tmpdir], check=True) 57 58def Run(cmd): 59 subprocess.check_call(cmd, shell=True) 60 61def Benchmark(outbase, bench_cpu=True, runs=12, fasttable=False): 62 tmpfile = "/tmp/bench-output.json" 63 Run("rm -rf {}".format(tmpfile)) 64 #Run("CC=clang bazel test ...") 65 if fasttable: 66 extra_args = " --//:fasttable_enabled=true" 67 else: 68 extra_args = "" 69 70 if bench_cpu: 71 Run("CC=clang bazel build -c opt --copt=-march=native benchmarks:benchmark" + extra_args) 72 Run("./bazel-bin/benchmarks/benchmark --benchmark_out_format=json --benchmark_out={} --benchmark_repetitions={} --benchmark_min_time=0.05 --benchmark_enable_random_interleaving=true".format(tmpfile, runs)) 73 with open(tmpfile) as f: 74 bench_json = json.load(f) 75 76 # Translate into the format expected by benchstat. 77 txt_filename = outbase + ".txt" 78 with open(txt_filename, "w") as f: 79 for run in bench_json["benchmarks"]: 80 if run["run_type"] == "aggregate": 81 continue 82 name = run["name"] 83 name = name.replace(" ", "") 84 name = re.sub(r'^BM_', 'Benchmark', name) 85 values = (name, run["iterations"], run["cpu_time"]) 86 print("{} {} {} ns/op".format(*values), file=f) 87 Run("sort {} -o {} ".format(txt_filename, txt_filename)) 88 89 Run("CC=clang bazel build -c opt --copt=-g --copt=-march=native :conformance_upb" 90 + extra_args) 91 Run("cp -f bazel-bin/conformance_upb {}.bin".format(outbase)) 92 93 94baseline = "main" 95bench_cpu = True 96fasttable = False 97 98if len(sys.argv) > 1: 99 baseline = sys.argv[1] 100 101 # Quickly verify that the baseline exists. 102 with GitWorktree(baseline): 103 pass 104 105# Benchmark our current directory first, since it's more likely to be broken. 106Benchmark("/tmp/new", bench_cpu, fasttable=fasttable) 107 108# Benchmark the baseline. 109with GitWorktree(baseline): 110 Benchmark("/tmp/old", bench_cpu, fasttable=fasttable) 111 112print() 113print() 114 115if bench_cpu: 116 Run("~/go/bin/benchstat /tmp/old.txt /tmp/new.txt") 117 118print() 119print() 120 121Run("objcopy --strip-debug /tmp/old.bin /tmp/old.bin.stripped") 122Run("objcopy --strip-debug /tmp/new.bin /tmp/new.bin.stripped") 123Run("~/code/bloaty/bloaty /tmp/new.bin.stripped -- /tmp/old.bin.stripped --debug-file=/tmp/old.bin --debug-file=/tmp/new.bin -d compileunits,symbols") 124