1 //===-- Benchmark ---------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "JSON.h"
10 #include "LibcBenchmark.h"
11 #include "LibcMemoryBenchmark.h"
12 #include "MemorySizeDistributions.h"
13 #include "src/__support/macros/config.h"
14 #include "llvm/Support/CommandLine.h"
15 #include "llvm/Support/ErrorHandling.h"
16 #include "llvm/Support/FileSystem.h"
17 #include "llvm/Support/JSON.h"
18 #include "llvm/Support/MathExtras.h"
19 #include "llvm/Support/MemoryBuffer.h"
20 #include "llvm/Support/raw_ostream.h"
21
22 #include <cstring>
23 #include <unistd.h>
24
25 namespace LIBC_NAMESPACE_DECL {
26
27 extern void *memcpy(void *__restrict, const void *__restrict, size_t);
28 extern void *memmove(void *, const void *, size_t);
29 extern void *memset(void *, int, size_t);
30 extern void bzero(void *, size_t);
31 extern int memcmp(const void *, const void *, size_t);
32 extern int bcmp(const void *, const void *, size_t);
33
34 } // namespace LIBC_NAMESPACE_DECL
35
36 namespace llvm {
37 namespace libc_benchmarks {
38
39 static cl::opt<std::string>
40 StudyName("study-name", cl::desc("The name for this study"), cl::Required);
41
42 static cl::opt<std::string>
43 SizeDistributionName("size-distribution-name",
44 cl::desc("The name of the distribution to use"));
45
46 static cl::opt<bool> SweepMode(
47 "sweep-mode",
48 cl::desc(
49 "If set, benchmark all sizes from sweep-min-size to sweep-max-size"));
50
51 static cl::opt<uint32_t>
52 SweepMinSize("sweep-min-size",
53 cl::desc("The minimum size to use in sweep-mode"),
54 cl::init(0));
55
56 static cl::opt<uint32_t>
57 SweepMaxSize("sweep-max-size",
58 cl::desc("The maximum size to use in sweep-mode"),
59 cl::init(256));
60
61 static cl::opt<uint32_t>
62 AlignedAccess("aligned-access",
63 cl::desc("The alignment to use when accessing the buffers\n"
64 "Default is unaligned\n"
65 "Use 0 to disable address randomization"),
66 cl::init(1));
67
68 static cl::opt<std::string> Output("output",
69 cl::desc("Specify output filename"),
70 cl::value_desc("filename"), cl::init("-"));
71
72 static cl::opt<uint32_t>
73 NumTrials("num-trials", cl::desc("The number of benchmarks run to perform"),
74 cl::init(1));
75
76 #if defined(LIBC_BENCHMARK_FUNCTION_MEMCPY)
77 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCPY
78 using BenchmarkSetup = CopySetup;
79 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMMOVE)
80 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMMOVE
81 using BenchmarkSetup = MoveSetup;
82 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMSET)
83 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMSET
84 using BenchmarkSetup = SetSetup;
85 #elif defined(LIBC_BENCHMARK_FUNCTION_BZERO)
86 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BZERO
87 using BenchmarkSetup = SetSetup;
88 #elif defined(LIBC_BENCHMARK_FUNCTION_MEMCMP)
89 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_MEMCMP
90 using BenchmarkSetup = ComparisonSetup;
91 #elif defined(LIBC_BENCHMARK_FUNCTION_BCMP)
92 #define LIBC_BENCHMARK_FUNCTION LIBC_BENCHMARK_FUNCTION_BCMP
93 using BenchmarkSetup = ComparisonSetup;
94 #else
95 #error "Missing LIBC_BENCHMARK_FUNCTION_XXX definition"
96 #endif
97
98 struct MemfunctionBenchmarkBase : public BenchmarkSetup {
MemfunctionBenchmarkBasellvm::libc_benchmarks::MemfunctionBenchmarkBase99 MemfunctionBenchmarkBase() : ReportProgress(isatty(fileno(stdout))) {}
~MemfunctionBenchmarkBasellvm::libc_benchmarks::MemfunctionBenchmarkBase100 virtual ~MemfunctionBenchmarkBase() {}
101
102 virtual Study run() = 0;
103
104 CircularArrayRef<ParameterBatch::ParameterType>
generateBatchllvm::libc_benchmarks::MemfunctionBenchmarkBase105 generateBatch(size_t Iterations) {
106 randomize();
107 return cycle(ArrayRef(Parameters), Iterations);
108 }
109
110 protected:
createStudyllvm::libc_benchmarks::MemfunctionBenchmarkBase111 Study createStudy() {
112 Study Study;
113 // Setup study.
114 Study.StudyName = StudyName;
115 Runtime &RI = Study.Runtime;
116 RI.Host = HostState::get();
117 RI.BufferSize = BufferSize;
118 RI.BatchParameterCount = BatchSize;
119
120 BenchmarkOptions &BO = RI.BenchmarkOptions;
121 BO.MinDuration = std::chrono::milliseconds(1);
122 BO.MaxDuration = std::chrono::seconds(1);
123 BO.MaxIterations = 10'000'000U;
124 BO.MinSamples = 4;
125 BO.MaxSamples = 1000;
126 BO.Epsilon = 0.01; // 1%
127 BO.ScalingFactor = 1.4;
128
129 StudyConfiguration &SC = Study.Configuration;
130 SC.NumTrials = NumTrials;
131 SC.IsSweepMode = SweepMode;
132 SC.AccessAlignment = MaybeAlign(AlignedAccess);
133 SC.Function = LIBC_BENCHMARK_FUNCTION_NAME;
134 return Study;
135 }
136
runTrialsllvm::libc_benchmarks::MemfunctionBenchmarkBase137 void runTrials(const BenchmarkOptions &Options,
138 std::vector<Duration> &Measurements) {
139 for (size_t i = 0; i < NumTrials; ++i) {
140 const BenchmarkResult Result = benchmark(
141 Options, *this, [this](ParameterBatch::ParameterType Parameter) {
142 return Call(Parameter, LIBC_BENCHMARK_FUNCTION);
143 });
144 Measurements.push_back(Result.BestGuess);
145 reportProgress(Measurements);
146 }
147 }
148
149 virtual void randomize() = 0;
150
151 private:
152 bool ReportProgress;
153
reportProgressllvm::libc_benchmarks::MemfunctionBenchmarkBase154 void reportProgress(const std::vector<Duration> &Measurements) {
155 if (!ReportProgress)
156 return;
157 static size_t LastPercent = -1;
158 const size_t TotalSteps = Measurements.capacity();
159 const size_t Steps = Measurements.size();
160 const size_t Percent = 100 * Steps / TotalSteps;
161 if (Percent == LastPercent)
162 return;
163 LastPercent = Percent;
164 size_t I = 0;
165 errs() << '[';
166 for (; I <= Percent; ++I)
167 errs() << '#';
168 for (; I <= 100; ++I)
169 errs() << '_';
170 errs() << "] " << Percent << '%' << '\r';
171 }
172 };
173
174 struct MemfunctionBenchmarkSweep final : public MemfunctionBenchmarkBase {
MemfunctionBenchmarkSweepllvm::libc_benchmarks::MemfunctionBenchmarkSweep175 MemfunctionBenchmarkSweep()
176 : OffsetSampler(MemfunctionBenchmarkBase::BufferSize, SweepMaxSize,
177 MaybeAlign(AlignedAccess)) {}
178
randomizellvm::libc_benchmarks::MemfunctionBenchmarkSweep179 virtual void randomize() override {
180 for (auto &P : Parameters) {
181 P.OffsetBytes = OffsetSampler(Gen);
182 P.SizeBytes = CurrentSweepSize;
183 checkValid(P);
184 }
185 }
186
runllvm::libc_benchmarks::MemfunctionBenchmarkSweep187 virtual Study run() override {
188 Study Study = createStudy();
189 Study.Configuration.SweepModeMaxSize = SweepMaxSize;
190 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
191 BO.MinDuration = std::chrono::milliseconds(1);
192 BO.InitialIterations = 100;
193 auto &Measurements = Study.Measurements;
194 Measurements.reserve(NumTrials * SweepMaxSize);
195 for (size_t Size = SweepMinSize; Size <= SweepMaxSize; ++Size) {
196 CurrentSweepSize = Size;
197 runTrials(BO, Measurements);
198 }
199 return Study;
200 }
201
202 private:
203 size_t CurrentSweepSize = 0;
204 OffsetDistribution OffsetSampler;
205 std::mt19937_64 Gen;
206 };
207
208 struct MemfunctionBenchmarkDistribution final
209 : public MemfunctionBenchmarkBase {
MemfunctionBenchmarkDistributionllvm::libc_benchmarks::MemfunctionBenchmarkDistribution210 MemfunctionBenchmarkDistribution(MemorySizeDistribution Distribution)
211 : Distribution(Distribution), Probabilities(Distribution.Probabilities),
212 SizeSampler(Probabilities.begin(), Probabilities.end()),
213 OffsetSampler(MemfunctionBenchmarkBase::BufferSize,
214 Probabilities.size() - 1, MaybeAlign(AlignedAccess)) {}
215
randomizellvm::libc_benchmarks::MemfunctionBenchmarkDistribution216 virtual void randomize() override {
217 for (auto &P : Parameters) {
218 P.OffsetBytes = OffsetSampler(Gen);
219 P.SizeBytes = SizeSampler(Gen);
220 checkValid(P);
221 }
222 }
223
runllvm::libc_benchmarks::MemfunctionBenchmarkDistribution224 virtual Study run() override {
225 Study Study = createStudy();
226 Study.Configuration.SizeDistributionName = Distribution.Name.str();
227 BenchmarkOptions &BO = Study.Runtime.BenchmarkOptions;
228 BO.MinDuration = std::chrono::milliseconds(10);
229 BO.InitialIterations = BatchSize * 10;
230 auto &Measurements = Study.Measurements;
231 Measurements.reserve(NumTrials);
232 runTrials(BO, Measurements);
233 return Study;
234 }
235
236 private:
237 MemorySizeDistribution Distribution;
238 ArrayRef<double> Probabilities;
239 std::discrete_distribution<unsigned> SizeSampler;
240 OffsetDistribution OffsetSampler;
241 std::mt19937_64 Gen;
242 };
243
writeStudy(const Study & S)244 void writeStudy(const Study &S) {
245 std::error_code EC;
246 raw_fd_ostream FOS(Output, EC);
247 if (EC)
248 report_fatal_error(Twine("Could not open file: ")
249 .concat(EC.message())
250 .concat(", ")
251 .concat(Output));
252 json::OStream JOS(FOS);
253 serializeToJson(S, JOS);
254 FOS << "\n";
255 }
256
main()257 void main() {
258 checkRequirements();
259 if (!isPowerOf2_32(AlignedAccess))
260 report_fatal_error(AlignedAccess.ArgStr +
261 Twine(" must be a power of two or zero"));
262
263 const bool HasDistributionName = !SizeDistributionName.empty();
264 if (SweepMode && HasDistributionName)
265 report_fatal_error("Select only one of `--" + Twine(SweepMode.ArgStr) +
266 "` or `--" + Twine(SizeDistributionName.ArgStr) + "`");
267
268 std::unique_ptr<MemfunctionBenchmarkBase> Benchmark;
269 if (SweepMode)
270 Benchmark.reset(new MemfunctionBenchmarkSweep());
271 else
272 Benchmark.reset(new MemfunctionBenchmarkDistribution(getDistributionOrDie(
273 BenchmarkSetup::getDistributions(), SizeDistributionName)));
274 writeStudy(Benchmark->run());
275 }
276
277 } // namespace libc_benchmarks
278 } // namespace llvm
279
280 #ifndef NDEBUG
281 #error For reproducibility benchmarks should not be compiled in DEBUG mode.
282 #endif
283
main(int argc,char ** argv)284 int main(int argc, char **argv) {
285 llvm::cl::ParseCommandLineOptions(argc, argv);
286 llvm::libc_benchmarks::main();
287 return EXIT_SUCCESS;
288 }
289