1 /* 2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"). 5 * You may not use this file except in compliance with the License. 6 * A copy of the License is located at 7 * 8 * http://aws.amazon.com/apache2.0 9 * 10 * or in the "license" file accompanying this file. This file is distributed 11 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 * express or implied. See the License for the specific language governing 13 * permissions and limitations under the License. 14 */ 15 16 package software.amazon.awssdk.benchmark; 17 18 import static software.amazon.awssdk.benchmark.utils.BenchmarkConstant.OBJECT_MAPPER; 19 20 import java.io.IOException; 21 import java.io.OutputStream; 22 import java.nio.file.Files; 23 import java.nio.file.Path; 24 import java.nio.file.Paths; 25 import java.util.ArrayList; 26 import java.util.Arrays; 27 import java.util.Collection; 28 import java.util.List; 29 import java.util.stream.Collectors; 30 import org.apache.commons.cli.CommandLine; 31 import org.apache.commons.cli.CommandLineParser; 32 import org.apache.commons.cli.DefaultParser; 33 import org.apache.commons.cli.Options; 34 import org.apache.commons.cli.ParseException; 35 import org.openjdk.jmh.results.RunResult; 36 import org.openjdk.jmh.runner.Runner; 37 import org.openjdk.jmh.runner.RunnerException; 38 import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; 39 import org.openjdk.jmh.runner.options.OptionsBuilder; 40 import software.amazon.awssdk.benchmark.apicall.MetricsEnabledBenchmark; 41 import software.amazon.awssdk.benchmark.apicall.httpclient.async.AwsCrtClientBenchmark; 42 import software.amazon.awssdk.benchmark.apicall.httpclient.async.NettyHttpClientH1Benchmark; 43 import software.amazon.awssdk.benchmark.apicall.httpclient.async.NettyHttpClientH2Benchmark; 44 import software.amazon.awssdk.benchmark.apicall.httpclient.sync.ApacheHttpClientBenchmark; 45 import software.amazon.awssdk.benchmark.apicall.httpclient.sync.CrtHttpClientBenchmark; 46 import software.amazon.awssdk.benchmark.apicall.httpclient.sync.UrlConnectionHttpClientBenchmark; 47 import software.amazon.awssdk.benchmark.apicall.protocol.Ec2ProtocolBenchmark; 48 import software.amazon.awssdk.benchmark.apicall.protocol.JsonProtocolBenchmark; 49 import software.amazon.awssdk.benchmark.apicall.protocol.QueryProtocolBenchmark; 50 import software.amazon.awssdk.benchmark.apicall.protocol.XmlProtocolBenchmark; 51 import software.amazon.awssdk.benchmark.coldstart.V2DefaultClientCreationBenchmark; 52 import software.amazon.awssdk.benchmark.coldstart.V2OptimizedClientCreationBenchmark; 53 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientDeleteV1MapperComparisonBenchmark; 54 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientGetOverheadBenchmark; 55 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientGetV1MapperComparisonBenchmark; 56 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientPutOverheadBenchmark; 57 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientPutV1MapperComparisonBenchmark; 58 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientQueryV1MapperComparisonBenchmark; 59 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientScanV1MapperComparisonBenchmark; 60 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientUpdateV1MapperComparisonBenchmark; 61 import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult; 62 import software.amazon.awssdk.benchmark.utils.BenchmarkProcessorOutput; 63 import software.amazon.awssdk.utils.Logger; 64 65 66 public class BenchmarkRunner { 67 68 private static final List<String> PROTOCOL_BENCHMARKS = Arrays.asList( 69 Ec2ProtocolBenchmark.class.getSimpleName(), JsonProtocolBenchmark.class.getSimpleName(), 70 QueryProtocolBenchmark.class.getSimpleName(), XmlProtocolBenchmark.class.getSimpleName()); 71 72 private static final List<String> ASYNC_BENCHMARKS = Arrays.asList( 73 NettyHttpClientH2Benchmark.class.getSimpleName(), 74 NettyHttpClientH1Benchmark.class.getSimpleName(), 75 AwsCrtClientBenchmark.class.getSimpleName()); 76 77 private static final List<String> SYNC_BENCHMARKS = Arrays.asList( 78 ApacheHttpClientBenchmark.class.getSimpleName(), 79 UrlConnectionHttpClientBenchmark.class.getSimpleName(), 80 CrtHttpClientBenchmark.class.getSimpleName()); 81 82 private static final List<String> COLD_START_BENCHMARKS = Arrays.asList( 83 V2OptimizedClientCreationBenchmark.class.getSimpleName(), 84 V2DefaultClientCreationBenchmark.class.getSimpleName()); 85 86 private static final List<String> MAPPER_BENCHMARKS = Arrays.asList( 87 EnhancedClientGetOverheadBenchmark.class.getSimpleName(), 88 EnhancedClientPutOverheadBenchmark.class.getSimpleName(), 89 EnhancedClientGetV1MapperComparisonBenchmark.class.getSimpleName(), 90 EnhancedClientPutV1MapperComparisonBenchmark.class.getSimpleName(), 91 EnhancedClientUpdateV1MapperComparisonBenchmark.class.getSimpleName(), 92 EnhancedClientDeleteV1MapperComparisonBenchmark.class.getSimpleName(), 93 EnhancedClientScanV1MapperComparisonBenchmark.class.getSimpleName(), 94 EnhancedClientQueryV1MapperComparisonBenchmark.class.getSimpleName() 95 ); 96 97 private static final List<String> METRIC_BENCHMARKS = Arrays.asList(MetricsEnabledBenchmark.class.getSimpleName()); 98 99 private static final Logger log = Logger.loggerFor(BenchmarkRunner.class); 100 101 private final List<String> benchmarksToRun; 102 private final BenchmarkResultProcessor resultProcessor; 103 private final BenchmarkRunnerOptions options; 104 BenchmarkRunner(List<String> benchmarksToRun, BenchmarkRunnerOptions options)105 private BenchmarkRunner(List<String> benchmarksToRun, BenchmarkRunnerOptions options) { 106 this.benchmarksToRun = benchmarksToRun; 107 this.resultProcessor = new BenchmarkResultProcessor(); 108 this.options = options; 109 } 110 main(String... args)111 public static void main(String... args) throws Exception { 112 List<String> benchmarksToRun = new ArrayList<>(); 113 benchmarksToRun.addAll(SYNC_BENCHMARKS); 114 benchmarksToRun.addAll(ASYNC_BENCHMARKS); 115 benchmarksToRun.addAll(PROTOCOL_BENCHMARKS); 116 benchmarksToRun.addAll(COLD_START_BENCHMARKS); 117 118 log.info(() -> "Skipping tests, to reduce benchmark times: \n" + MAPPER_BENCHMARKS + "\n" + METRIC_BENCHMARKS); 119 120 BenchmarkRunner runner = new BenchmarkRunner(benchmarksToRun, parseOptions(args)); 121 122 runner.runBenchmark(); 123 } 124 runBenchmark()125 private void runBenchmark() throws RunnerException { 126 log.info(() -> "Running with options: " + options); 127 128 ChainedOptionsBuilder optionsBuilder = new OptionsBuilder(); 129 130 benchmarksToRun.forEach(optionsBuilder::include); 131 132 log.info(() -> "Starting to run: " + benchmarksToRun); 133 134 Collection<RunResult> results = new Runner(optionsBuilder.build()).run(); 135 136 BenchmarkProcessorOutput processedResults = resultProcessor.processBenchmarkResult(results); 137 List<String> failedResults = processedResults.getFailedBenchmarks(); 138 139 if (options.outputPath != null) { 140 log.info(() -> "Writing results to " + options.outputPath); 141 writeResults(processedResults, options.outputPath); 142 } 143 144 if (options.check && !failedResults.isEmpty()) { 145 log.info(() -> "Failed perf regression tests: " + failedResults); 146 throw new RuntimeException("Perf regression tests failed: " + failedResults); 147 } 148 } 149 parseOptions(String[] args)150 private static BenchmarkRunnerOptions parseOptions(String[] args) throws ParseException { 151 Options cliOptions = new Options(); 152 cliOptions.addOption("o", "output", true, 153 "The path to write the benchmark results to."); 154 cliOptions.addOption("c", "check", false, 155 "If specified, exit with error code 1 if the results are not within the baseline."); 156 157 CommandLineParser parser = new DefaultParser(); 158 CommandLine cmdLine = parser.parse(cliOptions, args); 159 160 BenchmarkRunnerOptions options = new BenchmarkRunnerOptions() 161 .check(cmdLine.hasOption("c")); 162 163 if (cmdLine.hasOption("o")) { 164 options.outputPath(Paths.get(cmdLine.getOptionValue("o"))); 165 } 166 167 return options; 168 } 169 writeResults(BenchmarkProcessorOutput output, Path outputPath)170 private static void writeResults(BenchmarkProcessorOutput output, Path outputPath) { 171 List<SdkBenchmarkResult> results = output.getBenchmarkResults().values().stream().collect(Collectors.toList()); 172 try (OutputStream os = Files.newOutputStream(outputPath)) { 173 OBJECT_MAPPER.writeValue(os, results); 174 } catch (IOException e) { 175 log.error(() -> "Failed to write the results to " + outputPath, e); 176 throw new RuntimeException(e); 177 } 178 } 179 180 private static class BenchmarkRunnerOptions { 181 private Path outputPath; 182 private boolean check; 183 outputPath(Path outputPath)184 public BenchmarkRunnerOptions outputPath(Path outputPath) { 185 this.outputPath = outputPath; 186 return this; 187 } 188 check(boolean check)189 public BenchmarkRunnerOptions check(boolean check) { 190 this.check = check; 191 return this; 192 } 193 194 @Override toString()195 public String toString() { 196 return "BenchmarkRunnerOptions{" + 197 "outputPath=" + outputPath + 198 ", check=" + check + 199 '}'; 200 } 201 } 202 } 203