1 /* 2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"). 5 * You may not use this file except in compliance with the License. 6 * A copy of the License is located at 7 * 8 * http://aws.amazon.com/apache2.0 9 * 10 * or in the "license" file accompanying this file. This file is distributed 11 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 * express or implied. See the License for the specific language governing 13 * permissions and limitations under the License. 14 */ 15 16 package software.amazon.awssdk.benchmark; 17 18 import static software.amazon.awssdk.benchmark.utils.BenchmarkConstant.OBJECT_MAPPER; 19 import static software.amazon.awssdk.benchmark.utils.BenchmarkUtils.compare; 20 21 import com.fasterxml.jackson.core.JsonProcessingException; 22 import com.fasterxml.jackson.core.type.TypeReference; 23 import java.io.IOException; 24 import java.net.URL; 25 import java.util.ArrayList; 26 import java.util.Collection; 27 import java.util.HashMap; 28 import java.util.List; 29 import java.util.Map; 30 import java.util.Objects; 31 import java.util.stream.Collectors; 32 import org.openjdk.jmh.infra.BenchmarkParams; 33 import org.openjdk.jmh.results.RunResult; 34 import org.openjdk.jmh.util.Statistics; 35 import software.amazon.awssdk.benchmark.stats.SdkBenchmarkParams; 36 import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult; 37 import software.amazon.awssdk.benchmark.stats.SdkBenchmarkStatistics; 38 import software.amazon.awssdk.benchmark.utils.BenchmarkProcessorOutput; 39 import software.amazon.awssdk.utils.Logger; 40 41 42 /** 43 * Process benchmark score, compare the result with the baseline score and return 44 * the names of the benchmarks that exceed the baseline score. 45 */ 46 class BenchmarkResultProcessor { 47 48 private static final Logger log = Logger.loggerFor(BenchmarkResultProcessor.class); 49 private static final double TOLERANCE_LEVEL = 0.05; 50 51 private Map<String, SdkBenchmarkResult> baseline; 52 53 private List<String> failedBenchmarkIds = new ArrayList<>(); 54 BenchmarkResultProcessor()55 BenchmarkResultProcessor() { 56 try { 57 URL file = BenchmarkResultProcessor.class.getResource("baseline.json"); 58 List<SdkBenchmarkResult> baselineResults = 59 OBJECT_MAPPER.readValue(file, new TypeReference<List<SdkBenchmarkResult>>() {}); 60 61 baseline = baselineResults.stream().collect(Collectors.toMap(SdkBenchmarkResult::getId, b -> b)); 62 } catch (Exception e) { 63 throw new RuntimeException("Not able to retrieve baseline result.", e); 64 } 65 } 66 67 /** 68 * Process benchmark results 69 * 70 * @param results the results of the benchmark 71 * @return the benchmark results 72 */ processBenchmarkResult(Collection<RunResult> results)73 BenchmarkProcessorOutput processBenchmarkResult(Collection<RunResult> results) { 74 Map<String, SdkBenchmarkResult> benchmarkResults = new HashMap<>(); 75 76 for (RunResult result : results) { 77 String benchmarkId = getBenchmarkId(result.getParams()); 78 SdkBenchmarkResult sdkBenchmarkData = constructSdkBenchmarkResult(result); 79 80 benchmarkResults.put(benchmarkId, sdkBenchmarkData); 81 82 SdkBenchmarkResult baselineResult = baseline.get(benchmarkId); 83 84 if (baselineResult == null) { 85 log.warn(() -> { 86 String benchmarkResultJson = null; 87 try { 88 benchmarkResultJson = OBJECT_MAPPER.writeValueAsString(sdkBenchmarkData); 89 } catch (IOException e) { 90 log.error(() -> "Unable to serialize result data to JSON"); 91 } 92 return String.format("Unable to find the baseline for %s. Skipping regression validation. " + 93 "Results were: %s", benchmarkId, benchmarkResultJson); 94 }); 95 continue; 96 } 97 98 if (!validateBenchmarkResult(sdkBenchmarkData, baselineResult)) { 99 failedBenchmarkIds.add(benchmarkId); 100 } 101 } 102 103 BenchmarkProcessorOutput output = new BenchmarkProcessorOutput(benchmarkResults, failedBenchmarkIds); 104 log.info(() -> "Current result: " + serializeResult(output)); 105 return output; 106 } 107 constructSdkBenchmarkResult(RunResult runResult)108 private SdkBenchmarkResult constructSdkBenchmarkResult(RunResult runResult) { 109 Statistics statistics = runResult.getPrimaryResult().getStatistics(); 110 111 SdkBenchmarkStatistics sdkBenchmarkStatistics = new SdkBenchmarkStatistics(statistics); 112 SdkBenchmarkParams sdkBenchmarkParams = new SdkBenchmarkParams(runResult.getParams()); 113 114 return new SdkBenchmarkResult(getBenchmarkId(runResult.getParams()), 115 sdkBenchmarkParams, 116 sdkBenchmarkStatistics); 117 } 118 119 /** 120 * Validate benchmark result by comparing it with baseline result statistically. 121 * 122 * @param baseline the baseline result 123 * @param currentResult current result 124 * @return true if current result is equal to or better than the baseline result statistically, false otherwise. 125 */ validateBenchmarkResult(SdkBenchmarkResult currentResult, SdkBenchmarkResult baseline)126 private boolean validateBenchmarkResult(SdkBenchmarkResult currentResult, SdkBenchmarkResult baseline) { 127 if (!validateBenchmarkParams(currentResult.getParams(), baseline.getParams())) { 128 log.warn(() -> "Baseline result and current result are not comparable due to running from different environments." 129 + "Skipping validation for " + currentResult.getId()); 130 return true; 131 } 132 133 int comparison = compare(currentResult.getStatistics(), baseline.getStatistics()); 134 log.debug(() -> "comparison result for " + baseline.getId() + " is " + comparison); 135 136 switch (currentResult.getParams().getMode()) { 137 case Throughput: 138 if (comparison <= 0) { 139 return true; 140 } 141 return withinTolerance(currentResult.getStatistics().getMean(), baseline.getStatistics().getMean()); 142 case SampleTime: 143 case AverageTime: 144 case SingleShotTime: 145 if (comparison >= 0) { 146 return true; 147 } 148 return withinTolerance(currentResult.getStatistics().getMean(), baseline.getStatistics().getMean()); 149 default: 150 log.warn(() -> "Unsupported mode, skipping " + currentResult.getId()); 151 return true; 152 } 153 } 154 withinTolerance(double current, double baseline)155 private boolean withinTolerance(double current, double baseline) { 156 boolean positive = Math.abs(current - baseline) / baseline < TOLERANCE_LEVEL; 157 log.info(() -> "current: " + current + " baseline: " + baseline + 158 "The relative difference is within tolerance? " + positive); 159 return positive; 160 } 161 getBenchmarkId(BenchmarkParams params)162 private String getBenchmarkId(BenchmarkParams params) { 163 return params.id().replaceFirst("software.amazon.awssdk.benchmark.", ""); 164 } 165 validateBenchmarkParams(SdkBenchmarkParams current, SdkBenchmarkParams baseline)166 private boolean validateBenchmarkParams(SdkBenchmarkParams current, SdkBenchmarkParams baseline) { 167 if (!Objects.equals(current.getJdkVersion(), baseline.getJdkVersion())) { 168 log.warn(() -> "The current benchmark result was generated from a different Jdk version than the one of the " 169 + "baseline, so the results might not be comparable"); 170 return true; 171 } 172 173 return current.getMode() == baseline.getMode(); 174 } 175 serializeResult(BenchmarkProcessorOutput processorOutput)176 private String serializeResult(BenchmarkProcessorOutput processorOutput) { 177 try { 178 return OBJECT_MAPPER.writeValueAsString(processorOutput); 179 } catch (JsonProcessingException e) { 180 log.error(() -> "Failed to serialize current result", e); 181 } 182 return null; 183 } 184 } 185