1 /* 2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"). 5 * You may not use this file except in compliance with the License. 6 * A copy of the License is located at 7 * 8 * http://aws.amazon.com/apache2.0 9 * 10 * or in the "license" file accompanying this file. This file is distributed 11 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 * express or implied. See the License for the specific language governing 13 * permissions and limitations under the License. 14 */ 15 16 package software.amazon.awssdk.benchmark.utils; 17 18 import static software.amazon.awssdk.benchmark.utils.BenchmarkConstant.DEFAULT_JDK_SSL_PROVIDER; 19 import static software.amazon.awssdk.benchmark.utils.BenchmarkConstant.OPEN_SSL_PROVIDER; 20 import static software.amazon.awssdk.http.SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES; 21 22 import io.netty.handler.ssl.SslContext; 23 import io.netty.handler.ssl.SslProvider; 24 import java.io.IOException; 25 import java.net.ServerSocket; 26 import java.util.concurrent.CompletableFuture; 27 import java.util.concurrent.CountDownLatch; 28 import java.util.concurrent.TimeUnit; 29 import org.apache.commons.math3.stat.inference.TestUtils; 30 import org.openjdk.jmh.infra.Blackhole; 31 import org.openjdk.jmh.util.Statistics; 32 import software.amazon.awssdk.benchmark.stats.SdkBenchmarkStatistics; 33 import software.amazon.awssdk.utils.AttributeMap; 34 import software.amazon.awssdk.utils.Logger; 35 36 /** 37 * Contains utilities methods used by the benchmarks 38 */ 39 public final class BenchmarkUtils { 40 41 private static final Logger logger = Logger.loggerFor(BenchmarkConstant.class); 42 BenchmarkUtils()43 private BenchmarkUtils() { 44 } 45 countDownUponCompletion(Blackhole blackhole, CompletableFuture<?> completableFuture, CountDownLatch countDownLatch)46 public static void countDownUponCompletion(Blackhole blackhole, 47 CompletableFuture<?> completableFuture, 48 CountDownLatch countDownLatch) { 49 completableFuture.whenComplete((r, t) -> { 50 if (t != null) { 51 logger.error(() -> "Exception returned from the response ", t); 52 blackhole.consume(t); 53 } else { 54 blackhole.consume(r); 55 } 56 countDownLatch.countDown(); 57 }); 58 } 59 getSslProvider(String sslProviderValue)60 public static SslProvider getSslProvider(String sslProviderValue) { 61 switch (sslProviderValue) { 62 case DEFAULT_JDK_SSL_PROVIDER: 63 return SslProvider.JDK; 64 case OPEN_SSL_PROVIDER: 65 return SslProvider.OPENSSL; 66 default: 67 return SslContext.defaultClientProvider(); 68 } 69 } 70 awaitCountdownLatchUninterruptibly(CountDownLatch countDownLatch, int timeout, TimeUnit unit)71 public static void awaitCountdownLatchUninterruptibly(CountDownLatch countDownLatch, int timeout, TimeUnit unit) { 72 try { 73 if (!countDownLatch.await(timeout, unit)) { 74 throw new RuntimeException("Countdown latch did not successfully return within " + timeout + " " + unit); 75 } 76 } catch (InterruptedException e) { 77 // No need to re-interrupt. 78 logger.error(() -> "InterruptedException thrown ", e); 79 } 80 } 81 trustAllTlsAttributeMapBuilder()82 public static AttributeMap.Builder trustAllTlsAttributeMapBuilder() { 83 return AttributeMap.builder().put(TRUST_ALL_CERTIFICATES, true); 84 } 85 86 /** 87 * Returns an unused port in the localhost. 88 */ getUnusedPort()89 public static int getUnusedPort() throws IOException { 90 try (ServerSocket socket = new ServerSocket(0)) { 91 socket.setReuseAddress(true); 92 return socket.getLocalPort(); 93 } 94 } 95 96 /** 97 * Compare the results statistically. 98 * 99 * See {@link Statistics#compareTo(Statistics, double)} 100 * 101 * @param current the current result 102 * @param other the other result to compare with 103 * @param confidence the confidence level 104 * @return a negative integer, zero, or a positive integer as this statistics 105 * is less than, equal to, or greater than the specified statistics. 106 */ compare(SdkBenchmarkStatistics current, SdkBenchmarkStatistics other, double confidence)107 public static int compare(SdkBenchmarkStatistics current, SdkBenchmarkStatistics other, double confidence) { 108 if (isDifferent(current, other, confidence)) { 109 logger.info(() -> "isDifferent ? " + true); 110 double t = current.getMean(); 111 double o = other.getMean(); 112 return (t > o) ? -1 : 1; 113 } 114 115 return 0; 116 } 117 118 /** 119 * See {@link Statistics#compareTo(Statistics)} 120 */ compare(SdkBenchmarkStatistics current, SdkBenchmarkStatistics other)121 public static int compare(SdkBenchmarkStatistics current, SdkBenchmarkStatistics other) { 122 return compare(current, other, 0.99); 123 } 124 isDifferent(SdkBenchmarkStatistics current, SdkBenchmarkStatistics other, double confidence)125 private static boolean isDifferent(SdkBenchmarkStatistics current, SdkBenchmarkStatistics other, double confidence) { 126 return TestUtils.tTest(current, other, 1 - confidence); 127 } 128 } 129