1 /* 2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"). 5 * You may not use this file except in compliance with the License. 6 * A copy of the License is located at 7 * 8 * http://aws.amazon.com/apache2.0 9 * 10 * or in the "license" file accompanying this file. This file is distributed 11 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 * express or implied. See the License for the specific language governing 13 * permissions and limitations under the License. 14 */ 15 16 package software.amazon.awssdk.s3benchmarks; 17 18 import java.time.Duration; 19 import java.util.EnumMap; 20 import java.util.Locale; 21 import java.util.Map; 22 import java.util.function.Function; 23 import org.apache.commons.cli.CommandLine; 24 import org.apache.commons.cli.CommandLineParser; 25 import org.apache.commons.cli.DefaultParser; 26 import org.apache.commons.cli.Options; 27 import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; 28 29 public final class BenchmarkRunner { 30 31 private static final String PART_SIZE_IN_MB = "partSizeInMB"; 32 private static final String FILE = "file"; 33 private static final String BUCKET = "bucket"; 34 private static final String MAX_THROUGHPUT = "maxThroughput"; 35 private static final String KEY = "key"; 36 private static final String OPERATION = "operation"; 37 private static final String CHECKSUM_ALGORITHM = "checksumAlgo"; 38 private static final String ITERATION = "iteration"; 39 private static final String CONTENT_LENGTH = "contentLengthInMB"; 40 41 private static final String READ_BUFFER_IN_MB = "readBufferInMB"; 42 43 private static final String VERSION = "version"; 44 private static final String PREFIX = "prefix"; 45 46 private static final String TIMEOUT = "timeoutInMin"; 47 48 private static final String CONN_ACQ_TIMEOUT_IN_SEC = "connAcqTimeoutInSec"; 49 50 private static final String FORCE_CRT_HTTP_CLIENT = "crtHttp"; 51 private static final String MAX_CONCURRENCY = "maxConcurrency"; 52 53 private static final Map<TransferManagerOperation, Function<TransferManagerBenchmarkConfig, TransferManagerBenchmark>> 54 OPERATION_TO_BENCHMARK_V1 = new EnumMap<>(TransferManagerOperation.class); 55 private static final Map<TransferManagerOperation, Function<TransferManagerBenchmarkConfig, TransferManagerBenchmark>> 56 OPERATION_TO_BENCHMARK_V2 = new EnumMap<>(TransferManagerOperation.class); 57 58 static { OPERATION_TO_BENCHMARK_V2.put(TransferManagerOperation.COPY, TransferManagerBenchmark::copy)59 OPERATION_TO_BENCHMARK_V2.put(TransferManagerOperation.COPY, TransferManagerBenchmark::copy); OPERATION_TO_BENCHMARK_V2.put(TransferManagerOperation.DOWNLOAD, TransferManagerBenchmark::v2Download)60 OPERATION_TO_BENCHMARK_V2.put(TransferManagerOperation.DOWNLOAD, TransferManagerBenchmark::v2Download); OPERATION_TO_BENCHMARK_V2.put(TransferManagerOperation.UPLOAD, TransferManagerBenchmark::v2Upload)61 OPERATION_TO_BENCHMARK_V2.put(TransferManagerOperation.UPLOAD, TransferManagerBenchmark::v2Upload); OPERATION_TO_BENCHMARK_V2.put(TransferManagerOperation.DOWNLOAD_DIRECTORY, TransferManagerBenchmark::downloadDirectory)62 OPERATION_TO_BENCHMARK_V2.put(TransferManagerOperation.DOWNLOAD_DIRECTORY, TransferManagerBenchmark::downloadDirectory); OPERATION_TO_BENCHMARK_V2.put(TransferManagerOperation.UPLOAD_DIRECTORY, TransferManagerBenchmark::uploadDirectory)63 OPERATION_TO_BENCHMARK_V2.put(TransferManagerOperation.UPLOAD_DIRECTORY, TransferManagerBenchmark::uploadDirectory); 64 OPERATION_TO_BENCHMARK_V1.put(TransferManagerOperation.COPY, TransferManagerBenchmark::v1Copy)65 OPERATION_TO_BENCHMARK_V1.put(TransferManagerOperation.COPY, TransferManagerBenchmark::v1Copy); OPERATION_TO_BENCHMARK_V1.put(TransferManagerOperation.DOWNLOAD, TransferManagerBenchmark::v1Download)66 OPERATION_TO_BENCHMARK_V1.put(TransferManagerOperation.DOWNLOAD, TransferManagerBenchmark::v1Download); OPERATION_TO_BENCHMARK_V1.put(TransferManagerOperation.UPLOAD, TransferManagerBenchmark::v1Upload)67 OPERATION_TO_BENCHMARK_V1.put(TransferManagerOperation.UPLOAD, TransferManagerBenchmark::v1Upload); OPERATION_TO_BENCHMARK_V1.put(TransferManagerOperation.DOWNLOAD_DIRECTORY, TransferManagerBenchmark::v1DownloadDirectory)68 OPERATION_TO_BENCHMARK_V1.put(TransferManagerOperation.DOWNLOAD_DIRECTORY, TransferManagerBenchmark::v1DownloadDirectory); OPERATION_TO_BENCHMARK_V1.put(TransferManagerOperation.UPLOAD_DIRECTORY, TransferManagerBenchmark::v1UploadDirectory)69 OPERATION_TO_BENCHMARK_V1.put(TransferManagerOperation.UPLOAD_DIRECTORY, TransferManagerBenchmark::v1UploadDirectory); 70 } 71 BenchmarkRunner()72 private BenchmarkRunner() { 73 } 74 main(String... args)75 public static void main(String... args) throws org.apache.commons.cli.ParseException { 76 CommandLineParser parser = new DefaultParser(); 77 78 Options options = new Options(); 79 80 options.addRequiredOption(null, BUCKET, true, "The s3 bucket"); 81 options.addOption(null, KEY, true, "The s3 key"); 82 options.addRequiredOption(null, OPERATION, true, "The operation to run tests: download | upload | download_directory | " 83 + "upload_directory | copy"); 84 options.addOption(null, FILE, true, "Destination file path to be written to or source file path to be " 85 + "uploaded"); 86 options.addOption(null, PART_SIZE_IN_MB, true, "Part size in MB"); 87 options.addOption(null, MAX_THROUGHPUT, true, "The max throughput"); 88 options.addOption(null, CHECKSUM_ALGORITHM, true, "The checksum algorithm to use"); 89 options.addOption(null, ITERATION, true, "The number of iterations"); 90 options.addOption(null, READ_BUFFER_IN_MB, true, "Read buffer size in MB"); 91 options.addOption(null, VERSION, true, "The major version of the transfer manager to run test: " 92 + "v1 | v2 | crt | java, default: v2"); 93 options.addOption(null, PREFIX, true, "S3 Prefix used in downloadDirectory and uploadDirectory"); 94 95 options.addOption(null, CONTENT_LENGTH, true, "Content length to upload from memory. Used only in the " 96 + "CRT Upload Benchmark, but " 97 + "is required for this test case."); 98 99 options.addOption(null, TIMEOUT, true, "Amount of minute to wait before a single operation " 100 + "times out and is cancelled. Optional, defaults to 10 minutes if no specified"); 101 options.addOption(null, CONN_ACQ_TIMEOUT_IN_SEC, true, "Timeout for acquiring an already-established" 102 + " connection from a connection pool to a remote service."); 103 options.addOption(null, FORCE_CRT_HTTP_CLIENT, true, 104 "Force the CRT http client to be used in JavaBased benchmarks"); 105 options.addOption(null, MAX_CONCURRENCY, true, 106 "The Maximum number of allowed concurrent requests. For HTTP/1.1 this is the same as max connections."); 107 108 CommandLine cmd = parser.parse(options, args); 109 TransferManagerBenchmarkConfig config = parseConfig(cmd); 110 111 SdkVersion version = SdkVersion.valueOf(cmd.getOptionValue(VERSION, "V2") 112 .toUpperCase(Locale.ENGLISH)); 113 114 TransferManagerOperation operation = config.operation(); 115 TransferManagerBenchmark benchmark; 116 117 switch (version) { 118 case V1: 119 benchmark = OPERATION_TO_BENCHMARK_V1.get(operation).apply(config); 120 break; 121 case V2: 122 benchmark = OPERATION_TO_BENCHMARK_V2.get(operation).apply(config); 123 break; 124 case CRT: 125 if (operation == TransferManagerOperation.DOWNLOAD) { 126 benchmark = new CrtS3ClientDownloadBenchmark(config); 127 break; 128 } 129 if (operation == TransferManagerOperation.UPLOAD) { 130 benchmark = new CrtS3ClientUploadBenchmark(config); 131 break; 132 } 133 throw new UnsupportedOperationException(); 134 case JAVA: 135 if (operation == TransferManagerOperation.UPLOAD) { 136 benchmark = new JavaS3ClientUploadBenchmark(config); 137 break; 138 } 139 if (operation == TransferManagerOperation.COPY) { 140 benchmark = new JavaS3ClientCopyBenchmark(config); 141 break; 142 } 143 throw new UnsupportedOperationException("Java based s3 client benchmark only support upload and copy"); 144 default: 145 throw new UnsupportedOperationException(); 146 } 147 benchmark.run(); 148 } 149 parseConfig(CommandLine cmd)150 private static TransferManagerBenchmarkConfig parseConfig(CommandLine cmd) { 151 TransferManagerOperation operation = TransferManagerOperation.valueOf(cmd.getOptionValue(OPERATION) 152 .toUpperCase(Locale.ENGLISH)); 153 154 String filePath = cmd.getOptionValue(FILE); 155 String bucket = cmd.getOptionValue(BUCKET); 156 String key = cmd.getOptionValue(KEY); 157 158 Long partSize = cmd.getOptionValue(PART_SIZE_IN_MB) == null ? null : Long.parseLong(cmd.getOptionValue(PART_SIZE_IN_MB)); 159 160 Double maxThroughput = cmd.getOptionValue(MAX_THROUGHPUT) == null ? null : 161 Double.parseDouble(cmd.getOptionValue(MAX_THROUGHPUT)); 162 163 ChecksumAlgorithm checksumAlgorithm = null; 164 if (cmd.getOptionValue(CHECKSUM_ALGORITHM) != null) { 165 checksumAlgorithm = ChecksumAlgorithm.fromValue(cmd.getOptionValue(CHECKSUM_ALGORITHM) 166 .toUpperCase(Locale.ENGLISH)); 167 } 168 169 Integer iteration = cmd.getOptionValue(ITERATION) == null ? null : 170 Integer.parseInt(cmd.getOptionValue(ITERATION)); 171 172 Long readBufferInMB = cmd.getOptionValue(READ_BUFFER_IN_MB) == null ? null : 173 Long.parseLong(cmd.getOptionValue(READ_BUFFER_IN_MB)); 174 175 String prefix = cmd.getOptionValue(PREFIX); 176 177 Long contentLengthInMb = cmd.getOptionValue(CONTENT_LENGTH) == null ? null : 178 Long.parseLong(cmd.getOptionValue(CONTENT_LENGTH)); 179 180 Duration timeout = cmd.getOptionValue(TIMEOUT) == null ? null : 181 Duration.ofMinutes(Long.parseLong(cmd.getOptionValue(TIMEOUT))); 182 183 Long connAcqTimeoutInSec = cmd.getOptionValue(CONN_ACQ_TIMEOUT_IN_SEC) == null ? null : 184 Long.parseLong(cmd.getOptionValue(CONN_ACQ_TIMEOUT_IN_SEC)); 185 186 Boolean forceCrtHttpClient = cmd.getOptionValue(FORCE_CRT_HTTP_CLIENT) != null 187 && Boolean.parseBoolean(cmd.getOptionValue(FORCE_CRT_HTTP_CLIENT)); 188 189 Integer maxConcurrency = cmd.getOptionValue(MAX_CONCURRENCY) == null ? null : 190 Integer.parseInt(cmd.getOptionValue(MAX_CONCURRENCY)); 191 192 return TransferManagerBenchmarkConfig.builder() 193 .key(key) 194 .bucket(bucket) 195 .partSizeInMb(partSize) 196 .checksumAlgorithm(checksumAlgorithm) 197 .targetThroughput(maxThroughput) 198 .readBufferSizeInMb(readBufferInMB) 199 .filePath(filePath) 200 .iteration(iteration) 201 .operation(operation) 202 .prefix(prefix) 203 .contentLengthInMb(contentLengthInMb) 204 .timeout(timeout) 205 .connectionAcquisitionTimeoutInSec(connAcqTimeoutInSec) 206 .forceCrtHttpClient(forceCrtHttpClient) 207 .maxConcurrency(maxConcurrency) 208 .build(); 209 } 210 211 public enum TransferManagerOperation { 212 DOWNLOAD, 213 UPLOAD, 214 COPY, 215 DOWNLOAD_DIRECTORY, 216 UPLOAD_DIRECTORY 217 } 218 219 private enum SdkVersion { 220 V1, 221 V2, 222 CRT, 223 JAVA 224 } 225 } 226