1#!/usr/bin/env python3 2# SPDX-License-Identifier: Apache-2.0 3# ----------------------------------------------------------------------------- 4# Copyright 2019-2022 Arm Limited 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); you may not 7# use this file except in compliance with the License. You may obtain a copy 8# of the License at: 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15# License for the specific language governing permissions and limitations 16# under the License. 17# ----------------------------------------------------------------------------- 18""" 19The image test runner is used for image quality and performance testing. 20 21It is designed to process directories of arbitrary test images, using the 22directory structure and path naming conventions to self-describe how each image 23is to be compressed. Some built-in test sets are provided in the ./Test/Images 24directory, and others can be downloaded by running the astc_test_image_dl 25script. 26 27Attributes: 28 RESULT_THRESHOLD_WARN: The result threshold (dB) for getting a WARN. 29 RESULT_THRESHOLD_FAIL: The result threshold (dB) for getting a FAIL. 30 TEST_BLOCK_SIZES: The block sizes we can test. This is a subset of the 31 block sizes supported by ASTC, simply to keep test run times 32 manageable. 33""" 34 35import argparse 36import os 37import platform 38import sys 39 40import testlib.encoder as te 41import testlib.testset as tts 42import testlib.resultset as trs 43 44# Require bit exact with reference scores 45RESULT_THRESHOLD_WARN = -0.00 46RESULT_THRESHOLD_FAIL = -0.00 47RESULT_THRESHOLD_3D_FAIL = -0.00 48 49 50TEST_BLOCK_SIZES = ["4x4", "5x5", "6x6", "8x8", "12x12", 51 "3x3x3", "6x6x6"] 52 53TEST_QUALITIES = ["fastest", "fast", "medium", "thorough"] 54 55 56def is_3d(blockSize): 57 """ 58 Is the given block size a 3D block type? 59 60 Args: 61 blockSize (str): The block size. 62 63 Returns: 64 bool: ``True`` if the block string is a 3D block size, ``False`` if 2D. 65 """ 66 return blockSize.count("x") == 2 67 68 69def count_test_set(testSet, blockSizes): 70 """ 71 Count the number of test executions needed for a test set. 72 73 Args: 74 testSet (TestSet): The test set to run. 75 blockSizes (list(str)): The block sizes to run. 76 77 Returns: 78 int: The number of test executions needed. 79 """ 80 count = 0 81 for blkSz in blockSizes: 82 for image in testSet.tests: 83 # 3D block sizes require 3D images 84 if is_3d(blkSz) != image.is3D: 85 continue 86 87 count += 1 88 89 return count 90 91 92def determine_result(image, reference, result): 93 """ 94 Determine a test result against a reference and thresholds. 95 96 Args: 97 image (TestImage): The image being compressed. 98 reference (Record): The reference result to compare against. 99 result (Record): The test result. 100 101 Returns: 102 Result: The result code. 103 """ 104 dPSNR = result.psnr - reference.psnr 105 106 if (dPSNR < RESULT_THRESHOLD_FAIL) and (not image.is3D): 107 return trs.Result.FAIL 108 109 if (dPSNR < RESULT_THRESHOLD_3D_FAIL) and image.is3D: 110 return trs.Result.FAIL 111 112 if dPSNR < RESULT_THRESHOLD_WARN: 113 return trs.Result.WARN 114 115 return trs.Result.PASS 116 117 118def format_solo_result(image, result): 119 """ 120 Format a metrics string for a single (no compare) result. 121 122 Args: 123 image (TestImage): The image being tested. 124 result (Record): The test result. 125 126 Returns: 127 str: The metrics string. 128 """ 129 name = "%5s %s" % (result.blkSz, result.name) 130 tPSNR = "%2.3f dB" % result.psnr 131 tTTime = "%.3f s" % result.tTime 132 tCTime = "%.3f s" % result.cTime 133 tCMTS = "%.3f MT/s" % result.cRate 134 135 return "%-32s | %8s | %9s | %9s | %11s" % \ 136 (name, tPSNR, tTTime, tCTime, tCMTS) 137 138 139def format_result(image, reference, result): 140 """ 141 Format a metrics string for a comparison result. 142 143 Args: 144 image (TestImage): The image being tested. 145 reference (Record): The reference result to compare against. 146 result (Record): The test result. 147 148 Returns: 149 str: The metrics string. 150 """ 151 dPSNR = result.psnr - reference.psnr 152 sTTime = reference.tTime / result.tTime 153 sCTime = reference.cTime / result.cTime 154 155 name = "%5s %s" % (result.blkSz, result.name) 156 tPSNR = "%2.3f dB (% 1.3f dB)" % (result.psnr, dPSNR) 157 tTTime = "%.3f s (%1.2fx)" % (result.tTime, sTTime) 158 tCTime = "%.3f s (%1.2fx)" % (result.cTime, sCTime) 159 tCMTS = "%.3f MT/s" % (result.cRate) 160 result = determine_result(image, reference, result) 161 162 return "%-32s | %22s | %15s | %15s | %11s | %s" % \ 163 (name, tPSNR, tTTime, tCTime, tCMTS, result.name) 164 165 166def run_test_set(encoder, testRef, testSet, quality, blockSizes, testRuns, 167 keepOutput, threads): 168 """ 169 Execute all tests in the test set. 170 171 Args: 172 encoder (EncoderBase): The encoder to use. 173 testRef (ResultSet): The test reference results. 174 testSet (TestSet): The test set. 175 quality (str): The quality level to execute the test against. 176 blockSizes (list(str)): The block sizes to execute each test against. 177 testRuns (int): The number of test repeats to run for each image test. 178 keepOutput (bool): Should the test preserve output images? This is 179 only a hint and discarding output may be ignored if the encoder 180 version used can't do it natively. 181 threads (int or None): The thread count to use. 182 183 Returns: 184 ResultSet: The test results. 185 """ 186 resultSet = trs.ResultSet(testSet.name) 187 188 curCount = 0 189 maxCount = count_test_set(testSet, blockSizes) 190 191 dat = (testSet.name, encoder.name, quality) 192 title = "Test Set: %s / Encoder: %s -%s" % dat 193 print(title) 194 print("=" * len(title)) 195 196 for blkSz in blockSizes: 197 for image in testSet.tests: 198 # 3D block sizes require 3D images 199 if is_3d(blkSz) != image.is3D: 200 continue 201 202 curCount += 1 203 204 dat = (curCount, maxCount, blkSz, image.testFile) 205 print("Running %u/%u %s %s ... " % dat, end='', flush=True) 206 res = encoder.run_test(image, blkSz, "-%s" % quality, testRuns, 207 keepOutput, threads) 208 res = trs.Record(blkSz, image.testFile, res[0], res[1], res[2], res[3]) 209 resultSet.add_record(res) 210 211 if testRef: 212 refResult = testRef.get_matching_record(res) 213 res.set_status(determine_result(image, refResult, res)) 214 215 res.tTimeRel = refResult.tTime / res.tTime 216 res.cTimeRel = refResult.cTime / res.cTime 217 res.psnrRel = res.psnr - refResult.psnr 218 219 res = format_result(image, refResult, res) 220 else: 221 res = format_solo_result(image, res) 222 223 print("\r[%3u] %s" % (curCount, res)) 224 225 return resultSet 226 227 228def get_encoder_params(encoderName, referenceName, imageSet): 229 """ 230 The the encoder and image set parameters for a test run. 231 232 Args: 233 encoderName (str): The encoder name. 234 referenceName (str): The reference encoder name. 235 imageSet (str): The test image set. 236 237 Returns: 238 tuple(EncoderBase, str, str, str): The test parameters for the 239 requested encoder and test set. An instance of the encoder wrapper 240 class, the output data name, the output result directory, and the 241 reference to use. 242 """ 243 # 1.7 variants 244 if encoderName == "ref-1.7": 245 encoder = te.Encoder1_7() 246 name = "reference-1.7" 247 outDir = "Test/Images/%s" % imageSet 248 refName = None 249 return (encoder, name, outDir, refName) 250 251 if encoderName.startswith("ref"): 252 _, version, simd = encoderName.split("-") 253 254 # 2.x variants 255 if version.startswith("2."): 256 encoder = te.Encoder2xRel(version, simd) 257 name = f"reference-{version}-{simd}" 258 outDir = "Test/Images/%s" % imageSet 259 refName = None 260 return (encoder, name, outDir, refName) 261 262 # 3.x variants 263 if version.startswith("3."): 264 encoder = te.Encoder2xRel(version, simd) 265 name = f"reference-{version}-{simd}" 266 outDir = "Test/Images/%s" % imageSet 267 refName = None 268 return (encoder, name, outDir, refName) 269 270 # Latest main 271 if version == "main": 272 encoder = te.Encoder2x(simd) 273 name = f"reference-{version}-{simd}" 274 outDir = "Test/Images/%s" % imageSet 275 refName = None 276 return (encoder, name, outDir, refName) 277 278 assert False, f"Encoder {encoderName} not recognized" 279 280 encoder = te.Encoder2x(encoderName) 281 name = "develop-%s" % encoderName 282 outDir = "TestOutput/%s" % imageSet 283 refName = referenceName.replace("ref", "reference") 284 return (encoder, name, outDir, refName) 285 286 287def parse_command_line(): 288 """ 289 Parse the command line. 290 291 Returns: 292 Namespace: The parsed command line container. 293 """ 294 parser = argparse.ArgumentParser() 295 296 # All reference encoders 297 refcoders = ["ref-1.7", 298 "ref-2.5-neon", "ref-2.5-sse2", "ref-2.5-sse4.1", "ref-2.5-avx2", 299 "ref-3.6-neon", "ref-3.6-sse2", "ref-3.6-sse4.1", "ref-3.6-avx2", 300 "ref-3.7-neon", "ref-3.7-sse2", "ref-3.7-sse4.1", "ref-3.7-avx2", 301 "ref-main-neon", "ref-main-sse2", "ref-main-sse4.1", "ref-main-avx2"] 302 303 # All test encoders 304 testcoders = ["none", "neon", "sse2", "sse4.1", "avx2", "native"] 305 testcodersAArch64 = ["none", "neon", "native"] 306 testcodersX86 = ["none", "sse2", "sse4.1", "avx2", "native"] 307 308 coders = refcoders + testcoders + ["all-aarch64", "all-x86"] 309 310 parser.add_argument("--encoder", dest="encoders", default="avx2", 311 choices=coders, help="test encoder variant") 312 313 parser.add_argument("--reference", dest="reference", default="ref-3.7-avx2", 314 choices=refcoders, help="reference encoder variant") 315 316 astcProfile = ["ldr", "ldrs", "hdr", "all"] 317 parser.add_argument("--color-profile", dest="profiles", default="all", 318 choices=astcProfile, help="test color profile") 319 320 imgFormat = ["l", "xy", "rgb", "rgba", "all"] 321 parser.add_argument("--color-format", dest="formats", default="all", 322 choices=imgFormat, help="test color format") 323 324 choices = list(TEST_BLOCK_SIZES) + ["all"] 325 parser.add_argument("--block-size", dest="blockSizes", 326 action="append", choices=choices, 327 help="test block size") 328 329 testDir = os.path.dirname(__file__) 330 testDir = os.path.join(testDir, "Images") 331 testSets = [] 332 for path in os.listdir(testDir): 333 fqPath = os.path.join(testDir, path) 334 if os.path.isdir(fqPath): 335 testSets.append(path) 336 testSets.append("all") 337 338 parser.add_argument("--test-set", dest="testSets", default="Small", 339 choices=testSets, help="test image test set") 340 341 parser.add_argument("--test-image", dest="testImage", default=None, 342 help="select a specific test image from the test set") 343 344 choices = list(TEST_QUALITIES) + ["all"] 345 parser.add_argument("--test-quality", dest="testQual", default="thorough", 346 choices=choices, help="select a specific test quality") 347 348 parser.add_argument("--repeats", dest="testRepeats", default=1, 349 type=int, help="test iteration count") 350 351 parser.add_argument("--keep-output", dest="keepOutput", default=False, 352 action="store_true", help="keep image output") 353 354 parser.add_argument("-j", dest="threads", default=None, 355 type=int, help="thread count") 356 357 358 args = parser.parse_args() 359 360 # Turn things into canonical format lists 361 if args.encoders == "all-aarch64": 362 args.encoders = testcodersAArch64 363 elif args.encoders == "all-x86": 364 args.encoders = testcodersX86 365 else: 366 args.encoders = [args.encoders] 367 368 args.testQual = TEST_QUALITIES if args.testQual == "all" \ 369 else [args.testQual] 370 371 if not args.blockSizes or ("all" in args.blockSizes): 372 args.blockSizes = TEST_BLOCK_SIZES 373 374 args.testSets = testSets[:-1] if args.testSets == "all" \ 375 else [args.testSets] 376 377 args.profiles = astcProfile[:-1] if args.profiles == "all" \ 378 else [args.profiles] 379 380 args.formats = imgFormat[:-1] if args.formats == "all" \ 381 else [args.formats] 382 383 return args 384 385 386def main(): 387 """ 388 The main function. 389 390 Returns: 391 int: The process return code. 392 """ 393 # Parse command lines 394 args = parse_command_line() 395 396 testSetCount = 0 397 worstResult = trs.Result.NOTRUN 398 399 for quality in args.testQual: 400 for imageSet in args.testSets: 401 for encoderName in args.encoders: 402 (encoder, name, outDir, refName) = \ 403 get_encoder_params(encoderName, args.reference, imageSet) 404 405 testDir = "Test/Images/%s" % imageSet 406 testRes = "%s/astc_%s_%s_results.csv" % (outDir, name, quality) 407 408 testRef = None 409 if refName: 410 dat = (testDir, refName, quality) 411 testRefPath = "%s/astc_%s_%s_results.csv" % dat 412 testRef = trs.ResultSet(imageSet) 413 testRef.load_from_file(testRefPath) 414 415 testSetCount += 1 416 testSet = tts.TestSet(imageSet, testDir, 417 args.profiles, args.formats, args.testImage) 418 419 # The fast and fastest presets are now sufficiently fast that 420 # the results are noisy without more repeats 421 testRepeats = args.testRepeats 422 if quality == "fast" and testRepeats > 1: 423 testRepeats *= 2 424 elif quality == "fastest" and testRepeats > 1: 425 testRepeats *= 4 426 427 resultSet = run_test_set(encoder, testRef, testSet, quality, 428 args.blockSizes, testRepeats, 429 args.keepOutput, args.threads) 430 431 resultSet.save_to_file(testRes) 432 433 if refName: 434 summary = resultSet.get_results_summary() 435 worstResult = max(summary.get_worst_result(), worstResult) 436 print(summary) 437 438 if (testSetCount > 1) and (worstResult != trs.Result.NOTRUN): 439 print("OVERALL STATUS: %s" % worstResult.name) 440 441 if worstResult == trs.Result.FAIL: 442 return 1 443 444 return 0 445 446 447if __name__ == "__main__": 448 sys.exit(main()) 449