1#!/usr/bin/env perl 2# * © 2016 and later: Unicode, Inc. and others. 3# * License & terms of use: http://www.unicode.org/copyright.html 4# ******************************************************************************* 5# * Copyright (C) 2002-2012 International Business Machines Corporation and * 6# * others. All Rights Reserved. * 7# ******************************************************************************* 8 9use XML::LibXML; 10 11# Assume we are running within the icu4j root directory 12use lib '.'; 13use Dataset; 14 15my $OS=$^O; 16my $CLASSPATH; 17if ($^O eq "MSWin32") { 18 $CLASSPATH = './target/*;./target/dependency/*'; 19} else { 20 $CLASSPATH = './target/*:./target/dependency/*'; 21} 22 23#--------------------------------------------------------------------- 24 25# Methods to be tested. Each pair represents a test method and 26# a baseline method which is used for comparison. 27my @METHODS = ( 28 ['TestJDKConstruction', 'TestICUConstruction'], 29 ['TestJDKParse', 'TestICUParse'], 30 ['TestJDKFormat', 'TestICUFormat'] 31 ); 32# Patterns which define the set of characters used for testing. 33my @OPTIONS = ( 34# locale pattern date string 35 [ "en_US", "dddd MMM yyyy", "15 Jan 2007"], 36 [ "sw_KE", "dddd MMM yyyy", "15 Jan 2007"], 37 [ "en_US", "HH:mm", "13:13"], 38 [ "en_US", "HH:mm zzzz", "13:13 Pacific Standard Time"], 39 [ "en_US", "HH:mm z", "13:13 PST"], 40 [ "en_US", "HH:mm Z", "13:13 -0800"], 41 ); 42 43my $THREADS; # number of threads (input from command-line args) 44my $CALIBRATE = 2; # duration in seconds for initial calibration 45my $DURATION = 10; # duration in seconds for each pass 46my $NUMPASSES = 4; # number of passes. If > 1 then the first pass 47 # is discarded as a JIT warm-up pass. 48 49my $TABLEATTR = 'BORDER="1" CELLPADDING="4" CELLSPACING="0"'; 50 51my $PLUS_MINUS = "±"; 52 53if ($NUMPASSES < 3) { 54 die "Need at least 3 passes. One is discarded (JIT warmup) and need two to have 1 degree of freedom (t distribution)."; 55} 56 57 58# run all tests with the specified number of threads from command-line input 59# (if there is no arguments, use $THREADS = 1) 60foreach my $arg ($#ARGV >= 0 ? @ARGV : "1") { 61 $THREADS = $arg; 62 main(); 63} 64 65 66#--------------------------------------------------------------------- 67sub main { 68 69#-----------DATE FORMAT PERFORMANCE TESTS----------------------------- 70 my $testclass = 'com.ibm.icu.dev.test.perf.DateFormatPerformanceTest'; 71 #my $threads = ($THREADS > 1) ? "($THREADS threads)" : ""; 72 73 my $doc = XML::LibXML::Document->new('1.0', 'utf-8'); 74 my $root = $doc->createElement("perfTestResults"); 75 76 # my $raw = ""; 77 my @shortNames = ( "open" , "parse", "fmt"); 78 my $index=0; 79 80 for my $methodPair (@METHODS) { 81 82 my $testMethod = $methodPair->[0]; 83 my $baselineMethod = $methodPair->[1]; 84 my $testname = $shortNames[$index]; 85 $index++; 86 87 $OUT = ''; 88 my $patternCounter=1; 89 90 for my $pat (@OPTIONS) { 91 92 # measure the test method 93 my $t = measure2($testclass, $testMethod, $pat, -$DURATION); 94 my $testResult = $t->getMean(); 95 my $jdkElement = $doc->createElement("perfTestResult"); 96 my $testName = "DateFmt-$testname-pat$patternCounter-JDK"; 97 $jdkElement->setAttribute("test" => $testName); 98 $jdkElement->setAttribute("iterations" => 1); 99 $jdkElement->setAttribute("time" => $testResult); 100 $root->appendChild($jdkElement); 101 102 # measure baseline method 103 my $b = measure2($testclass, $baselineMethod, $pat, -$DURATION); 104 my $baseResult = $b->getMean(); 105 my $icuElement = $doc->createElement("perfTestResult"); 106 my $testName = "DateFmt-$testname-pat$patternCounter"; 107 $patternCounter++; 108 $icuElement->setAttribute("test"=> $testName); 109 $icuElement->setAttribute("iterations" => 1); 110 $icuElement->setAttribute("time" => $baseResult); 111 $root->appendChild($icuElement); 112 113 } 114 } 115 116#------------------DECIMAL FORMAT TESTS--------------------------------- 117 118 my $testclass = 'com.ibm.icu.dev.test.perf.DecimalFormatPerformanceTest'; 119 my @OPTIONS = ( 120# locale pattern date string 121 [ "en_US", "#,###.##", "1,234.56"], 122 [ "de_DE", "#,###.##", "1.234,56"], 123 ); 124 my $index=0; 125 for my $methodPair (@METHODS) { 126 127 my $testMethod = $methodPair->[0]; 128 my $baselineMethod = $methodPair->[1]; 129 my $testname = $shortNames[$index]; 130 $index++; 131 132 133 for my $pat (@OPTIONS) { 134 my $patternName = $pat->[0]; 135 136 # measure the test method 137 my $t = measure2($testclass, $testMethod, $pat, -$DURATION); 138 my $testResult = $t->getMean(); 139 my $jdkElement = $doc->createElement("perfTestResult"); 140 my $testName = "NumFmt-$testname-$patternName-JDK"; 141 $jdkElement->setAttribute("test" => $testName); 142 $jdkElement->setAttribute("iterations"=>1); 143 $jdkElement->setAttribute("time" => $testResult); 144 $root->appendChild($jdkElement); 145 146 # measure baseline method 147 my $b = measure2($testclass, $baselineMethod, $pat, -$DURATION); 148 my $baseResult = $b->getMean(); 149 my $icuElement = $doc->createElement("perfTestResult"); 150 my $testName = "NumFmt-$testname-$patternName"; 151 $icuElement->setAttribute("test"=> $testName); 152 $icuElement->setAttribute("iterations"=>1); 153 $icuElement->setAttribute("time" => $baseResult); 154 $root->appendChild($icuElement); 155 } 156 } 157 158#----------------COLLATION PERFORMANCE TESTS--------------------------_ 159 160 %dataFiles = ( 161 "en_US", "TestNames_Latin.txt", 162 "da_DK", "TestNames_Latin.txt", 163 "de_DE", "TestNames_Latin.txt", 164 "de__PHONEBOOK", "TestNames_Latin.txt", 165 "fr_FR", "TestNames_Latin.txt", 166 "ja_JP", "TestNames_Latin.txt TestNames_Japanese_h.txt TestNames_Japanese_k.txt TestNames_Asian.txt", 167 "zh_CN", "TestNames_Latin.txt TestNames_Chinese.txt", 168 "zh_TW", "TestNames_Latin.txt TestNames_Chinese.txt", 169 "zh__PINYIN", "TestNames_Latin.txt TestNames_Chinese.txt", 170 "ru_RU", "TestNames_Latin.txt TestNames_Russian.txt", 171 "th", "TestNames_Latin.txt TestNames_Thai.txt", 172 "ko_KR", "TestNames_Latin.txt TestNames_Korean.txt", 173 ); 174 175 # Outer loop runs through the locales to test 176 # (Edit this list directly to make changes) 177 # 178 foreach $locale ( 179 "en_US", 180 "da_DK", 181 "de_DE", 182 "de__PHONEBOOK", 183 "fr_FR", 184 "ja_JP", 185 "zh_CN", 186 "zh_TW", 187 "zh__PINYIN", 188 "ko_KR", 189 "ru_RU", 190 "th", 191 ) 192 { 193 194 195 # 196 # Inner loop runs over the set of data files specified for each locale. 197 # (Edit the %datafiles initialization, above, to make changes. 198 # 199 $ff = $dataFiles{$locale}; 200 @ff = split(/[\s]+/, $ff); 201 $counter = 1; 202 foreach $data (@ff) { 203 # 204 # Run ICU Test for this (locale, data file) pair. 205 # 206 $iStrCol = `java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch`; 207print "java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch\n"; 208 $iStrCol =~s/[,\s]*//g; # whack off the leading " ," in the returned result. 209 doKeyTimes("java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -keygen", 210 my $iKeyGen, my $iKeyLen); 211 212 # 213 # Run Windows test for this (locale, data file) pair. Only do if 214 # we are not on Windows 98/ME and we hava a windows langID 215 # for the locale. 216 # 217 $wStrCol = $wKeyGen = $wKeyLen = 0; 218 my $wStrCol = `java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch -java`; 219 $wStrCol =~s/[,\s]*//g; # whack off the leading " ," in the returned result. 220 doKeyTimes("java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -keygen -java", 221 $wKeyGen, $wKeyLen); 222 223 $collDiff = $keyGenDiff = $keyLenDiff = 0; 224 if ($wKeyLen > 0) { 225 $collDiff = (($wStrCol - $iStrCol) / $iStrCol) * 100; 226 $keyGenDiff = (($wKeyGen - $iKeyGen) / $iKeyGen) * 100; 227 $keyLenDiff = (($wKeyLen - $iKeyLen) / $iKeyLen) * 100; 228 } 229 230 my $ICU = $doc->createElement("perfTestResult"); 231 my $testname = "Coll-$locale-data$counter-StrCol"; 232 #write the results corresponding to this local,data pair 233 $ICU->setAttribute("test"=> $testname); 234 $ICU->setAttribute("iterations"=>1000); 235 $ICU->setAttribute("time"=> $iStrCol); 236 $root->appendChild($ICU); 237 238 my $Key = $doc->createElement("perfTestResult"); 239 my $testname = "Coll-$locale-data$counter-keyGen"; 240 $Key->setAttribute("test"=> $testname); 241 $Key->setAttribute("iterations"=>1000); 242 $Key->setAttribute("time"=>$iKeyGen); 243 $root->appendChild($Key); 244 245 my $JDK = $doc->createElement("perfTestResult"); 246 my $testname = "Coll-$locale-data$counter-StrCol-JDK"; 247 $JDK->setAttribute("test"=>$testname); 248 $JDK->setAttribute("iterations"=>1000); 249 $JDK->setAttribute("time"=>$wStrCol); 250 $root->appendChild($JDK); 251 252 my $Key = $doc->createElement("perfTestResult"); 253 my $testname = "Coll-$locale-data$counter-keyGen-JDK"; 254 $Key->setAttribute("test"=>$testname); 255 $Key->setAttribute("iterations"=>1000); 256 $Key->setAttribute("time"=>$wKeyGen); 257 $root->appendChild($Key); 258 $counter++; 259 } 260 } 261 262 263 264#----------WRITE RESULTS TO perf.xml----------------------- 265 $doc->setDocumentElement($root); 266 open my $out_fh, '>', "perf.xml"; 267 print {$out_fh} $doc->toString; 268} 269 270 271#--------------------------------------------------------------------- 272# Append text to the global variable $OUT 273sub out { 274 $OUT .= join('', @_); 275} 276 277 278#--------------------------------------------------------------------- 279# Measure a given test method with a give test pattern using the 280# global run parameters. 281# 282# @param the method to run 283# @param the pattern defining characters to test 284# @param if >0 then the number of iterations per pass. If <0 then 285# (negative of) the number of seconds per pass. 286# 287# @return a Dataset object, scaled by iterations per pass and 288# events per iteration, to give time per event 289# 290sub measure2 { 291 my @data = measure1(@_); 292 my $iterPerPass = shift(@data); 293 my $eventPerIter = shift(@data); 294 295 shift(@data) if (@data > 1); # discard first run 296 297 my $ds = Dataset->new(@data); 298 $ds->setScale(1.0e-3 / ($iterPerPass * $eventPerIter)); 299 $ds; 300} 301 302#--------------------------------------------------------------------- 303# Measure a given test method with a give test pattern using the 304# global run parameters. 305# 306# @param the method to run 307# @param the pattern defining characters to test 308# @param if >0 then the number of iterations per pass. If <0 then 309# (negative of) the number of seconds per pass. 310# 311# @return array of: 312# [0] iterations per pass 313# [1] events per iteration 314# [2..] ms reported for each pass, in order 315# 316sub measure1 { 317 my $testclass = shift; 318 my $method = shift; 319 my $pat = shift; 320 my $iterCount = shift; # actually might be -seconds/pass 321 322 # is $iterCount actually -seconds/pass? 323 if ($iterCount < 0) { 324 325 # calibrate: estimate ms/iteration 326 print "Calibrating..."; 327 my @t = callJava($testclass, $method, $pat, -$CALIBRATE, 1); 328 print "done.\n"; 329 330 my @data = split(/\s+/, $t[0]->[2]); 331 $data[0] *= 1.0e+3; 332 333 my $timePerIter = 1.0e-3 * $data[0] / $data[1]; 334 335 # determine iterations/pass 336 $iterCount = int(-$iterCount / $timePerIter + 0.5); 337 } 338 339 # run passes 340 print "Measuring $iterCount iterations x $NUMPASSES passes..."; 341 my @t = callJava($testclass, $method, $pat, $iterCount, $NUMPASSES); 342 print "done.\n"; 343 my @ms = (); 344 my @b; # scratch 345 for my $a (@t) { 346 # $a->[0]: method name, corresponds to $method 347 # $a->[1]: 'begin' data, == $iterCount 348 # $a->[2]: 'end' data, of the form <ms> <loops> <eventsPerIter> 349 # $a->[3...]: gc messages from JVM during pass 350 @b = split(/\s+/, $a->[2]); 351 push(@ms, $b[0] * 1.0e+3); 352 } 353 my $eventsPerIter = $b[2]; 354 355 my @ms_str = @ms; 356 $ms_str[0] .= " (discarded)" if (@ms_str > 1); 357 358 ($iterCount, $eventsPerIter, @ms); 359} 360 361#--------------------------------------------------------------------- 362# Invoke java to run $TESTCLASS, passing it the given parameters. 363# 364# @param the method to run 365# @param the number of iterations, or if negative, the duration 366# in seconds. If more than on pass is desired, pass in 367# a string, e.g., "100 100 100". 368# @param the pattern defining characters to test 369# 370# @return an array of results. Each result is an array REF 371# describing one pass. The array REF contains: 372# ->[0]: The method name as reported 373# ->[1]: The params on the '= <meth> begin ...' line 374# ->[2]: The params on the '= <meth> end ...' line 375# ->[3..]: GC messages from the JVM, if any 376# 377sub callJava { 378 my $testclass = shift; 379 my $method = shift; 380 my $pat = shift; 381 my $n = shift; 382 my $passes = shift; 383 384 my $n = ($n < 0) ? "-t ".(-$n) : "-i ".$n; 385 386 my $cmd = "java -classpath $CLASSPATH $testclass $method $n -p $passes -L @$pat[0] \"@$pat[1]\" \"@$pat[2]\" -r $THREADS"; 387 print "[$cmd]\n"; # for debugging 388 open(PIPE, "$cmd|") or die "Can't run \"$cmd\""; 389 my @out; 390 while (<PIPE>) { 391 push(@out, $_); 392 } 393 close(PIPE) or die "Java failed: \"$cmd\""; 394 395 @out = grep(!/^\#/, @out); # filter out comments 396 397 #print "[", join("\n", @out), "]\n"; 398 399 my @results; 400 my $method = ''; 401 my $data = []; 402 foreach (@out) { 403 next unless (/\S/); 404 405 if (/^=\s*(\w+)\s*(\w+)\s*(.*)/) { 406 my ($m, $state, $d) = ($1, $2, $3); 407 #print "$_ => [[$m $state $data]]\n"; 408 if ($state eq 'begin') { 409 die "$method was begun but not finished" if ($method); 410 $method = $m; 411 push(@$data, $d); 412 push(@$data, ''); # placeholder for end data 413 } elsif ($state eq 'end') { 414 if ($m ne $method) { 415 die "$method end does not match: $_"; 416 } 417 $data->[1] = $d; # insert end data at [1] 418 #print "#$method:", join(";",@$data), "\n"; 419 unshift(@$data, $method); # add method to start 420 421 push(@results, $data); 422 $method = ''; 423 $data = []; 424 } else { 425 die "Can't parse: $_"; 426 } 427 } 428 elsif (/^\[/) { 429 if ($method) { 430 push(@$data, $_); 431 } else { 432 # ignore extraneous GC notices 433 } 434 } 435 else { 436 die "Can't parse: $_"; 437 } 438 } 439 440 die "$method was begun but not finished" if ($method); 441 442 @results; 443} 444 445#----------------------------------------------------------------------------------- 446# doKeyGenTimes($Command_to_run, $time, $key_length) 447# Do a key-generation test and return the time and key length/char values. 448# 449sub doKeyTimes($$$) { 450 # print "$_[0]\n"; 451 local($x) = `$_[0]`; # execute the collperf command. 452 ($_[1], $_[2]) = split(/\,/, $x); # collperf returns "time, keylength" string. 453} 454 455 456#eof 457