1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.performance.tests; 18 19 import com.android.ddmlib.IDevice; 20 import com.android.ddmlib.MultiLineReceiver; 21 import com.android.ddmlib.NullOutputReceiver; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.device.DeviceNotAvailableException; 24 import com.android.tradefed.device.ITestDevice; 25 import com.android.tradefed.log.LogUtil.CLog; 26 import com.android.tradefed.result.FileInputStreamSource; 27 import com.android.tradefed.result.ITestInvocationListener; 28 import com.android.tradefed.result.InputStreamSource; 29 import com.android.tradefed.result.LogDataType; 30 import com.android.tradefed.testtype.IDeviceTest; 31 import com.android.tradefed.testtype.IRemoteTest; 32 import com.android.tradefed.util.FileUtil; 33 import com.android.tradefed.util.StreamUtil; 34 import com.android.tradefed.util.proto.TfMetricProtoUtil; 35 36 import junit.framework.TestCase; 37 38 import org.junit.Assert; 39 40 import java.io.File; 41 import java.util.HashMap; 42 import java.util.HashSet; 43 import java.util.LinkedList; 44 import java.util.List; 45 import java.util.Map; 46 import java.util.Map.Entry; 47 import java.util.Set; 48 import java.util.concurrent.TimeUnit; 49 50 /** 51 * Runs the FIO benchmarks. 52 * 53 * <p>This test pushes the FIO executable to the device and runs several benchmarks. Running each 54 * benchmark consists of creating a config file, creating one or more data files, clearing the disk 55 * cache and then running FIO. The test runs a variety of different configurations including a 56 * simple benchmark with a single thread, a storage benchmark with 4 threads, a media server 57 * emulator, and a media scanner emulator. 58 */ 59 public class FioBenchmarkTest implements IDeviceTest, IRemoteTest { 60 // TODO: Refactor this to only pick out fields we care about. 61 private static final String[] FIO_V0_RESULT_FIELDS = { 62 "jobname", 63 "groupid", 64 "error", 65 // Read stats 66 "read-kb-io", 67 "read-bandwidth", 68 "read-runtime", 69 "read-slat-min", 70 "read-slat-max", 71 "read-slat-mean", 72 "read-slat-stddev", 73 "read-clat-min", 74 "read-clat-max", 75 "read-clat-mean", 76 "read-clat-stddev", 77 "read-bandwidth-min", 78 "read-bandwidth-max", 79 "read-bandwidth-percent", 80 "read-bandwidth-mean", 81 "read-bandwidth-stddev", 82 // Write stats 83 "write-kb-io", 84 "write-bandwidth", 85 "write-runtime", 86 "write-slat-min", 87 "write-slat-max", 88 "write-slat-mean", 89 "write-slat-stddev", 90 "write-clat-min", 91 "write-clat-max", 92 "write-clat-mean", 93 "write-clat-stddev", 94 "write-bandwidth-min", 95 "write-bandwidth-max", 96 "write-bandwidth-percent", 97 "write-bandwidth-mean", 98 "write-bandwidth-stddev", 99 // CPU stats 100 "cpu-user", 101 "cpu-system", 102 "cpu-context-switches", 103 "cpu-major-page-faults", 104 "cpu-minor-page-faults", 105 // IO depth stats 106 "io-depth-1", 107 "io-depth-2", 108 "io-depth-4", 109 "io-depth-8", 110 "io-depth-16", 111 "io-depth-32", 112 "io-depth-64", 113 // IO lat stats 114 "io-lat-2-ms", 115 "io-lat-4-ms", 116 "io-lat-10-ms", 117 "io-lat-20-ms", 118 "io-lat-50-ms", 119 "io-lat-100-ms", 120 "io-lat-250-ms", 121 "io-lat-500-ms", 122 "io-lat-750-ms", 123 "io-lat-1000-ms", 124 "io-lat-2000-ms" 125 }; 126 private static final String[] FIO_V3_RESULT_FIELDS = { 127 "terse-version", 128 "fio-version", 129 "jobname", 130 "groupid", 131 "error", 132 // Read stats 133 "read-kb-io", 134 "read-bandwidth", 135 "read-iops", 136 "read-runtime", 137 "read-slat-min", 138 "read-slat-max", 139 "read-slat-mean", 140 "read-slat-stddev", 141 "read-clat-min", 142 "read-clat-max", 143 "read-clat-mean", 144 "read-clat-stddev", 145 null, 146 null, 147 null, 148 null, 149 null, 150 null, 151 null, 152 null, 153 null, 154 null, 155 null, 156 null, 157 null, 158 null, 159 null, 160 null, 161 null, 162 null, 163 null, 164 null, 165 "read-lat-min", 166 "read-lat-max", 167 "read-lat-mean", 168 "read-lat-stddev", 169 "read-bandwidth-min", 170 "read-bandwidth-max", 171 "read-bandwidth-percent", 172 "read-bandwidth-mean", 173 "read-bandwidth-stddev", 174 // Write stats 175 "write-kb-io", 176 "write-bandwidth", 177 "write-iops", 178 "write-runtime", 179 "write-slat-min", 180 "write-slat-max", 181 "write-slat-mean", 182 "write-slat-stddev", 183 "write-clat-min", 184 "write-clat-max", 185 "write-clat-mean", 186 "write-clat-stddev", 187 null, 188 null, 189 null, 190 null, 191 null, 192 null, 193 null, 194 null, 195 null, 196 null, 197 null, 198 null, 199 null, 200 null, 201 null, 202 null, 203 null, 204 null, 205 null, 206 null, 207 "write-lat-min", 208 "write-lat-max", 209 "write-lat-mean", 210 "write-lat-stddev", 211 "write-bandwidth-min", 212 "write-bandwidth-max", 213 "write-bandwidth-percent", 214 "write-bandwidth-mean", 215 "write-bandwidth-stddev", 216 // CPU stats 217 "cpu-user", 218 "cpu-system", 219 "cpu-context-switches", 220 "cpu-major-page-faults", 221 "cpu-minor-page-faults", 222 // IO depth stats 223 "io-depth-1", 224 "io-depth-2", 225 "io-depth-4", 226 "io-depth-8", 227 "io-depth-16", 228 "io-depth-32", 229 "io-depth-64", 230 // IO lat stats 231 "io-lat-2-us", 232 "io-lat-4-us", 233 "io-lat-10-us", 234 "io-lat-20-us", 235 "io-lat-50-us", 236 "io-lat-100-us", 237 "io-lat-250-us", 238 "io-lat-500-us", 239 "io-lat-750-us", 240 "io-lat-1000-us", 241 "io-lat-2-ms", 242 "io-lat-4-ms", 243 "io-lat-10-ms", 244 "io-lat-20-ms", 245 "io-lat-50-ms", 246 "io-lat-100-ms", 247 "io-lat-250-ms", 248 "io-lat-500-ms", 249 "io-lat-750-ms", 250 "io-lat-1000-ms", 251 "io-lat-2000-ms", 252 "io-lat-greater" 253 }; 254 255 private List<TestInfo> mTestCases = null; 256 257 /** 258 * Holds info about a job. The job translates into a job in the FIO config file. Contains the 259 * job name and a map from keys to values. 260 */ 261 private static class JobInfo { 262 public String mJobName = null; 263 public Map<String, String> mParameters = new HashMap<>(); 264 265 /** 266 * Gets the job as a string. 267 * 268 * @return a string of the job formatted for the config file. 269 */ createJob()270 public String createJob() { 271 if (mJobName == null) { 272 return ""; 273 } 274 StringBuilder sb = new StringBuilder(); 275 sb.append(String.format("[%s]\n", mJobName)); 276 for (Entry<String, String> parameter : mParameters.entrySet()) { 277 if (parameter.getValue() == null) { 278 sb.append(String.format("%s\n", parameter.getKey())); 279 } else { 280 sb.append(String.format("%s=%s\n", parameter.getKey(), parameter.getValue())); 281 } 282 } 283 return sb.toString(); 284 } 285 } 286 287 /** 288 * Holds info about a file used in the benchmark. Because of limitations in FIO on Android, the 289 * file needs to be created before the tests are run. Contains the file name and size in kB. 290 */ 291 private static class TestFileInfo { 292 public String mFileName = null; 293 public int mSize = -1; 294 } 295 296 /** Holds info about the perf metric that are cared about for a given job. */ 297 private static class PerfMetricInfo { 298 public enum ResultType { 299 STRING, 300 INT, 301 FLOAT, 302 PERCENT; 303 value(String input)304 String value(String input) { 305 switch (this) { 306 case STRING: 307 case INT: 308 case FLOAT: 309 return input; 310 case PERCENT: 311 if (input.length() < 2 || !input.endsWith("%")) { 312 return null; 313 } 314 try { 315 return String.format( 316 "%f", 317 Double.parseDouble(input.substring(0, input.length() - 1)) 318 / 100); 319 } catch (NumberFormatException e) { 320 return null; 321 } 322 default: 323 return null; 324 } 325 } 326 } 327 328 public String mJobName = null; 329 public String mFieldName = null; 330 public String mPostKey = null; 331 public ResultType mType = ResultType.STRING; 332 } 333 334 /** 335 * Holds the info associated with a test. 336 * 337 * <p>Contains the test name, key, a list of {@link JobInfo}, a set of {@link TestFileInfo}, and 338 * a set of {@link PerfMetricInfo}. 339 */ 340 private static class TestInfo { 341 public String mTestName = null; 342 public String mKey = null; 343 public List<JobInfo> mJobs = new LinkedList<>(); 344 public Set<TestFileInfo> mTestFiles = new HashSet<>(); 345 public Set<PerfMetricInfo> mPerfMetrics = new HashSet<>(); 346 347 /** 348 * Gets the config file. 349 * 350 * @return a string containing the contents of the config file needed to run the benchmark. 351 */ createConfig()352 private String createConfig() { 353 StringBuilder sb = new StringBuilder(); 354 for (JobInfo job : mJobs) { 355 sb.append(String.format("%s\n", job.createJob())); 356 } 357 return sb.toString(); 358 } 359 } 360 361 /** 362 * Parses the output of the FIO and allows the values to be looked up by job name and property. 363 */ 364 private static class FioParser extends MultiLineReceiver { 365 public Map<String, Map<String, String>> mResults = new HashMap<>(); 366 367 /** 368 * Gets the result for a job and property, or null if the job or the property do not exist. 369 * 370 * @param job the name of the job. 371 * @param property the name of the property. See {@code FIO_RESULT_FIELDS}. 372 * @return the fio results for the job and property or null if it does not exist. 373 */ getResult(String job, String property)374 public String getResult(String job, String property) { 375 if (!mResults.containsKey(job)) { 376 return null; 377 } 378 return mResults.get(job).get(property); 379 } 380 381 /** {@inheritDoc} */ 382 @Override processNewLines(String[] lines)383 public void processNewLines(String[] lines) { 384 for (String line : lines) { 385 CLog.d(line); 386 String[] fields = line.split(";"); 387 if (fields.length < FIO_V0_RESULT_FIELDS.length) { 388 continue; 389 } 390 if (fields.length < FIO_V3_RESULT_FIELDS.length) { 391 Map<String, String> r = new HashMap<>(); 392 for (int i = 0; i < FIO_V0_RESULT_FIELDS.length; i++) { 393 r.put(FIO_V0_RESULT_FIELDS[i], fields[i]); 394 } 395 mResults.put(fields[0], r); // Job name is index 0 396 } else if ("3".equals(fields[0])) { 397 Map<String, String> r = new HashMap<>(); 398 for (int i = 0; i < FIO_V3_RESULT_FIELDS.length; i++) { 399 r.put(FIO_V3_RESULT_FIELDS[i], fields[i]); 400 } 401 mResults.put(fields[2], r); // Job name is index 2 402 } else { 403 Assert.fail("Unknown fio terse output version"); 404 } 405 } 406 } 407 408 /** {@inheritDoc} */ 409 @Override isCancelled()410 public boolean isCancelled() { 411 return false; 412 } 413 } 414 415 ITestDevice mTestDevice = null; 416 417 private String mFioDir = null; 418 private String mFioBin = null; 419 private String mFioConfig = null; 420 421 @Option( 422 name = "fio-location", 423 description = 424 "The path to the precompiled FIO executable. If " 425 + "unset, try to use fio from the system image.") 426 private String mFioLocation = null; 427 428 @Option(name = "tmp-dir", description = "The directory used for interal benchmarks.") 429 private String mTmpDir = "/data/tmp/fio"; 430 431 @Option(name = "internal-test-dir", description = "The directory used for interal benchmarks.") 432 private String mInternalTestDir = "/data/fio/data"; 433 434 @Option(name = "media-test-dir", description = "The directory used for media benchmarks.") 435 private String mMediaTestDir = "${EXTERNAL_STORAGE}/fio"; 436 437 @Option(name = "external-test-dir", description = "The directory used for external benchmarks.") 438 private String mExternalTestDir = "${EXTERNAL_STORAGE}/fio"; 439 440 @Option(name = "collect-yaffs-logs", description = "Collect yaffs logs before and after tests") 441 private Boolean mCollectYaffsLogs = false; 442 443 @Option(name = "run-simple-internal-test", description = "Run the simple internal benchmark.") 444 private Boolean mRunSimpleInternalTest = true; 445 446 @Option( 447 name = "simple-internal-file-size", 448 description = "The file size of the simple internal benchmark in MB.") 449 private int mSimpleInternalFileSize = 256; 450 451 @Option(name = "run-simple-external-test", description = "Run the simple external benchmark.") 452 private Boolean mRunSimpleExternalTest = false; 453 454 @Option( 455 name = "simple-external-file-size", 456 description = "The file size of the simple external benchmark in MB.") 457 private int mSimpleExternalFileSize = 256; 458 459 @Option(name = "run-storage-internal-test", description = "Run the storage internal benchmark.") 460 private Boolean mRunStorageInternalTest = true; 461 462 @Option( 463 name = "storage-internal-file-size", 464 description = "The file size of the storage internal benchmark in MB.") 465 private int mStorageInternalFileSize = 256; 466 467 @Option( 468 name = "storage-internal-job-count", 469 description = "The number of jobs for the storage internal benchmark.") 470 private int mStorageInternalJobCount = 4; 471 472 @Option(name = "run-storage-external-test", description = "Run the storage external benchmark.") 473 private Boolean mRunStorageExternalTest = false; 474 475 @Option( 476 name = "storage-external-file-size", 477 description = "The file size of the storage external benchmark in MB.") 478 private int mStorageExternalFileSize = 256; 479 480 @Option( 481 name = "storage-external-job-count", 482 description = "The number of jobs for the storage external benchmark.") 483 private int mStorageExternalJobCount = 4; 484 485 @Option(name = "run-media-server-test", description = "Run the media server benchmark.") 486 private Boolean mRunMediaServerTest = false; 487 488 @Option( 489 name = "media-server-duration", 490 description = "The duration of the media server benchmark in secs.") 491 private long mMediaServerDuration = 30; 492 493 @Option( 494 name = "media-server-media-file-size", 495 description = "The media file size of the media server benchmark in MB.") 496 private int mMediaServerMediaFileSize = 256; 497 498 @Option( 499 name = "media-server-worker-file-size", 500 description = "The worker file size of the media server benchmark in MB.") 501 private int mMediaServerWorkerFileSize = 256; 502 503 @Option( 504 name = "media-server-worker-job-count", 505 description = "The number of worker jobs for the media server benchmark.") 506 private int mMediaServerWorkerJobCount = 4; 507 508 @Option(name = "run-media-scanner-test", description = "Run the media scanner benchmark.") 509 private Boolean mRunMediaScannerTest = false; 510 511 @Option( 512 name = "media-scanner-media-file-size", 513 description = "The media file size of the media scanner benchmark in kB.") 514 private int mMediaScannerMediaFileSize = 8; 515 516 @Option( 517 name = "media-scanner-media-file-count", 518 description = "The number of media files to scan.") 519 private int mMediaScannerMediaFileCount = 256; 520 521 @Option( 522 name = "media-scanner-worker-file-size", 523 description = "The worker file size of the media scanner benchmark in MB.") 524 private int mMediaScannerWorkerFileSize = 256; 525 526 @Option( 527 name = "media-scanner-worker-job-count", 528 description = "The number of worker jobs for the media server benchmark.") 529 private int mMediaScannerWorkerJobCount = 4; 530 531 @Option( 532 name = "key-suffix", 533 description = "The suffix to add to the reporting key in order to override the default") 534 private String mKeySuffix = null; 535 536 /** Sets up all the benchmarks. */ setupTests()537 private void setupTests() { 538 if (mTestCases != null) { 539 // assume already set up 540 return; 541 } 542 543 mTestCases = new LinkedList<>(); 544 545 if (mRunSimpleInternalTest) { 546 addSimpleTest("read", "sync", true); 547 addSimpleTest("write", "sync", true); 548 addSimpleTest("randread", "sync", true); 549 addSimpleTest("randwrite", "sync", true); 550 addSimpleTest("randread", "mmap", true); 551 addSimpleTest("randwrite", "mmap", true); 552 } 553 554 if (mRunSimpleExternalTest) { 555 addSimpleTest("read", "sync", false); 556 addSimpleTest("write", "sync", false); 557 addSimpleTest("randread", "sync", false); 558 addSimpleTest("randwrite", "sync", false); 559 addSimpleTest("randread", "mmap", false); 560 addSimpleTest("randwrite", "mmap", false); 561 } 562 563 if (mRunStorageInternalTest) { 564 addStorageTest("read", "sync", true); 565 addStorageTest("write", "sync", true); 566 addStorageTest("randread", "sync", true); 567 addStorageTest("randwrite", "sync", true); 568 addStorageTest("randread", "mmap", true); 569 addStorageTest("randwrite", "mmap", true); 570 } 571 572 if (mRunStorageExternalTest) { 573 addStorageTest("read", "sync", false); 574 addStorageTest("write", "sync", false); 575 addStorageTest("randread", "sync", false); 576 addStorageTest("randwrite", "sync", false); 577 addStorageTest("randread", "mmap", false); 578 addStorageTest("randwrite", "mmap", false); 579 } 580 581 if (mRunMediaServerTest) { 582 addMediaServerTest("read"); 583 addMediaServerTest("write"); 584 } 585 586 if (mRunMediaScannerTest) { 587 addMediaScannerTest(); 588 } 589 } 590 591 /** 592 * Sets up the simple FIO benchmark. 593 * 594 * <p>The test consists of a single process reading or writing to a file. 595 * 596 * @param rw the type of IO pattern. One of {@code read}, {@code write}, {@code randread}, or 597 * {@code randwrite}. 598 * @param ioengine defines how the job issues I/O. Such as {@code sync}, {@code vsync}, {@code 599 * mmap}, or {@code cpuio} and others. 600 * @param internal whether the test should be run on the internal (/data) partition or the 601 * external partition. 602 */ addSimpleTest(String rw, String ioengine, boolean internal)603 private void addSimpleTest(String rw, String ioengine, boolean internal) { 604 String fileName = "testfile"; 605 String jobName = "job"; 606 607 String directory; 608 int fileSize; 609 610 TestInfo t = new TestInfo(); 611 if (internal) { 612 t.mTestName = String.format("SimpleBenchmark-int-%s-%s", ioengine, rw); 613 t.mKey = "fio_simple_int_benchmark"; 614 directory = mInternalTestDir; 615 fileSize = mSimpleInternalFileSize; 616 } else { 617 t.mTestName = String.format("SimpleBenchmark-ext-%s-%s", ioengine, rw); 618 t.mKey = "fio_simple_ext_benchmark"; 619 directory = mExternalTestDir; 620 fileSize = mSimpleExternalFileSize; 621 } 622 623 TestFileInfo f = new TestFileInfo(); 624 f.mFileName = new File(directory, fileName).getAbsolutePath(); 625 f.mSize = fileSize * 1024; // fileSize is in MB but we want it in kB. 626 t.mTestFiles.add(f); 627 628 JobInfo j = new JobInfo(); 629 j.mJobName = jobName; 630 j.mParameters.put("directory", directory); 631 j.mParameters.put("filename", fileName); 632 j.mParameters.put("fsync", "1024"); 633 j.mParameters.put("ioengine", ioengine); 634 j.mParameters.put("rw", rw); 635 j.mParameters.put("size", String.format("%dM", fileSize)); 636 t.mJobs.add(j); 637 638 PerfMetricInfo m = new PerfMetricInfo(); 639 m.mJobName = jobName; 640 if ("sync".equals(ioengine)) { 641 m.mPostKey = String.format("%s_bandwidth", rw); 642 } else { 643 m.mPostKey = String.format("%s_%s_bandwidth", rw, ioengine); 644 } 645 m.mType = PerfMetricInfo.ResultType.FLOAT; 646 if (rw.endsWith("read")) { 647 m.mFieldName = "read-bandwidth-mean"; 648 } else if (rw.endsWith("write")) { 649 m.mFieldName = "write-bandwidth-mean"; 650 } 651 t.mPerfMetrics.add(m); 652 653 mTestCases.add(t); 654 } 655 656 /** 657 * Sets up the storage FIO benchmark. 658 * 659 * <p>The test consists of several processes reading or writing to a file. 660 * 661 * @param rw the type of IO pattern. One of {@code read}, {@code write}, {@code randread}, or 662 * {@code randwrite}. 663 * @param ioengine defines how the job issues I/O. Such as {@code sync}, {@code vsync}, {@code 664 * mmap}, or {@code cpuio} and others. 665 * @param internal whether the test should be run on the internal (/data) partition or the 666 * external partition. 667 */ addStorageTest(String rw, String ioengine, boolean internal)668 private void addStorageTest(String rw, String ioengine, boolean internal) { 669 String fileName = "testfile"; 670 String jobName = "workers"; 671 672 String directory; 673 int fileSize; 674 int jobCount; 675 676 TestInfo t = new TestInfo(); 677 if (internal) { 678 t.mTestName = String.format("StorageBenchmark-int-%s-%s", ioengine, rw); 679 t.mKey = "fio_storage_int_benchmark"; 680 directory = mInternalTestDir; 681 fileSize = mStorageInternalFileSize; 682 jobCount = mStorageInternalJobCount; 683 } else { 684 t.mTestName = String.format("StorageBenchmark-ext-%s-%s", ioengine, rw); 685 t.mKey = "fio_storage_ext_benchmark"; 686 directory = mExternalTestDir; 687 fileSize = mStorageExternalFileSize; 688 jobCount = mStorageExternalJobCount; 689 } 690 691 TestFileInfo f = new TestFileInfo(); 692 f.mFileName = new File(directory, fileName).getAbsolutePath(); 693 f.mSize = fileSize * 1024; // fileSize is in MB but we want it in kB. 694 t.mTestFiles.add(f); 695 696 JobInfo j = new JobInfo(); 697 j.mJobName = jobName; 698 j.mParameters.put("directory", directory); 699 j.mParameters.put("filename", fileName); 700 j.mParameters.put("fsync", "1024"); 701 j.mParameters.put("group_reporting", null); 702 j.mParameters.put("ioengine", ioengine); 703 j.mParameters.put("new_group", null); 704 j.mParameters.put("numjobs", String.format("%d", jobCount)); 705 j.mParameters.put("rw", rw); 706 j.mParameters.put("size", String.format("%dM", fileSize)); 707 t.mJobs.add(j); 708 709 PerfMetricInfo m = new PerfMetricInfo(); 710 m.mJobName = jobName; 711 if ("sync".equals(ioengine)) { 712 m.mPostKey = String.format("%s_bandwidth", rw); 713 } else { 714 m.mPostKey = String.format("%s_%s_bandwidth", rw, ioengine); 715 } 716 m.mType = PerfMetricInfo.ResultType.FLOAT; 717 if (rw.endsWith("read")) { 718 m.mFieldName = "read-bandwidth-mean"; 719 } else if (rw.endsWith("write")) { 720 m.mFieldName = "write-bandwidth-mean"; 721 } 722 t.mPerfMetrics.add(m); 723 724 m = new PerfMetricInfo(); 725 m.mJobName = jobName; 726 if ("sync".equals(ioengine)) { 727 m.mPostKey = String.format("%s_latency", rw); 728 } else { 729 m.mPostKey = String.format("%s_%s_latency", rw, ioengine); 730 } 731 m.mType = PerfMetricInfo.ResultType.FLOAT; 732 if (rw.endsWith("read")) { 733 m.mFieldName = "read-clat-mean"; 734 } else if (rw.endsWith("write")) { 735 m.mFieldName = "write-clat-mean"; 736 } 737 t.mPerfMetrics.add(m); 738 739 mTestCases.add(t); 740 } 741 742 /** 743 * Sets up the media server benchmark. 744 * 745 * <p>The test consists of a single process at a higher priority reading or writing to a file 746 * while several other worker processes read and write to a different file. 747 * 748 * @param rw the type of IO pattern. One of {@code read}, {@code write} 749 */ addMediaServerTest(String rw)750 private void addMediaServerTest(String rw) { 751 String mediaJob = "media-server"; 752 String mediaFile = "mediafile"; 753 String workerJob = "workers"; 754 String workerFile = "workerfile"; 755 756 TestInfo t = new TestInfo(); 757 t.mTestName = String.format("MediaServerBenchmark-%s", rw); 758 t.mKey = "fio_media_server_benchmark"; 759 760 TestFileInfo f = new TestFileInfo(); 761 f.mFileName = new File(mMediaTestDir, mediaFile).getAbsolutePath(); 762 f.mSize = mMediaServerMediaFileSize * 1024; // File size is in MB but we want it in kB. 763 t.mTestFiles.add(f); 764 765 f = new TestFileInfo(); 766 f.mFileName = new File(mMediaTestDir, workerFile).getAbsolutePath(); 767 f.mSize = mMediaServerWorkerFileSize * 1024; // File size is in MB but we want it in kB. 768 t.mTestFiles.add(f); 769 770 JobInfo j = new JobInfo(); 771 j.mJobName = "global"; 772 j.mParameters.put("directory", mMediaTestDir); 773 j.mParameters.put("fsync", "1024"); 774 j.mParameters.put("ioengine", "sync"); 775 j.mParameters.put("runtime", String.format("%d", mMediaServerDuration)); 776 j.mParameters.put("time_based", null); 777 t.mJobs.add(j); 778 779 j = new JobInfo(); 780 j.mJobName = mediaJob; 781 j.mParameters.put("filename", mediaFile); 782 j.mParameters.put("iodepth", "32"); 783 j.mParameters.put("nice", "-16"); 784 j.mParameters.put("rate", "6m"); 785 j.mParameters.put("rw", rw); 786 j.mParameters.put("size", String.format("%dM", mMediaServerMediaFileSize)); 787 t.mJobs.add(j); 788 789 j = new JobInfo(); 790 j.mJobName = workerJob; 791 j.mParameters.put("filename", workerFile); 792 j.mParameters.put("group_reporting", null); 793 j.mParameters.put("new_group", null); 794 j.mParameters.put("nice", "0"); 795 j.mParameters.put("numjobs", String.format("%d", mMediaServerWorkerJobCount)); 796 j.mParameters.put("rw", "randrw"); 797 j.mParameters.put("size", String.format("%dM", mMediaServerWorkerFileSize)); 798 t.mJobs.add(j); 799 800 PerfMetricInfo m = new PerfMetricInfo(); 801 m.mJobName = mediaJob; 802 m.mPostKey = String.format("%s_media_bandwidth", rw); 803 m.mType = PerfMetricInfo.ResultType.FLOAT; 804 if (rw.endsWith("read")) { 805 m.mFieldName = "read-bandwidth-mean"; 806 } else if (rw.endsWith("write")) { 807 m.mFieldName = "write-bandwidth-mean"; 808 } 809 t.mPerfMetrics.add(m); 810 811 m = new PerfMetricInfo(); 812 m.mJobName = mediaJob; 813 m.mPostKey = String.format("%s_media_latency", rw); 814 m.mType = PerfMetricInfo.ResultType.FLOAT; 815 if (rw.endsWith("read")) { 816 m.mFieldName = "read-clat-mean"; 817 } else if (rw.endsWith("write")) { 818 m.mFieldName = "write-clat-mean"; 819 } 820 t.mPerfMetrics.add(m); 821 822 m = new PerfMetricInfo(); 823 m.mJobName = workerJob; 824 m.mPostKey = String.format("%s_workers_read_bandwidth", rw); 825 m.mFieldName = "read-bandwidth-mean"; 826 m.mType = PerfMetricInfo.ResultType.FLOAT; 827 t.mPerfMetrics.add(m); 828 829 m = new PerfMetricInfo(); 830 m.mJobName = workerJob; 831 m.mPostKey = String.format("%s_workers_write_bandwidth", rw); 832 m.mFieldName = "write-bandwidth-mean"; 833 m.mType = PerfMetricInfo.ResultType.FLOAT; 834 t.mPerfMetrics.add(m); 835 836 mTestCases.add(t); 837 } 838 839 /** 840 * Sets up the media scanner benchmark. 841 * 842 * <p>The test consists of a single process reading several small files while several other 843 * worker processes read and write to a different file. 844 */ addMediaScannerTest()845 private void addMediaScannerTest() { 846 String mediaJob = "media-server"; 847 String mediaFile = "mediafile.%d"; 848 String workerJob = "workers"; 849 String workerFile = "workerfile"; 850 851 TestInfo t = new TestInfo(); 852 t.mTestName = "MediaScannerBenchmark"; 853 t.mKey = "fio_media_scanner_benchmark"; 854 855 TestFileInfo f; 856 for (int i = 0; i < mMediaScannerMediaFileCount; i++) { 857 f = new TestFileInfo(); 858 f.mFileName = new File(mMediaTestDir, String.format(mediaFile, i)).getAbsolutePath(); 859 f.mSize = mMediaScannerMediaFileSize; // File size is already in kB so do nothing. 860 t.mTestFiles.add(f); 861 } 862 863 f = new TestFileInfo(); 864 f.mFileName = new File(mMediaTestDir, workerFile).getAbsolutePath(); 865 f.mSize = mMediaScannerWorkerFileSize * 1024; // File size is in MB but we want it in kB. 866 t.mTestFiles.add(f); 867 868 JobInfo j = new JobInfo(); 869 j.mJobName = "global"; 870 j.mParameters.put("directory", mMediaTestDir); 871 j.mParameters.put("fsync", "1024"); 872 j.mParameters.put("ioengine", "sync"); 873 t.mJobs.add(j); 874 875 j = new JobInfo(); 876 j.mJobName = mediaJob; 877 StringBuilder fileNames = new StringBuilder(); 878 fileNames.append(String.format(mediaFile, 0)); 879 for (int i = 1; i < mMediaScannerMediaFileCount; i++) { 880 fileNames.append(String.format(":%s", String.format(mediaFile, i))); 881 } 882 j.mParameters.put("filename", fileNames.toString()); 883 j.mParameters.put("exitall", null); 884 j.mParameters.put("openfiles", "4"); 885 j.mParameters.put("rw", "read"); 886 t.mJobs.add(j); 887 888 j = new JobInfo(); 889 j.mJobName = workerJob; 890 j.mParameters.put("filename", workerFile); 891 j.mParameters.put("group_reporting", null); 892 j.mParameters.put("new_group", null); 893 j.mParameters.put("numjobs", String.format("%d", mMediaScannerWorkerJobCount)); 894 j.mParameters.put("rw", "randrw"); 895 j.mParameters.put("size", String.format("%dM", mMediaScannerWorkerFileSize)); 896 t.mJobs.add(j); 897 898 PerfMetricInfo m = new PerfMetricInfo(); 899 m.mJobName = mediaJob; 900 m.mPostKey = "media_bandwidth"; 901 m.mFieldName = "read-bandwidth-mean"; 902 m.mType = PerfMetricInfo.ResultType.FLOAT; 903 t.mPerfMetrics.add(m); 904 905 m = new PerfMetricInfo(); 906 m.mJobName = mediaJob; 907 m.mPostKey = "media_latency"; 908 m.mFieldName = "read-clat-mean"; 909 m.mType = PerfMetricInfo.ResultType.FLOAT; 910 t.mPerfMetrics.add(m); 911 912 m = new PerfMetricInfo(); 913 m.mJobName = workerJob; 914 m.mPostKey = "workers_read_bandwidth"; 915 m.mFieldName = "read-bandwidth-mean"; 916 m.mType = PerfMetricInfo.ResultType.FLOAT; 917 t.mPerfMetrics.add(m); 918 919 m = new PerfMetricInfo(); 920 m.mJobName = workerJob; 921 m.mPostKey = "workers_write_bandwidth"; 922 m.mFieldName = "write-bandwidth-mean"; 923 m.mType = PerfMetricInfo.ResultType.FLOAT; 924 t.mPerfMetrics.add(m); 925 926 mTestCases.add(t); 927 } 928 929 /** 930 * Creates the directories needed to run FIO, pushes the FIO executable to the device, and stops 931 * the runtime. 932 * 933 * @throws DeviceNotAvailableException if the device is not available. 934 */ setupDevice()935 private void setupDevice() throws DeviceNotAvailableException { 936 mTestDevice.executeShellCommand("stop"); 937 mTestDevice.executeShellCommand(String.format("mkdir -p %s", mFioDir)); 938 mTestDevice.executeShellCommand(String.format("mkdir -p %s", mTmpDir)); 939 mTestDevice.executeShellCommand(String.format("mkdir -p %s", mInternalTestDir)); 940 mTestDevice.executeShellCommand(String.format("mkdir -p %s", mMediaTestDir)); 941 if (mExternalTestDir != null) { 942 mTestDevice.executeShellCommand(String.format("mkdir -p %s", mExternalTestDir)); 943 } 944 if (mFioLocation != null) { 945 mTestDevice.pushFile(new File(mFioLocation), mFioBin); 946 mTestDevice.executeShellCommand(String.format("chmod 755 %s", mFioBin)); 947 } 948 } 949 950 /** 951 * Reverses the actions of {@link #setDevice(ITestDevice)}. 952 * 953 * @throws DeviceNotAvailableException If the device is not available. 954 */ cleanupDevice()955 private void cleanupDevice() throws DeviceNotAvailableException { 956 if (mExternalTestDir != null) { 957 mTestDevice.executeShellCommand(String.format("rm -r %s", mExternalTestDir)); 958 } 959 mTestDevice.executeShellCommand(String.format("rm -r %s", mMediaTestDir)); 960 mTestDevice.executeShellCommand(String.format("rm -r %s", mInternalTestDir)); 961 mTestDevice.executeShellCommand(String.format("rm -r %s", mTmpDir)); 962 mTestDevice.executeShellCommand(String.format("rm -r %s", mFioDir)); 963 mTestDevice.executeShellCommand("start"); 964 mTestDevice.waitForDeviceAvailable(); 965 } 966 967 /** 968 * Runs a single test, including creating the test files, clearing the cache, collecting before 969 * and after files, running the benchmark, and reporting the results. 970 * 971 * @param test the benchmark. 972 * @param listener the ITestInvocationListener 973 * @throws DeviceNotAvailableException if the device is not available. 974 */ runTest(TestInfo test, ITestInvocationListener listener)975 private void runTest(TestInfo test, ITestInvocationListener listener) 976 throws DeviceNotAvailableException { 977 CLog.i("Running %s benchmark", test.mTestName); 978 mTestDevice.executeShellCommand(String.format("rm -r %s/*", mTmpDir)); 979 mTestDevice.executeShellCommand(String.format("rm -r %s/*", mInternalTestDir)); 980 mTestDevice.executeShellCommand(String.format("rm -r %s/*", mMediaTestDir)); 981 if (mExternalTestDir != null) { 982 mTestDevice.executeShellCommand(String.format("rm -r %s/*", mExternalTestDir)); 983 } 984 985 for (TestFileInfo file : test.mTestFiles) { 986 CLog.v("Creating file: %s, size: %dkB", file.mFileName, file.mSize); 987 String cmd = 988 String.format( 989 "dd if=/dev/urandom of=%s bs=1024 count=%d", 990 file.mFileName, file.mSize); 991 int timeout = file.mSize * 2 * 1000; // Timeout is 2 seconds per kB. 992 mTestDevice.executeShellCommand( 993 cmd, new NullOutputReceiver(), timeout, TimeUnit.MILLISECONDS, 2); 994 } 995 996 CLog.i("Creating config"); 997 CLog.d("Config file:\n%s", test.createConfig()); 998 mTestDevice.pushString(test.createConfig(), mFioConfig); 999 1000 CLog.i("Dropping cache"); 1001 mTestDevice.executeShellCommand("echo 3 > /proc/sys/vm/drop_caches"); 1002 1003 collectLogs(test, listener, "before"); 1004 1005 CLog.i("Running test"); 1006 FioParser output = new FioParser(); 1007 // Run FIO with a timeout of 1 hour. 1008 mTestDevice.executeShellCommand( 1009 String.format("%s --minimal %s", mFioBin, mFioConfig), 1010 output, 1011 60 * 60 * 1000, 1012 TimeUnit.MILLISECONDS, 1013 2); 1014 1015 collectLogs(test, listener, "after"); 1016 1017 // Report metrics 1018 Map<String, String> metrics = new HashMap<>(); 1019 String key = mKeySuffix == null ? test.mKey : test.mKey + mKeySuffix; 1020 1021 listener.testRunStarted(key, 0); 1022 for (PerfMetricInfo m : test.mPerfMetrics) { 1023 if (!output.mResults.containsKey(m.mJobName)) { 1024 CLog.w("Job name %s was not found in the results", m.mJobName); 1025 continue; 1026 } 1027 1028 String value = output.getResult(m.mJobName, m.mFieldName); 1029 if (value != null) { 1030 metrics.put(m.mPostKey, m.mType.value(value)); 1031 } else { 1032 CLog.w("%s was not in results for the job %s", m.mFieldName, m.mJobName); 1033 } 1034 } 1035 1036 CLog.d("About to report metrics to %s: %s", key, metrics); 1037 listener.testRunEnded(0, TfMetricProtoUtil.upgradeConvert(metrics)); 1038 } 1039 collectLogs(TestInfo testInfo, ITestInvocationListener listener, String descriptor)1040 private void collectLogs(TestInfo testInfo, ITestInvocationListener listener, String descriptor) 1041 throws DeviceNotAvailableException { 1042 if (mCollectYaffsLogs && mTestDevice.doesFileExist("/proc/yaffs")) { 1043 logFile( 1044 "/proc/yaffs", 1045 String.format("%s-yaffs-%s", testInfo.mTestName, descriptor), 1046 mTestDevice, 1047 listener); 1048 } 1049 } 1050 logFile( String remoteFileName, String localFileName, ITestDevice testDevice, ITestInvocationListener listener)1051 private void logFile( 1052 String remoteFileName, 1053 String localFileName, 1054 ITestDevice testDevice, 1055 ITestInvocationListener listener) 1056 throws DeviceNotAvailableException { 1057 File outputFile = null; 1058 InputStreamSource outputSource = null; 1059 try { 1060 outputFile = testDevice.pullFile(remoteFileName); 1061 if (outputFile != null) { 1062 CLog.d("Sending %d byte file %s to logosphere!", outputFile.length(), outputFile); 1063 outputSource = new FileInputStreamSource(outputFile); 1064 listener.testLog(localFileName, LogDataType.TEXT, outputSource); 1065 } 1066 } finally { 1067 FileUtil.deleteFile(outputFile); 1068 StreamUtil.cancel(outputSource); 1069 } 1070 } 1071 1072 /** {@inheritDoc} */ 1073 @Override run(ITestInvocationListener listener)1074 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 1075 Assert.assertNotNull(mTestDevice); 1076 1077 mFioDir = new File(mTestDevice.getMountPoint(IDevice.MNT_DATA), "fio").getAbsolutePath(); 1078 if (mFioLocation != null) { 1079 mFioBin = new File(mFioDir, "fio").getAbsolutePath(); 1080 } else { 1081 mFioBin = "fio"; 1082 } 1083 mFioConfig = new File(mFioDir, "config.fio").getAbsolutePath(); 1084 1085 setupTests(); 1086 setupDevice(); 1087 1088 for (TestInfo test : mTestCases) { 1089 runTest(test, listener); 1090 } 1091 1092 cleanupDevice(); 1093 } 1094 1095 /** {@inheritDoc} */ 1096 @Override setDevice(ITestDevice device)1097 public void setDevice(ITestDevice device) { 1098 mTestDevice = device; 1099 } 1100 1101 /** {@inheritDoc} */ 1102 @Override getDevice()1103 public ITestDevice getDevice() { 1104 return mTestDevice; 1105 } 1106 1107 /** A meta-test to ensure that the bits of FioBenchmarkTest are working properly. */ 1108 public static class MetaTest extends TestCase { 1109 1110 /** Test that {@link JobInfo#createJob()} properly formats a job. */ testCreateJob()1111 public void testCreateJob() { 1112 JobInfo j = new JobInfo(); 1113 assertEquals("", j.createJob()); 1114 j.mJobName = "job"; 1115 assertEquals("[job]\n", j.createJob()); 1116 j.mParameters.put("param1", null); 1117 j.mParameters.put("param2", "value"); 1118 String[] lines = j.createJob().split("\n"); 1119 assertEquals(3, lines.length); 1120 assertEquals("[job]", lines[0]); 1121 Set<String> params = new HashSet<>(2); 1122 params.add(lines[1]); 1123 params.add(lines[2]); 1124 assertTrue(params.contains("param1")); 1125 assertTrue(params.contains("param2=value")); 1126 } 1127 1128 /** Test that {@link TestInfo#createConfig()} properly formats a config. */ testCreateConfig()1129 public void testCreateConfig() { 1130 TestInfo t = new TestInfo(); 1131 JobInfo j = new JobInfo(); 1132 j.mJobName = "job1"; 1133 j.mParameters.put("param1", "value1"); 1134 t.mJobs.add(j); 1135 1136 j = new JobInfo(); 1137 j.mJobName = "job2"; 1138 j.mParameters.put("param2", "value2"); 1139 t.mJobs.add(j); 1140 1141 j = new JobInfo(); 1142 j.mJobName = "job3"; 1143 j.mParameters.put("param3", "value3"); 1144 t.mJobs.add(j); 1145 1146 assertEquals( 1147 "[job1]\nparam1=value1\n\n" 1148 + "[job2]\nparam2=value2\n\n" 1149 + "[job3]\nparam3=value3\n\n", 1150 t.createConfig()); 1151 } 1152 1153 /** 1154 * Test that output lines are parsed correctly by the FioParser, invalid lines are ignored, 1155 * and that the various fields are accessible with {@link FioParser#getResult(String, 1156 * String)}. 1157 */ testFioParser()1158 public void testFioParser() { 1159 String[] lines = new String[4]; 1160 // We build the lines up as follows (assuming FIO_RESULTS_FIELDS.length == 58): 1161 // 0;3;6;...;171 1162 // 1;4;7;...;172 1163 // 2;5;8;...;173 1164 for (int i = 0; i < 3; i++) { 1165 StringBuilder sb = new StringBuilder(); 1166 sb.append(i); 1167 for (int j = 1; j < FIO_V0_RESULT_FIELDS.length; j++) { 1168 sb.append(";"); 1169 sb.append(j * 3 + i); 1170 } 1171 lines[i] = sb.toString(); 1172 } 1173 // A line may have an optional description on the end which we don't care about, make 1174 // sure it still parses. 1175 lines[2] = lines[2] += ";description"; 1176 // Make sure an invalid output line does not parse. 1177 lines[3] = "invalid"; 1178 1179 FioParser p = new FioParser(); 1180 p.processNewLines(lines); 1181 1182 for (int i = 0; i < 3; i++) { 1183 for (int j = 0; j < FIO_V0_RESULT_FIELDS.length; j++) { 1184 assertEquals( 1185 String.format("job=%d, field=%s", i, FIO_V0_RESULT_FIELDS[j]), 1186 String.format("%d", j * 3 + i), 1187 p.getResult(String.format("%d", i), FIO_V0_RESULT_FIELDS[j])); 1188 } 1189 } 1190 assertNull(p.getResult("missing", "jobname")); 1191 assertNull(p.getResult("invalid", "jobname")); 1192 assertNull(p.getResult("0", "missing")); 1193 } 1194 1195 /** 1196 * Test that {@link PerfMetricInfo.ResultType#value(String)} correctly transforms strings 1197 * based on the result type. 1198 */ testResultTypeValue()1199 public void testResultTypeValue() { 1200 assertEquals("test", PerfMetricInfo.ResultType.STRING.value("test")); 1201 assertEquals("1", PerfMetricInfo.ResultType.INT.value("1")); 1202 assertEquals("3.14159", PerfMetricInfo.ResultType.FLOAT.value("3.14159")); 1203 assertEquals( 1204 String.format("%f", 0.34567), 1205 PerfMetricInfo.ResultType.PERCENT.value("34.567%")); 1206 assertNull(PerfMetricInfo.ResultType.PERCENT.value("")); 1207 assertNull(PerfMetricInfo.ResultType.PERCENT.value("34.567")); 1208 assertNull(PerfMetricInfo.ResultType.PERCENT.value("test%")); 1209 } 1210 } 1211 } 1212