/*
 * Copyright (C) 2011 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.performance.tests;

import com.android.ddmlib.IDevice;
import com.android.ddmlib.MultiLineReceiver;
import com.android.ddmlib.NullOutputReceiver;
import com.android.tradefed.config.Option;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.FileInputStreamSource;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.InputStreamSource;
import com.android.tradefed.result.LogDataType;
import com.android.tradefed.testtype.IDeviceTest;
import com.android.tradefed.testtype.IRemoteTest;
import com.android.tradefed.util.FileUtil;
import com.android.tradefed.util.StreamUtil;
import com.android.tradefed.util.proto.TfMetricProtoUtil;

import junit.framework.TestCase;

import org.junit.Assert;

import java.io.File;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.TimeUnit;

/**
 * Runs the FIO benchmarks.
 *
 * <p>This test pushes the FIO executable to the device and runs several benchmarks. Running each
 * benchmark consists of creating a config file, creating one or more data files, clearing the disk
 * cache and then running FIO. The test runs a variety of different configurations including a
 * simple benchmark with a single thread, a storage benchmark with 4 threads, a media server
 * emulator, and a media scanner emulator.
 */
public class FioBenchmarkTest implements IDeviceTest, IRemoteTest {
    // TODO: Refactor this to only pick out fields we care about.
    private static final String[] FIO_V0_RESULT_FIELDS = {
        "jobname",
        "groupid",
        "error",
        // Read stats
        "read-kb-io",
        "read-bandwidth",
        "read-runtime",
        "read-slat-min",
        "read-slat-max",
        "read-slat-mean",
        "read-slat-stddev",
        "read-clat-min",
        "read-clat-max",
        "read-clat-mean",
        "read-clat-stddev",
        "read-bandwidth-min",
        "read-bandwidth-max",
        "read-bandwidth-percent",
        "read-bandwidth-mean",
        "read-bandwidth-stddev",
        // Write stats
        "write-kb-io",
        "write-bandwidth",
        "write-runtime",
        "write-slat-min",
        "write-slat-max",
        "write-slat-mean",
        "write-slat-stddev",
        "write-clat-min",
        "write-clat-max",
        "write-clat-mean",
        "write-clat-stddev",
        "write-bandwidth-min",
        "write-bandwidth-max",
        "write-bandwidth-percent",
        "write-bandwidth-mean",
        "write-bandwidth-stddev",
        // CPU stats
        "cpu-user",
        "cpu-system",
        "cpu-context-switches",
        "cpu-major-page-faults",
        "cpu-minor-page-faults",
        // IO depth stats
        "io-depth-1",
        "io-depth-2",
        "io-depth-4",
        "io-depth-8",
        "io-depth-16",
        "io-depth-32",
        "io-depth-64",
        // IO lat stats
        "io-lat-2-ms",
        "io-lat-4-ms",
        "io-lat-10-ms",
        "io-lat-20-ms",
        "io-lat-50-ms",
        "io-lat-100-ms",
        "io-lat-250-ms",
        "io-lat-500-ms",
        "io-lat-750-ms",
        "io-lat-1000-ms",
        "io-lat-2000-ms"
    };
    private static final String[] FIO_V3_RESULT_FIELDS = {
        "terse-version",
        "fio-version",
        "jobname",
        "groupid",
        "error",
        // Read stats
        "read-kb-io",
        "read-bandwidth",
        "read-iops",
        "read-runtime",
        "read-slat-min",
        "read-slat-max",
        "read-slat-mean",
        "read-slat-stddev",
        "read-clat-min",
        "read-clat-max",
        "read-clat-mean",
        "read-clat-stddev",
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        "read-lat-min",
        "read-lat-max",
        "read-lat-mean",
        "read-lat-stddev",
        "read-bandwidth-min",
        "read-bandwidth-max",
        "read-bandwidth-percent",
        "read-bandwidth-mean",
        "read-bandwidth-stddev",
        // Write stats
        "write-kb-io",
        "write-bandwidth",
        "write-iops",
        "write-runtime",
        "write-slat-min",
        "write-slat-max",
        "write-slat-mean",
        "write-slat-stddev",
        "write-clat-min",
        "write-clat-max",
        "write-clat-mean",
        "write-clat-stddev",
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        null,
        "write-lat-min",
        "write-lat-max",
        "write-lat-mean",
        "write-lat-stddev",
        "write-bandwidth-min",
        "write-bandwidth-max",
        "write-bandwidth-percent",
        "write-bandwidth-mean",
        "write-bandwidth-stddev",
        // CPU stats
        "cpu-user",
        "cpu-system",
        "cpu-context-switches",
        "cpu-major-page-faults",
        "cpu-minor-page-faults",
        // IO depth stats
        "io-depth-1",
        "io-depth-2",
        "io-depth-4",
        "io-depth-8",
        "io-depth-16",
        "io-depth-32",
        "io-depth-64",
        // IO lat stats
        "io-lat-2-us",
        "io-lat-4-us",
        "io-lat-10-us",
        "io-lat-20-us",
        "io-lat-50-us",
        "io-lat-100-us",
        "io-lat-250-us",
        "io-lat-500-us",
        "io-lat-750-us",
        "io-lat-1000-us",
        "io-lat-2-ms",
        "io-lat-4-ms",
        "io-lat-10-ms",
        "io-lat-20-ms",
        "io-lat-50-ms",
        "io-lat-100-ms",
        "io-lat-250-ms",
        "io-lat-500-ms",
        "io-lat-750-ms",
        "io-lat-1000-ms",
        "io-lat-2000-ms",
        "io-lat-greater"
    };

    private List<TestInfo> mTestCases = null;

    /**
     * Holds info about a job. The job translates into a job in the FIO config file. Contains the
     * job name and a map from keys to values.
     */
    private static class JobInfo {
        public String mJobName = null;
        public Map<String, String> mParameters = new HashMap<>();

        /**
         * Gets the job as a string.
         *
         * @return a string of the job formatted for the config file.
         */
        public String createJob() {
            if (mJobName == null) {
                return "";
            }
            StringBuilder sb = new StringBuilder();
            sb.append(String.format("[%s]\n", mJobName));
            for (Entry<String, String> parameter : mParameters.entrySet()) {
                if (parameter.getValue() == null) {
                    sb.append(String.format("%s\n", parameter.getKey()));
                } else {
                    sb.append(String.format("%s=%s\n", parameter.getKey(), parameter.getValue()));
                }
            }
            return sb.toString();
        }
    }

    /**
     * Holds info about a file used in the benchmark. Because of limitations in FIO on Android, the
     * file needs to be created before the tests are run. Contains the file name and size in kB.
     */
    private static class TestFileInfo {
        public String mFileName = null;
        public int mSize = -1;
    }

    /** Holds info about the perf metric that are cared about for a given job. */
    private static class PerfMetricInfo {
        public enum ResultType {
            STRING,
            INT,
            FLOAT,
            PERCENT;

            String value(String input) {
                switch (this) {
                    case STRING:
                    case INT:
                    case FLOAT:
                        return input;
                    case PERCENT:
                        if (input.length() < 2 || !input.endsWith("%")) {
                            return null;
                        }
                        try {
                            return String.format(
                                    "%f",
                                    Double.parseDouble(input.substring(0, input.length() - 1))
                                            / 100);
                        } catch (NumberFormatException e) {
                            return null;
                        }
                    default:
                        return null;
                }
            }
        }

        public String mJobName = null;
        public String mFieldName = null;
        public String mPostKey = null;
        public ResultType mType = ResultType.STRING;
    }

    /**
     * Holds the info associated with a test.
     *
     * <p>Contains the test name, key, a list of {@link JobInfo}, a set of {@link TestFileInfo}, and
     * a set of {@link PerfMetricInfo}.
     */
    private static class TestInfo {
        public String mTestName = null;
        public String mKey = null;
        public List<JobInfo> mJobs = new LinkedList<>();
        public Set<TestFileInfo> mTestFiles = new HashSet<>();
        public Set<PerfMetricInfo> mPerfMetrics = new HashSet<>();

        /**
         * Gets the config file.
         *
         * @return a string containing the contents of the config file needed to run the benchmark.
         */
        private String createConfig() {
            StringBuilder sb = new StringBuilder();
            for (JobInfo job : mJobs) {
                sb.append(String.format("%s\n", job.createJob()));
            }
            return sb.toString();
        }
    }

    /**
     * Parses the output of the FIO and allows the values to be looked up by job name and property.
     */
    private static class FioParser extends MultiLineReceiver {
        public Map<String, Map<String, String>> mResults = new HashMap<>();

        /**
         * Gets the result for a job and property, or null if the job or the property do not exist.
         *
         * @param job the name of the job.
         * @param property the name of the property. See {@code FIO_RESULT_FIELDS}.
         * @return the fio results for the job and property or null if it does not exist.
         */
        public String getResult(String job, String property) {
            if (!mResults.containsKey(job)) {
                return null;
            }
            return mResults.get(job).get(property);
        }

        /** {@inheritDoc} */
        @Override
        public void processNewLines(String[] lines) {
            for (String line : lines) {
                CLog.d(line);
                String[] fields = line.split(";");
                if (fields.length < FIO_V0_RESULT_FIELDS.length) {
                    continue;
                }
                if (fields.length < FIO_V3_RESULT_FIELDS.length) {
                    Map<String, String> r = new HashMap<>();
                    for (int i = 0; i < FIO_V0_RESULT_FIELDS.length; i++) {
                        r.put(FIO_V0_RESULT_FIELDS[i], fields[i]);
                    }
                    mResults.put(fields[0], r); // Job name is index 0
                } else if ("3".equals(fields[0])) {
                    Map<String, String> r = new HashMap<>();
                    for (int i = 0; i < FIO_V3_RESULT_FIELDS.length; i++) {
                        r.put(FIO_V3_RESULT_FIELDS[i], fields[i]);
                    }
                    mResults.put(fields[2], r); // Job name is index 2
                } else {
                    Assert.fail("Unknown fio terse output version");
                }
            }
        }

        /** {@inheritDoc} */
        @Override
        public boolean isCancelled() {
            return false;
        }
    }

    ITestDevice mTestDevice = null;

    private String mFioDir = null;
    private String mFioBin = null;
    private String mFioConfig = null;

    @Option(
            name = "fio-location",
            description =
                    "The path to the precompiled FIO executable. If "
                            + "unset, try to use fio from the system image.")
    private String mFioLocation = null;

    @Option(name = "tmp-dir", description = "The directory used for interal benchmarks.")
    private String mTmpDir = "/data/tmp/fio";

    @Option(name = "internal-test-dir", description = "The directory used for interal benchmarks.")
    private String mInternalTestDir = "/data/fio/data";

    @Option(name = "media-test-dir", description = "The directory used for media benchmarks.")
    private String mMediaTestDir = "${EXTERNAL_STORAGE}/fio";

    @Option(name = "external-test-dir", description = "The directory used for external benchmarks.")
    private String mExternalTestDir = "${EXTERNAL_STORAGE}/fio";

    @Option(name = "collect-yaffs-logs", description = "Collect yaffs logs before and after tests")
    private Boolean mCollectYaffsLogs = false;

    @Option(name = "run-simple-internal-test", description = "Run the simple internal benchmark.")
    private Boolean mRunSimpleInternalTest = true;

    @Option(
            name = "simple-internal-file-size",
            description = "The file size of the simple internal benchmark in MB.")
    private int mSimpleInternalFileSize = 256;

    @Option(name = "run-simple-external-test", description = "Run the simple external benchmark.")
    private Boolean mRunSimpleExternalTest = false;

    @Option(
            name = "simple-external-file-size",
            description = "The file size of the simple external benchmark in MB.")
    private int mSimpleExternalFileSize = 256;

    @Option(name = "run-storage-internal-test", description = "Run the storage internal benchmark.")
    private Boolean mRunStorageInternalTest = true;

    @Option(
            name = "storage-internal-file-size",
            description = "The file size of the storage internal benchmark in MB.")
    private int mStorageInternalFileSize = 256;

    @Option(
            name = "storage-internal-job-count",
            description = "The number of jobs for the storage internal benchmark.")
    private int mStorageInternalJobCount = 4;

    @Option(name = "run-storage-external-test", description = "Run the storage external benchmark.")
    private Boolean mRunStorageExternalTest = false;

    @Option(
            name = "storage-external-file-size",
            description = "The file size of the storage external benchmark in MB.")
    private int mStorageExternalFileSize = 256;

    @Option(
            name = "storage-external-job-count",
            description = "The number of jobs for the storage external benchmark.")
    private int mStorageExternalJobCount = 4;

    @Option(name = "run-media-server-test", description = "Run the media server benchmark.")
    private Boolean mRunMediaServerTest = false;

    @Option(
            name = "media-server-duration",
            description = "The duration of the media server benchmark in secs.")
    private long mMediaServerDuration = 30;

    @Option(
            name = "media-server-media-file-size",
            description = "The media file size of the media server benchmark in MB.")
    private int mMediaServerMediaFileSize = 256;

    @Option(
            name = "media-server-worker-file-size",
            description = "The worker file size of the media server benchmark in MB.")
    private int mMediaServerWorkerFileSize = 256;

    @Option(
            name = "media-server-worker-job-count",
            description = "The number of worker jobs for the media server benchmark.")
    private int mMediaServerWorkerJobCount = 4;

    @Option(name = "run-media-scanner-test", description = "Run the media scanner benchmark.")
    private Boolean mRunMediaScannerTest = false;

    @Option(
            name = "media-scanner-media-file-size",
            description = "The media file size of the media scanner benchmark in kB.")
    private int mMediaScannerMediaFileSize = 8;

    @Option(
            name = "media-scanner-media-file-count",
            description = "The number of media files to scan.")
    private int mMediaScannerMediaFileCount = 256;

    @Option(
            name = "media-scanner-worker-file-size",
            description = "The worker file size of the media scanner benchmark in MB.")
    private int mMediaScannerWorkerFileSize = 256;

    @Option(
            name = "media-scanner-worker-job-count",
            description = "The number of worker jobs for the media server benchmark.")
    private int mMediaScannerWorkerJobCount = 4;

    @Option(
            name = "key-suffix",
            description = "The suffix to add to the reporting key in order to override the default")
    private String mKeySuffix = null;

    /** Sets up all the benchmarks. */
    private void setupTests() {
        if (mTestCases != null) {
            // assume already set up
            return;
        }

        mTestCases = new LinkedList<>();

        if (mRunSimpleInternalTest) {
            addSimpleTest("read", "sync", true);
            addSimpleTest("write", "sync", true);
            addSimpleTest("randread", "sync", true);
            addSimpleTest("randwrite", "sync", true);
            addSimpleTest("randread", "mmap", true);
            addSimpleTest("randwrite", "mmap", true);
        }

        if (mRunSimpleExternalTest) {
            addSimpleTest("read", "sync", false);
            addSimpleTest("write", "sync", false);
            addSimpleTest("randread", "sync", false);
            addSimpleTest("randwrite", "sync", false);
            addSimpleTest("randread", "mmap", false);
            addSimpleTest("randwrite", "mmap", false);
        }

        if (mRunStorageInternalTest) {
            addStorageTest("read", "sync", true);
            addStorageTest("write", "sync", true);
            addStorageTest("randread", "sync", true);
            addStorageTest("randwrite", "sync", true);
            addStorageTest("randread", "mmap", true);
            addStorageTest("randwrite", "mmap", true);
        }

        if (mRunStorageExternalTest) {
            addStorageTest("read", "sync", false);
            addStorageTest("write", "sync", false);
            addStorageTest("randread", "sync", false);
            addStorageTest("randwrite", "sync", false);
            addStorageTest("randread", "mmap", false);
            addStorageTest("randwrite", "mmap", false);
        }

        if (mRunMediaServerTest) {
            addMediaServerTest("read");
            addMediaServerTest("write");
        }

        if (mRunMediaScannerTest) {
            addMediaScannerTest();
        }
    }

    /**
     * Sets up the simple FIO benchmark.
     *
     * <p>The test consists of a single process reading or writing to a file.
     *
     * @param rw the type of IO pattern. One of {@code read}, {@code write}, {@code randread}, or
     *     {@code randwrite}.
     * @param ioengine defines how the job issues I/O. Such as {@code sync}, {@code vsync}, {@code
     *     mmap}, or {@code cpuio} and others.
     * @param internal whether the test should be run on the internal (/data) partition or the
     *     external partition.
     */
    private void addSimpleTest(String rw, String ioengine, boolean internal) {
        String fileName = "testfile";
        String jobName = "job";

        String directory;
        int fileSize;

        TestInfo t = new TestInfo();
        if (internal) {
            t.mTestName = String.format("SimpleBenchmark-int-%s-%s", ioengine, rw);
            t.mKey = "fio_simple_int_benchmark";
            directory = mInternalTestDir;
            fileSize = mSimpleInternalFileSize;
        } else {
            t.mTestName = String.format("SimpleBenchmark-ext-%s-%s", ioengine, rw);
            t.mKey = "fio_simple_ext_benchmark";
            directory = mExternalTestDir;
            fileSize = mSimpleExternalFileSize;
        }

        TestFileInfo f = new TestFileInfo();
        f.mFileName = new File(directory, fileName).getAbsolutePath();
        f.mSize = fileSize * 1024; // fileSize is in MB but we want it in kB.
        t.mTestFiles.add(f);

        JobInfo j = new JobInfo();
        j.mJobName = jobName;
        j.mParameters.put("directory", directory);
        j.mParameters.put("filename", fileName);
        j.mParameters.put("fsync", "1024");
        j.mParameters.put("ioengine", ioengine);
        j.mParameters.put("rw", rw);
        j.mParameters.put("size", String.format("%dM", fileSize));
        t.mJobs.add(j);

        PerfMetricInfo m = new PerfMetricInfo();
        m.mJobName = jobName;
        if ("sync".equals(ioengine)) {
            m.mPostKey = String.format("%s_bandwidth", rw);
        } else {
            m.mPostKey = String.format("%s_%s_bandwidth", rw, ioengine);
        }
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        if (rw.endsWith("read")) {
            m.mFieldName = "read-bandwidth-mean";
        } else if (rw.endsWith("write")) {
            m.mFieldName = "write-bandwidth-mean";
        }
        t.mPerfMetrics.add(m);

        mTestCases.add(t);
    }

    /**
     * Sets up the storage FIO benchmark.
     *
     * <p>The test consists of several processes reading or writing to a file.
     *
     * @param rw the type of IO pattern. One of {@code read}, {@code write}, {@code randread}, or
     *     {@code randwrite}.
     * @param ioengine defines how the job issues I/O. Such as {@code sync}, {@code vsync}, {@code
     *     mmap}, or {@code cpuio} and others.
     * @param internal whether the test should be run on the internal (/data) partition or the
     *     external partition.
     */
    private void addStorageTest(String rw, String ioengine, boolean internal) {
        String fileName = "testfile";
        String jobName = "workers";

        String directory;
        int fileSize;
        int jobCount;

        TestInfo t = new TestInfo();
        if (internal) {
            t.mTestName = String.format("StorageBenchmark-int-%s-%s", ioengine, rw);
            t.mKey = "fio_storage_int_benchmark";
            directory = mInternalTestDir;
            fileSize = mStorageInternalFileSize;
            jobCount = mStorageInternalJobCount;
        } else {
            t.mTestName = String.format("StorageBenchmark-ext-%s-%s", ioengine, rw);
            t.mKey = "fio_storage_ext_benchmark";
            directory = mExternalTestDir;
            fileSize = mStorageExternalFileSize;
            jobCount = mStorageExternalJobCount;
        }

        TestFileInfo f = new TestFileInfo();
        f.mFileName = new File(directory, fileName).getAbsolutePath();
        f.mSize = fileSize * 1024; // fileSize is in MB but we want it in kB.
        t.mTestFiles.add(f);

        JobInfo j = new JobInfo();
        j.mJobName = jobName;
        j.mParameters.put("directory", directory);
        j.mParameters.put("filename", fileName);
        j.mParameters.put("fsync", "1024");
        j.mParameters.put("group_reporting", null);
        j.mParameters.put("ioengine", ioengine);
        j.mParameters.put("new_group", null);
        j.mParameters.put("numjobs", String.format("%d", jobCount));
        j.mParameters.put("rw", rw);
        j.mParameters.put("size", String.format("%dM", fileSize));
        t.mJobs.add(j);

        PerfMetricInfo m = new PerfMetricInfo();
        m.mJobName = jobName;
        if ("sync".equals(ioengine)) {
            m.mPostKey = String.format("%s_bandwidth", rw);
        } else {
            m.mPostKey = String.format("%s_%s_bandwidth", rw, ioengine);
        }
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        if (rw.endsWith("read")) {
            m.mFieldName = "read-bandwidth-mean";
        } else if (rw.endsWith("write")) {
            m.mFieldName = "write-bandwidth-mean";
        }
        t.mPerfMetrics.add(m);

        m = new PerfMetricInfo();
        m.mJobName = jobName;
        if ("sync".equals(ioengine)) {
            m.mPostKey = String.format("%s_latency", rw);
        } else {
            m.mPostKey = String.format("%s_%s_latency", rw, ioengine);
        }
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        if (rw.endsWith("read")) {
            m.mFieldName = "read-clat-mean";
        } else if (rw.endsWith("write")) {
            m.mFieldName = "write-clat-mean";
        }
        t.mPerfMetrics.add(m);

        mTestCases.add(t);
    }

    /**
     * Sets up the media server benchmark.
     *
     * <p>The test consists of a single process at a higher priority reading or writing to a file
     * while several other worker processes read and write to a different file.
     *
     * @param rw the type of IO pattern. One of {@code read}, {@code write}
     */
    private void addMediaServerTest(String rw) {
        String mediaJob = "media-server";
        String mediaFile = "mediafile";
        String workerJob = "workers";
        String workerFile = "workerfile";

        TestInfo t = new TestInfo();
        t.mTestName = String.format("MediaServerBenchmark-%s", rw);
        t.mKey = "fio_media_server_benchmark";

        TestFileInfo f = new TestFileInfo();
        f.mFileName = new File(mMediaTestDir, mediaFile).getAbsolutePath();
        f.mSize = mMediaServerMediaFileSize * 1024; // File size is in MB but we want it in kB.
        t.mTestFiles.add(f);

        f = new TestFileInfo();
        f.mFileName = new File(mMediaTestDir, workerFile).getAbsolutePath();
        f.mSize = mMediaServerWorkerFileSize * 1024; // File size is in MB but we want it in kB.
        t.mTestFiles.add(f);

        JobInfo j = new JobInfo();
        j.mJobName = "global";
        j.mParameters.put("directory", mMediaTestDir);
        j.mParameters.put("fsync", "1024");
        j.mParameters.put("ioengine", "sync");
        j.mParameters.put("runtime", String.format("%d", mMediaServerDuration));
        j.mParameters.put("time_based", null);
        t.mJobs.add(j);

        j = new JobInfo();
        j.mJobName = mediaJob;
        j.mParameters.put("filename", mediaFile);
        j.mParameters.put("iodepth", "32");
        j.mParameters.put("nice", "-16");
        j.mParameters.put("rate", "6m");
        j.mParameters.put("rw", rw);
        j.mParameters.put("size", String.format("%dM", mMediaServerMediaFileSize));
        t.mJobs.add(j);

        j = new JobInfo();
        j.mJobName = workerJob;
        j.mParameters.put("filename", workerFile);
        j.mParameters.put("group_reporting", null);
        j.mParameters.put("new_group", null);
        j.mParameters.put("nice", "0");
        j.mParameters.put("numjobs", String.format("%d", mMediaServerWorkerJobCount));
        j.mParameters.put("rw", "randrw");
        j.mParameters.put("size", String.format("%dM", mMediaServerWorkerFileSize));
        t.mJobs.add(j);

        PerfMetricInfo m = new PerfMetricInfo();
        m.mJobName = mediaJob;
        m.mPostKey = String.format("%s_media_bandwidth", rw);
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        if (rw.endsWith("read")) {
            m.mFieldName = "read-bandwidth-mean";
        } else if (rw.endsWith("write")) {
            m.mFieldName = "write-bandwidth-mean";
        }
        t.mPerfMetrics.add(m);

        m = new PerfMetricInfo();
        m.mJobName = mediaJob;
        m.mPostKey = String.format("%s_media_latency", rw);
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        if (rw.endsWith("read")) {
            m.mFieldName = "read-clat-mean";
        } else if (rw.endsWith("write")) {
            m.mFieldName = "write-clat-mean";
        }
        t.mPerfMetrics.add(m);

        m = new PerfMetricInfo();
        m.mJobName = workerJob;
        m.mPostKey = String.format("%s_workers_read_bandwidth", rw);
        m.mFieldName = "read-bandwidth-mean";
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        t.mPerfMetrics.add(m);

        m = new PerfMetricInfo();
        m.mJobName = workerJob;
        m.mPostKey = String.format("%s_workers_write_bandwidth", rw);
        m.mFieldName = "write-bandwidth-mean";
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        t.mPerfMetrics.add(m);

        mTestCases.add(t);
    }

    /**
     * Sets up the media scanner benchmark.
     *
     * <p>The test consists of a single process reading several small files while several other
     * worker processes read and write to a different file.
     */
    private void addMediaScannerTest() {
        String mediaJob = "media-server";
        String mediaFile = "mediafile.%d";
        String workerJob = "workers";
        String workerFile = "workerfile";

        TestInfo t = new TestInfo();
        t.mTestName = "MediaScannerBenchmark";
        t.mKey = "fio_media_scanner_benchmark";

        TestFileInfo f;
        for (int i = 0; i < mMediaScannerMediaFileCount; i++) {
            f = new TestFileInfo();
            f.mFileName = new File(mMediaTestDir, String.format(mediaFile, i)).getAbsolutePath();
            f.mSize = mMediaScannerMediaFileSize; // File size is already in kB so do nothing.
            t.mTestFiles.add(f);
        }

        f = new TestFileInfo();
        f.mFileName = new File(mMediaTestDir, workerFile).getAbsolutePath();
        f.mSize = mMediaScannerWorkerFileSize * 1024; // File size is in MB but we want it in kB.
        t.mTestFiles.add(f);

        JobInfo j = new JobInfo();
        j.mJobName = "global";
        j.mParameters.put("directory", mMediaTestDir);
        j.mParameters.put("fsync", "1024");
        j.mParameters.put("ioengine", "sync");
        t.mJobs.add(j);

        j = new JobInfo();
        j.mJobName = mediaJob;
        StringBuilder fileNames = new StringBuilder();
        fileNames.append(String.format(mediaFile, 0));
        for (int i = 1; i < mMediaScannerMediaFileCount; i++) {
            fileNames.append(String.format(":%s", String.format(mediaFile, i)));
        }
        j.mParameters.put("filename", fileNames.toString());
        j.mParameters.put("exitall", null);
        j.mParameters.put("openfiles", "4");
        j.mParameters.put("rw", "read");
        t.mJobs.add(j);

        j = new JobInfo();
        j.mJobName = workerJob;
        j.mParameters.put("filename", workerFile);
        j.mParameters.put("group_reporting", null);
        j.mParameters.put("new_group", null);
        j.mParameters.put("numjobs", String.format("%d", mMediaScannerWorkerJobCount));
        j.mParameters.put("rw", "randrw");
        j.mParameters.put("size", String.format("%dM", mMediaScannerWorkerFileSize));
        t.mJobs.add(j);

        PerfMetricInfo m = new PerfMetricInfo();
        m.mJobName = mediaJob;
        m.mPostKey = "media_bandwidth";
        m.mFieldName = "read-bandwidth-mean";
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        t.mPerfMetrics.add(m);

        m = new PerfMetricInfo();
        m.mJobName = mediaJob;
        m.mPostKey = "media_latency";
        m.mFieldName = "read-clat-mean";
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        t.mPerfMetrics.add(m);

        m = new PerfMetricInfo();
        m.mJobName = workerJob;
        m.mPostKey = "workers_read_bandwidth";
        m.mFieldName = "read-bandwidth-mean";
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        t.mPerfMetrics.add(m);

        m = new PerfMetricInfo();
        m.mJobName = workerJob;
        m.mPostKey = "workers_write_bandwidth";
        m.mFieldName = "write-bandwidth-mean";
        m.mType = PerfMetricInfo.ResultType.FLOAT;
        t.mPerfMetrics.add(m);

        mTestCases.add(t);
    }

    /**
     * Creates the directories needed to run FIO, pushes the FIO executable to the device, and stops
     * the runtime.
     *
     * @throws DeviceNotAvailableException if the device is not available.
     */
    private void setupDevice() throws DeviceNotAvailableException {
        mTestDevice.executeShellCommand("stop");
        mTestDevice.executeShellCommand(String.format("mkdir -p %s", mFioDir));
        mTestDevice.executeShellCommand(String.format("mkdir -p %s", mTmpDir));
        mTestDevice.executeShellCommand(String.format("mkdir -p %s", mInternalTestDir));
        mTestDevice.executeShellCommand(String.format("mkdir -p %s", mMediaTestDir));
        if (mExternalTestDir != null) {
            mTestDevice.executeShellCommand(String.format("mkdir -p %s", mExternalTestDir));
        }
        if (mFioLocation != null) {
            mTestDevice.pushFile(new File(mFioLocation), mFioBin);
            mTestDevice.executeShellCommand(String.format("chmod 755 %s", mFioBin));
        }
    }

    /**
     * Reverses the actions of {@link #setDevice(ITestDevice)}.
     *
     * @throws DeviceNotAvailableException If the device is not available.
     */
    private void cleanupDevice() throws DeviceNotAvailableException {
        if (mExternalTestDir != null) {
            mTestDevice.executeShellCommand(String.format("rm -r %s", mExternalTestDir));
        }
        mTestDevice.executeShellCommand(String.format("rm -r %s", mMediaTestDir));
        mTestDevice.executeShellCommand(String.format("rm -r %s", mInternalTestDir));
        mTestDevice.executeShellCommand(String.format("rm -r %s", mTmpDir));
        mTestDevice.executeShellCommand(String.format("rm -r %s", mFioDir));
        mTestDevice.executeShellCommand("start");
        mTestDevice.waitForDeviceAvailable();
    }

    /**
     * Runs a single test, including creating the test files, clearing the cache, collecting before
     * and after files, running the benchmark, and reporting the results.
     *
     * @param test the benchmark.
     * @param listener the ITestInvocationListener
     * @throws DeviceNotAvailableException if the device is not available.
     */
    private void runTest(TestInfo test, ITestInvocationListener listener)
            throws DeviceNotAvailableException {
        CLog.i("Running %s benchmark", test.mTestName);
        mTestDevice.executeShellCommand(String.format("rm -r %s/*", mTmpDir));
        mTestDevice.executeShellCommand(String.format("rm -r %s/*", mInternalTestDir));
        mTestDevice.executeShellCommand(String.format("rm -r %s/*", mMediaTestDir));
        if (mExternalTestDir != null) {
            mTestDevice.executeShellCommand(String.format("rm -r %s/*", mExternalTestDir));
        }

        for (TestFileInfo file : test.mTestFiles) {
            CLog.v("Creating file: %s, size: %dkB", file.mFileName, file.mSize);
            String cmd =
                    String.format(
                            "dd if=/dev/urandom of=%s bs=1024 count=%d",
                            file.mFileName, file.mSize);
            int timeout = file.mSize * 2 * 1000; // Timeout is 2 seconds per kB.
            mTestDevice.executeShellCommand(
                    cmd, new NullOutputReceiver(), timeout, TimeUnit.MILLISECONDS, 2);
        }

        CLog.i("Creating config");
        CLog.d("Config file:\n%s", test.createConfig());
        mTestDevice.pushString(test.createConfig(), mFioConfig);

        CLog.i("Dropping cache");
        mTestDevice.executeShellCommand("echo 3 > /proc/sys/vm/drop_caches");

        collectLogs(test, listener, "before");

        CLog.i("Running test");
        FioParser output = new FioParser();
        // Run FIO with a timeout of 1 hour.
        mTestDevice.executeShellCommand(
                String.format("%s --minimal %s", mFioBin, mFioConfig),
                output,
                60 * 60 * 1000,
                TimeUnit.MILLISECONDS,
                2);

        collectLogs(test, listener, "after");

        // Report metrics
        Map<String, String> metrics = new HashMap<>();
        String key = mKeySuffix == null ? test.mKey : test.mKey + mKeySuffix;

        listener.testRunStarted(key, 0);
        for (PerfMetricInfo m : test.mPerfMetrics) {
            if (!output.mResults.containsKey(m.mJobName)) {
                CLog.w("Job name %s was not found in the results", m.mJobName);
                continue;
            }

            String value = output.getResult(m.mJobName, m.mFieldName);
            if (value != null) {
                metrics.put(m.mPostKey, m.mType.value(value));
            } else {
                CLog.w("%s was not in results for the job %s", m.mFieldName, m.mJobName);
            }
        }

        CLog.d("About to report metrics to %s: %s", key, metrics);
        listener.testRunEnded(0, TfMetricProtoUtil.upgradeConvert(metrics));
    }

    private void collectLogs(TestInfo testInfo, ITestInvocationListener listener, String descriptor)
            throws DeviceNotAvailableException {
        if (mCollectYaffsLogs && mTestDevice.doesFileExist("/proc/yaffs")) {
            logFile(
                    "/proc/yaffs",
                    String.format("%s-yaffs-%s", testInfo.mTestName, descriptor),
                    mTestDevice,
                    listener);
        }
    }

    private void logFile(
            String remoteFileName,
            String localFileName,
            ITestDevice testDevice,
            ITestInvocationListener listener)
            throws DeviceNotAvailableException {
        File outputFile = null;
        InputStreamSource outputSource = null;
        try {
            outputFile = testDevice.pullFile(remoteFileName);
            if (outputFile != null) {
                CLog.d("Sending %d byte file %s to logosphere!", outputFile.length(), outputFile);
                outputSource = new FileInputStreamSource(outputFile);
                listener.testLog(localFileName, LogDataType.TEXT, outputSource);
            }
        } finally {
            FileUtil.deleteFile(outputFile);
            StreamUtil.cancel(outputSource);
        }
    }

    /** {@inheritDoc} */
    @Override
    public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
        Assert.assertNotNull(mTestDevice);

        mFioDir = new File(mTestDevice.getMountPoint(IDevice.MNT_DATA), "fio").getAbsolutePath();
        if (mFioLocation != null) {
            mFioBin = new File(mFioDir, "fio").getAbsolutePath();
        } else {
            mFioBin = "fio";
        }
        mFioConfig = new File(mFioDir, "config.fio").getAbsolutePath();

        setupTests();
        setupDevice();

        for (TestInfo test : mTestCases) {
            runTest(test, listener);
        }

        cleanupDevice();
    }

    /** {@inheritDoc} */
    @Override
    public void setDevice(ITestDevice device) {
        mTestDevice = device;
    }

    /** {@inheritDoc} */
    @Override
    public ITestDevice getDevice() {
        return mTestDevice;
    }

    /** A meta-test to ensure that the bits of FioBenchmarkTest are working properly. */
    public static class MetaTest extends TestCase {

        /** Test that {@link JobInfo#createJob()} properly formats a job. */
        public void testCreateJob() {
            JobInfo j = new JobInfo();
            assertEquals("", j.createJob());
            j.mJobName = "job";
            assertEquals("[job]\n", j.createJob());
            j.mParameters.put("param1", null);
            j.mParameters.put("param2", "value");
            String[] lines = j.createJob().split("\n");
            assertEquals(3, lines.length);
            assertEquals("[job]", lines[0]);
            Set<String> params = new HashSet<>(2);
            params.add(lines[1]);
            params.add(lines[2]);
            assertTrue(params.contains("param1"));
            assertTrue(params.contains("param2=value"));
        }

        /** Test that {@link TestInfo#createConfig()} properly formats a config. */
        public void testCreateConfig() {
            TestInfo t = new TestInfo();
            JobInfo j = new JobInfo();
            j.mJobName = "job1";
            j.mParameters.put("param1", "value1");
            t.mJobs.add(j);

            j = new JobInfo();
            j.mJobName = "job2";
            j.mParameters.put("param2", "value2");
            t.mJobs.add(j);

            j = new JobInfo();
            j.mJobName = "job3";
            j.mParameters.put("param3", "value3");
            t.mJobs.add(j);

            assertEquals(
                    "[job1]\nparam1=value1\n\n"
                            + "[job2]\nparam2=value2\n\n"
                            + "[job3]\nparam3=value3\n\n",
                    t.createConfig());
        }

        /**
         * Test that output lines are parsed correctly by the FioParser, invalid lines are ignored,
         * and that the various fields are accessible with {@link FioParser#getResult(String,
         * String)}.
         */
        public void testFioParser() {
            String[] lines = new String[4];
            // We build the lines up as follows (assuming FIO_RESULTS_FIELDS.length == 58):
            // 0;3;6;...;171
            // 1;4;7;...;172
            // 2;5;8;...;173
            for (int i = 0; i < 3; i++) {
                StringBuilder sb = new StringBuilder();
                sb.append(i);
                for (int j = 1; j < FIO_V0_RESULT_FIELDS.length; j++) {
                    sb.append(";");
                    sb.append(j * 3 + i);
                }
                lines[i] = sb.toString();
            }
            // A line may have an optional description on the end which we don't care about, make
            // sure it still parses.
            lines[2] = lines[2] += ";description";
            // Make sure an invalid output line does not parse.
            lines[3] = "invalid";

            FioParser p = new FioParser();
            p.processNewLines(lines);

            for (int i = 0; i < 3; i++) {
                for (int j = 0; j < FIO_V0_RESULT_FIELDS.length; j++) {
                    assertEquals(
                            String.format("job=%d, field=%s", i, FIO_V0_RESULT_FIELDS[j]),
                            String.format("%d", j * 3 + i),
                            p.getResult(String.format("%d", i), FIO_V0_RESULT_FIELDS[j]));
                }
            }
            assertNull(p.getResult("missing", "jobname"));
            assertNull(p.getResult("invalid", "jobname"));
            assertNull(p.getResult("0", "missing"));
        }

        /**
         * Test that {@link PerfMetricInfo.ResultType#value(String)} correctly transforms strings
         * based on the result type.
         */
        public void testResultTypeValue() {
            assertEquals("test", PerfMetricInfo.ResultType.STRING.value("test"));
            assertEquals("1", PerfMetricInfo.ResultType.INT.value("1"));
            assertEquals("3.14159", PerfMetricInfo.ResultType.FLOAT.value("3.14159"));
            assertEquals(
                    String.format("%f", 0.34567),
                    PerfMetricInfo.ResultType.PERCENT.value("34.567%"));
            assertNull(PerfMetricInfo.ResultType.PERCENT.value(""));
            assertNull(PerfMetricInfo.ResultType.PERCENT.value("34.567"));
            assertNull(PerfMetricInfo.ResultType.PERCENT.value("test%"));
        }
    }
}