1// Copyright 2023 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5package main 6 7import ( 8 "context" 9 "fmt" 10 "io" 11 "os" 12 "path/filepath" 13 "strings" 14 "testing" 15 16 "github.com/stretchr/testify/assert" 17 "github.com/stretchr/testify/require" 18 "go.skia.org/infra/go/exec" 19) 20 21func TestParseTestRunnerExtraArgsFlag(t *testing.T) { 22 test := func(name, rawFlag, expectedTestRunnerExtraArgs, expectedDeviceSpecificBazelConfig string) { 23 t.Run(name, func(t *testing.T) { 24 testRunnerExtraArgs, deviceSpecificBazelConfig := parseTestRunnerExtraArgsFlag(rawFlag) 25 assert.Equal(t, expectedTestRunnerExtraArgs, testRunnerExtraArgs) 26 assert.Equal(t, expectedDeviceSpecificBazelConfig, deviceSpecificBazelConfig) 27 }) 28 } 29 30 test( 31 "no device-specific config", 32 "--foo aaa bbb --bar ccc", 33 "--foo aaa bbb --bar ccc", 34 "") 35 36 test( 37 "device-specific config as single flag", 38 "--foo aaa bbb --device-specific-bazel-config=SomeDevice --bar ccc", 39 "--foo aaa bbb --bar ccc", 40 "SomeDevice") 41 42 test( 43 "device-specific config as single flag with extra spaces", 44 "--foo aaa bbb --device-specific-bazel-config=SomeDevice --bar ccc", 45 "--foo aaa bbb --bar ccc", 46 "SomeDevice") 47 48 test("device-specific config as two flags", 49 "--foo aaa bbb --device-specific-bazel-config SomeDevice --bar ccc", 50 "--foo aaa bbb --bar ccc", 51 "SomeDevice") 52 53 test("device-specific config as two flags with extra spaces", 54 "--foo aaa bbb --device-specific-bazel-config SomeDevice --bar ccc", 55 "--foo aaa bbb --bar ccc", 56 "SomeDevice") 57} 58 59func TestRunTest_GMOrUnitTest_Success(t *testing.T) { 60 outputDir := t.TempDir() 61 62 // The below adb commands prepare the device to run a GM or unit test, which the test runner 63 // refers to as "performance tests". 64 // 65 // Unlike benchmark tests, performance tests are not sensitive to variations in the device's 66 // performance. For example, it is OK for the device to run as fast as possible and potentially 67 // throttle the CPU if it runs too hot. Thus, performance tests use a CPU governor that optimizes 68 // for speed. They also ensure that all CPU cores are enabled, which is important because some of 69 // them could have been disabled by a previous benchmark test. 70 commandCollector, assertCollectedAllExpectedCommands := makeCommandCollector(t, 71 // Clean up device from previous test run. 72 cmdAndArgs("adb", "shell", "su", "root", "rm", "-rf", "/sdcard/bazel-adb-test.tar.gz", "/data/bazel-adb-test", "/sdcard/bazel-adb-test-output-dir"), 73 // Enable CPU cores that might have been disabled by an earlier benchmark test. 74 cmdAndArgs("adb", "root"), 75 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu4/online").withStdout("0\n"), 76 cmdAndArgs("adb", "shell", `echo "1" > /sys/devices/system/cpu/cpu4/online`), 77 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu4/online").withStdout("1\n"), 78 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu5/online").withStdout("0\n"), 79 cmdAndArgs("adb", "shell", `echo "1" > /sys/devices/system/cpu/cpu5/online`), 80 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu5/online").withStdout("1\n"), 81 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu6/online").withStdout("0\n"), 82 cmdAndArgs("adb", "shell", `echo "1" > /sys/devices/system/cpu/cpu6/online`), 83 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu6/online").withStdout("1\n"), 84 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu7/online").withStdout("0\n"), 85 cmdAndArgs("adb", "shell", `echo "1" > /sys/devices/system/cpu/cpu7/online`), 86 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu7/online").withStdout("1\n"), 87 // Set CPU governor. 88 cmdAndArgs("adb", "shell", `echo "performance" > /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor`), 89 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor").withStdout("performance\n"), 90 // Push and extract test. 91 cmdAndArgs("adb", "push", "path/to/archive.tar.gz", "/sdcard/bazel-adb-test.tar.gz"), 92 cmdAndArgs("adb", "shell", "su", "root", "mkdir", "-p", "/data/bazel-adb-test"), 93 cmdAndArgs("adb", "shell", "su", "root", "tar", "xzvf", "/sdcard/bazel-adb-test.tar.gz", "-C", "/data/bazel-adb-test"), 94 // Create output directory. 95 cmdAndArgs("adb", "shell", "su", "root", "mkdir", "-p", "/sdcard/bazel-adb-test-output-dir"), 96 // Run test. 97 cmdAndArgs("adb", "shell", "su", "root").withStdin( 98 "cd /data/bazel-adb-test && "+ 99 "ADB_TEST_OUTPUT_DIR=/sdcard/bazel-adb-test-output-dir "+ 100 "path/to/testrunner --test-runner-fake-arg value", 101 ), 102 // Pull test outputs. 103 cmdAndArgs("adb", "pull", "/sdcard/bazel-adb-test-output-dir", outputDir), 104 // Reboot device. 105 cmdAndArgs("adb", "reboot"), 106 cmdAndArgs("adb", "wait-for-device"), 107 // Clean up device. 108 cmdAndArgs("adb", "shell", "su", "root", "rm", "-rf", "/sdcard/bazel-adb-test.tar.gz", "/data/bazel-adb-test", "/sdcard/bazel-adb-test-output-dir"), 109 ) 110 111 // Populate the output dir with the results of pulling output files from the device. 112 require.NoError(t, os.Mkdir(filepath.Join(outputDir, "bazel-adb-test-output-dir"), 0755)) 113 require.NoError(t, os.WriteFile(filepath.Join(outputDir, "bazel-adb-test-output-dir", "fake-output-1.txt"), []byte("Contents do not matter"), 0644)) 114 require.NoError(t, os.WriteFile(filepath.Join(outputDir, "bazel-adb-test-output-dir", "fake-output-2.txt"), []byte("Contents do not matter"), 0644)) 115 116 ctx := exec.NewContext(context.Background(), commandCollector.Run) 117 // Pixel 6 was chosen because it exercises the behavior where certain CPU cores are re-enabled 118 // after potentially being disabled by an earlier benchmark test. 119 err := runTest(ctx, "Pixel6", false /* =isBenchmarkTest */, "path/to/archive.tar.gz", "path/to/testrunner", "--test-runner-fake-arg value", outputDir) 120 require.NoError(t, err) 121 122 assertCollectedAllExpectedCommands() 123 124 // Assert that the expected files were produced. 125 entries, err := os.ReadDir(outputDir) 126 require.NoError(t, err) 127 require.Len(t, entries, 2) 128 assert.Equal(t, "fake-output-1.txt", entries[0].Name()) 129 assert.False(t, entries[0].IsDir()) 130 assert.Equal(t, "fake-output-2.txt", entries[1].Name()) 131 assert.False(t, entries[1].IsDir()) 132} 133 134func TestRunTest_BenchmarkTest_Success(t *testing.T) { 135 outputDir := t.TempDir() 136 137 // The below adb commands prepare the device to run a benchmark test. 138 // 139 // For devices with multiple kinds of CPU cores (e.g. big.LITTLE), it's important to only enable 140 // one kind of CPU, or the test could alternate between CPU core kinds across subsequent test 141 // runs and produce noisy benchmarks. It's also important to use a CPU governor that will prevent 142 // the cores from throttling, which can also cause noisy results. 143 commandCollector, assertCollectedAllExpectedCommands := makeCommandCollector(t, 144 // Clean up device from previous test run. 145 cmdAndArgs("adb", "shell", "su", "root", "rm", "-rf", "/sdcard/bazel-adb-test.tar.gz", "/data/bazel-adb-test", "/sdcard/bazel-adb-test-output-dir"), 146 // Set CPU governor. 147 cmdAndArgs("adb", "root"), 148 cmdAndArgs("adb", "shell", `echo "powersave" > /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor`), 149 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor").withStdout("powersave\n"), 150 // Disable CPUs. 151 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu4/online").withStdout("1\n"), 152 cmdAndArgs("adb", "shell", `echo "0" > /sys/devices/system/cpu/cpu4/online`), 153 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu4/online").withStdout("0\n"), 154 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu5/online").withStdout("1\n"), 155 cmdAndArgs("adb", "shell", `echo "0" > /sys/devices/system/cpu/cpu5/online`), 156 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu5/online").withStdout("0\n"), 157 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu6/online").withStdout("1\n"), 158 cmdAndArgs("adb", "shell", `echo "0" > /sys/devices/system/cpu/cpu6/online`), 159 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu6/online").withStdout("0\n"), 160 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu7/online").withStdout("1\n"), 161 cmdAndArgs("adb", "shell", `echo "0" > /sys/devices/system/cpu/cpu7/online`), 162 cmdAndArgs("adb", "shell", "cat /sys/devices/system/cpu/cpu7/online").withStdout("0\n"), 163 // Push and extract test. 164 cmdAndArgs("adb", "push", "path/to/archive.tar.gz", "/sdcard/bazel-adb-test.tar.gz"), 165 cmdAndArgs("adb", "shell", "su", "root", "mkdir", "-p", "/data/bazel-adb-test"), 166 cmdAndArgs("adb", "shell", "su", "root", "tar", "xzvf", "/sdcard/bazel-adb-test.tar.gz", "-C", "/data/bazel-adb-test"), 167 // Create output directory. 168 cmdAndArgs("adb", "shell", "su", "root", "mkdir", "-p", "/sdcard/bazel-adb-test-output-dir"), 169 // Run test. 170 cmdAndArgs("adb", "shell", "su", "root").withStdin( 171 "cd /data/bazel-adb-test && "+ 172 "ADB_TEST_OUTPUT_DIR=/sdcard/bazel-adb-test-output-dir "+ 173 "path/to/testrunner --test-runner-fake-arg value", 174 ), 175 // Pull test outputs. 176 cmdAndArgs("adb", "pull", "/sdcard/bazel-adb-test-output-dir", outputDir), 177 // Reboot device. 178 cmdAndArgs("adb", "reboot"), 179 cmdAndArgs("adb", "wait-for-device"), 180 // Clean up device. 181 cmdAndArgs("adb", "shell", "su", "root", "rm", "-rf", "/sdcard/bazel-adb-test.tar.gz", "/data/bazel-adb-test", "/sdcard/bazel-adb-test-output-dir"), 182 ) 183 184 // Populate the output dir with the results of pulling output files from the device. 185 require.NoError(t, os.Mkdir(filepath.Join(outputDir, "bazel-adb-test-output-dir"), 0755)) 186 require.NoError(t, os.WriteFile(filepath.Join(outputDir, "bazel-adb-test-output-dir", "fake-output-1.txt"), []byte("Contents do not matter"), 0644)) 187 require.NoError(t, os.WriteFile(filepath.Join(outputDir, "bazel-adb-test-output-dir", "fake-output-2.txt"), []byte("Contents do not matter"), 0644)) 188 189 ctx := exec.NewContext(context.Background(), commandCollector.Run) 190 // Pixel 6 was chosen because it exercises the behavior where certain CPU cores are disabled for 191 // benchmark tests. 192 err := runTest(ctx, "Pixel6", true /* =isBenchmarkTest */, "path/to/archive.tar.gz", "path/to/testrunner", "--test-runner-fake-arg value", outputDir) 193 require.NoError(t, err) 194 195 assertCollectedAllExpectedCommands() 196 197 // Assert that the expected files were produced. 198 entries, err := os.ReadDir(outputDir) 199 require.NoError(t, err) 200 require.Len(t, entries, 2) 201 assert.Equal(t, "fake-output-1.txt", entries[0].Name()) 202 assert.False(t, entries[0].IsDir()) 203 assert.Equal(t, "fake-output-2.txt", entries[1].Name()) 204 assert.False(t, entries[1].IsDir()) 205} 206 207// commandAndResult represents and expected command and its mocked output. 208type commandAndResult struct { 209 // cmdAndArgs is the expected command and args 210 cmdAndArgs []string 211 212 // stdin is the expected standard input, if any. 213 stdin string 214 215 // fakeStdout is the command's simulated fakeStdout. 216 fakeStdout string 217} 218 219// withStdin sets a commandAndResult's stdin and returns the resulting struct. 220func (c commandAndResult) withStdin(stdin string) commandAndResult { 221 c.stdin = stdin 222 return c 223} 224 225// withStdout sets a commandAndResult's stdout and returns the resulting struct. 226func (c commandAndResult) withStdout(stdout string) commandAndResult { 227 c.fakeStdout = stdout 228 return c 229} 230 231// cmdAndArgs returns a commandAndResult with the given command and arguments. 232func cmdAndArgs(cmd string, args ...string) commandAndResult { 233 return commandAndResult{ 234 cmdAndArgs: append([]string{cmd}, args...), 235 } 236} 237 238// makeCommandCollector takes a list of expected commands and returns a CommandCollector that 239// emulates the commands in order, and produces a test failure if any of the commands it collects 240// do not match the expected commands. It also returns a function that asserst that all expected 241// commands were collected. 242func makeCommandCollector(t *testing.T, expectedCommands ...commandAndResult) (*exec.CommandCollector, func()) { 243 collector := &exec.CommandCollector{} 244 curCmdIdx := 0 245 246 collector.SetDelegateRun(func(ctx context.Context, cmd *exec.Command) error { 247 defer func() { curCmdIdx++ }() 248 249 // We assert, rather than require, because the latter would mask any subsequent errors in the 250 // test. 251 assert.Less(t, curCmdIdx, len(expectedCommands), "Command and args: %q", strings.Join(append([]string{cmd.Name}, cmd.Args...), " ")) 252 if curCmdIdx >= len(expectedCommands) { 253 return nil 254 } 255 256 assert.Equal(t, expectedCommands[curCmdIdx].cmdAndArgs[0], cmd.Name, "Command with index %d", curCmdIdx) 257 assert.EqualValues(t, expectedCommands[curCmdIdx].cmdAndArgs[1:], cmd.Args, "Command with index %d", curCmdIdx) 258 if expectedCommands[curCmdIdx].stdin != "" { 259 bytes, err := io.ReadAll(cmd.Stdin) 260 require.NoError(t, err) 261 assert.Equal(t, expectedCommands[curCmdIdx].stdin, string(bytes)) 262 } 263 264 _, err := fmt.Fprintf(cmd.CombinedOutput, expectedCommands[curCmdIdx].fakeStdout) 265 require.NoError(t, err) 266 return nil 267 }) 268 269 assertCollectedAllExpectedCommands := func() { 270 assert.Equal(t, curCmdIdx, len(expectedCommands), "Number of collected commands does not match the number of expected commands") 271 } 272 273 return collector, assertCollectedAllExpectedCommands 274} 275