1/* 2 * Copyright 2018 Google Inc. 3 * 4 * Use of this source code is governed by a BSD-style license that can be 5 * found in the LICENSE file. 6 */ 7 8package main 9 10import ( 11 "bytes" 12 "encoding/json" 13 "flag" 14 "fmt" 15 "net/http" 16 "os" 17 "os/exec" 18 "sort" 19 "strconv" 20 "strings" 21 "syscall" 22 "time" 23 24 gstorage "google.golang.org/api/storage/v1" 25 26 "go.skia.org/infra/go/auth" 27 "go.skia.org/infra/go/common" 28 "go.skia.org/infra/go/sklog" 29 "go.skia.org/infra/go/util" 30 "go.skia.org/infra/golden/go/tsuite" 31) 32 33// TODO(stephana): Convert the hard coded whitelist to a command line flag that 34// loads a file with the whitelisted devices and versions. Make sure to include 35// human readable names for the devices. 36 37var ( 38 // WHITELIST_DEV_IDS contains a mapping from the device id to the list of 39 // Android API versions that we should run agains. Usually this will be the 40 // latest version. To see available devices and version run with 41 // --dryrun flag or run '$ gcloud firebase test android models list' 42 43 WHITELIST_DEV_IDS = map[string][]string{ 44 "A0001": {"22"}, 45 // "E5803": {"22"}, deprecated 46 // "F5121": {"23"}, deprecated 47 "G8142": {"25"}, 48 "HWMHA": {"24"}, 49 "SH-04H": {"23"}, 50 "athene": {"23"}, 51 "athene_f": {"23"}, 52 "hammerhead": {"23"}, 53 "harpia": {"23"}, 54 "hero2lte": {"23"}, 55 "herolte": {"24"}, 56 "j1acevelte": {"22"}, 57 "j5lte": {"23"}, 58 "j7xelte": {"23"}, 59 "lucye": {"24"}, 60 // "mako": {"22"}, deprecated 61 "osprey_umts": {"22"}, 62 // "p1": {"22"}, deprecated 63 "sailfish": {"26"}, 64 "shamu": {"23"}, 65 "trelte": {"22"}, 66 "zeroflte": {"22"}, 67 "zerolte": {"22"}, 68 } 69) 70 71const ( 72 META_DATA_FILENAME = "meta.json" 73) 74 75// Command line flags. 76var ( 77 serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.") 78 dryRun = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.") 79 minAPIVersion = flag.Int("min_api", 22, "Minimum API version required by device.") 80 maxAPIVersion = flag.Int("max_api", 23, "Maximum API version required by device.") 81) 82 83const ( 84 RUN_TESTS_TEMPLATE = `gcloud beta firebase test android run 85 --type=game-loop 86 --app=%s 87 --results-bucket=%s 88 --results-dir=%s 89 --directories-to-pull=/sdcard/Android/data/org.skia.skqp 90 --timeout 30m 91 %s 92` 93 MODEL_VERSION_TMPL = "--device model=%s,version=%s,orientation=portrait" 94 RESULT_BUCKET = "skia-firebase-test-lab" 95 RESULT_DIR_TMPL = "testruns/%s/%s" 96 RUN_ID_TMPL = "testrun-%d" 97 CMD_AVAILABE_DEVICES = "gcloud firebase test android models list --format json" 98) 99 100func main() { 101 common.Init() 102 103 // Get the apk. 104 args := flag.Args() 105 apk_path := args[0] 106 107 // Make sure we can get the service account client. 108 client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email") 109 if err != nil { 110 sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err) 111 } 112 113 // Get list of all available devices. 114 devices, ignoredDevices, err := getAvailableDevices(WHITELIST_DEV_IDS, *minAPIVersion, *maxAPIVersion) 115 if err != nil { 116 sklog.Fatalf("Unable to retrieve available devices: %s", err) 117 } 118 sklog.Infof("---") 119 sklog.Infof("Selected devices:") 120 logDevices(devices) 121 122 if err := runTests(apk_path, devices, ignoredDevices, client, *dryRun); err != nil { 123 sklog.Fatalf("Error triggering tests on Firebase: %s", err) 124 } 125} 126 127// getAvailableDevices is given a whitelist. It queries Firebase Testlab for all 128// available devices and then returns a list of devices to be tested and the list 129// of ignored devices. 130func getAvailableDevices(whiteList map[string][]string, minAPIVersion, maxAPIVersion int) ([]*tsuite.DeviceVersions, []*tsuite.DeviceVersions, error) { 131 // Get the list of all devices in JSON format from Firebase testlab. 132 var buf bytes.Buffer 133 cmd := parseCommand(CMD_AVAILABE_DEVICES) 134 cmd.Stdout = &buf 135 cmd.Stderr = os.Stdout 136 if err := cmd.Run(); err != nil { 137 return nil, nil, err 138 } 139 140 // Unmarshal the result. 141 foundDevices := []*tsuite.FirebaseDevice{} 142 bufBytes := buf.Bytes() 143 if err := json.Unmarshal(bufBytes, &foundDevices); err != nil { 144 return nil, nil, sklog.FmtErrorf("Unmarshal of device information failed: %s \nJSON Input: %s\n", err, string(bufBytes)) 145 } 146 147 // iterate over the available devices and partition them. 148 allDevices := make([]*tsuite.DeviceVersions, 0, len(foundDevices)) 149 ret := make([]*tsuite.DeviceVersions, 0, len(foundDevices)) 150 ignored := make([]*tsuite.DeviceVersions, 0, len(foundDevices)) 151 for _, dev := range foundDevices { 152 // Filter out all the virtual devices. 153 if dev.Form == "PHYSICAL" { 154 // Only include devices that are on the whitelist and have versions defined. 155 if foundVersions, ok := whiteList[dev.ID]; ok && (len(foundVersions) > 0) { 156 versionSet := util.NewStringSet(dev.VersionIDs) 157 reqVersions := util.NewStringSet(filterVersions(foundVersions, minAPIVersion, maxAPIVersion)) 158 whiteListVersions := versionSet.Intersect(reqVersions).Keys() 159 ignoredVersions := versionSet.Complement(reqVersions).Keys() 160 sort.Strings(whiteListVersions) 161 sort.Strings(ignoredVersions) 162 ret = append(ret, &tsuite.DeviceVersions{Device: dev, Versions: whiteListVersions}) 163 ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: ignoredVersions}) 164 } else { 165 ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs}) 166 } 167 allDevices = append(allDevices, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs}) 168 } 169 } 170 171 sklog.Infof("All devices:") 172 logDevices(allDevices) 173 174 return ret, ignored, nil 175} 176 177// filterVersions returns the elements in versionIDs where minVersion <= element <= maxVersion. 178func filterVersions(versionIDs []string, minVersion, maxVersion int) []string { 179 ret := make([]string, 0, len(versionIDs)) 180 for _, versionID := range versionIDs { 181 id, err := strconv.Atoi(versionID) 182 if err != nil { 183 sklog.Fatalf("Error parsing version id '%s': %s", versionID, err) 184 } 185 if (id >= minVersion) && (id <= maxVersion) { 186 ret = append(ret, versionID) 187 } 188 } 189 return ret 190} 191 192// runTests runs the given apk on the given list of devices. 193func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions, client *http.Client, dryRun bool) error { 194 // Get the model-version we want to test. Assume on average each model has 5 supported versions. 195 modelSelectors := make([]string, 0, len(devices)*5) 196 for _, devRec := range devices { 197 for _, version := range devRec.Versions { 198 modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.Device.ID, version)) 199 } 200 } 201 202 now := time.Now() 203 nowMs := now.UnixNano() / int64(time.Millisecond) 204 runID := fmt.Sprintf(RUN_ID_TMPL, nowMs) 205 resultsDir := fmt.Sprintf(RESULT_DIR_TMPL, now.Format("2006/01/02/15"), runID) 206 cmdStr := fmt.Sprintf(RUN_TESTS_TEMPLATE, apk_path, RESULT_BUCKET, resultsDir, strings.Join(modelSelectors, "\n")) 207 cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1)) 208 209 // Run the command. 210 cmd := parseCommand(cmdStr) 211 cmd.Stdout = os.Stdout 212 cmd.Stderr = os.Stdout 213 exitCode := 0 214 215 if dryRun { 216 fmt.Printf("[dry run]: Would have run this command: %s\n", cmdStr) 217 return nil 218 } 219 220 if err := cmd.Run(); err != nil { 221 // Get the exit code. 222 if exitError, ok := err.(*exec.ExitError); ok { 223 ws := exitError.Sys().(syscall.WaitStatus) 224 exitCode = ws.ExitStatus() 225 } 226 sklog.Errorf("Error running tests: %s", err) 227 sklog.Errorf("Exit code: %d", exitCode) 228 229 // Exit code 10 means triggering on Testlab succeeded, but but some of the 230 // runs on devices failed. We consider it a success for this script. 231 if exitCode != 10 { 232 return err 233 } 234 } 235 236 // Store the result in a meta json file. 237 meta := &tsuite.TestRunMeta{ 238 ID: runID, 239 TS: nowMs, 240 Devices: devices, 241 IgnoredDevices: ignoredDevices, 242 ExitCode: exitCode, 243 } 244 245 meta.WriteToGCS(RESULT_BUCKET, resultsDir+"/"+META_DATA_FILENAME, client) 246 return nil 247} 248 249// logDevices logs the given list of devices. 250func logDevices(devices []*tsuite.DeviceVersions) { 251 sklog.Infof("Found %d devices.", len(devices)) 252 for _, dev := range devices { 253 sklog.Infof("%-15s %-30s %v / %v", dev.Device.ID, dev.Device.Name, dev.Device.VersionIDs, dev.Versions) 254 } 255} 256 257// parseCommad parses a command line and wraps it in an exec.Command instance. 258func parseCommand(cmdStr string) *exec.Cmd { 259 cmdArgs := strings.Split(strings.TrimSpace(cmdStr), " ") 260 for idx := range cmdArgs { 261 cmdArgs[idx] = strings.TrimSpace(cmdArgs[idx]) 262 } 263 return exec.Command(cmdArgs[0], cmdArgs[1:]...) 264} 265