1/* 2 * Copyright 2018 Google Inc. 3 * 4 * Use of this source code is governed by a BSD-style license that can be 5 * found in the LICENSE file. 6 */ 7 8package main 9 10import ( 11 "bytes" 12 "context" 13 "encoding/json" 14 "flag" 15 "fmt" 16 "io" 17 "io/ioutil" 18 "net/http" 19 "os" 20 "os/exec" 21 "sort" 22 "strconv" 23 "strings" 24 "syscall" 25 "time" 26 27 "go.skia.org/infra/go/gcs" 28 29 "cloud.google.com/go/storage" 30 "google.golang.org/api/option" 31 gstorage "google.golang.org/api/storage/v1" 32 33 "go.skia.org/infra/go/auth" 34 "go.skia.org/infra/go/common" 35 "go.skia.org/infra/go/sklog" 36 "go.skia.org/infra/go/util" 37) 38 39const ( 40 META_DATA_FILENAME = "meta.json" 41) 42 43// Command line flags. 44var ( 45 devicesFile = flag.String("devices", "", "JSON file that maps device ids to versions to run on. Same format as produced by the dump_devices flag.") 46 dryRun = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.") 47 dumpDevFile = flag.String("dump_devices", "", "Creates a JSON file with all physical devices that are not deprecated.") 48 minAPIVersion = flag.Int("min_api", 0, "Minimum API version required by device.") 49 maxAPIVersion = flag.Int("max_api", 99, "Maximum API version required by device.") 50 properties = flag.String("properties", "", "Custom meta data to be added to the uploaded APK. Comma separated list of key=value pairs, i.e. 'k1=v1,k2=v2,k3=v3.") 51 serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.") 52 uploadGCSPath = flag.String("upload_path", "", "GCS path (bucket/path) to where the APK should be uploaded to. It's assume to a full path (not a directory).") 53) 54 55const ( 56 RUN_TESTS_TEMPLATE = `gcloud beta firebase test android run 57 --type=game-loop 58 --app=%s 59 --results-bucket=%s 60 --results-dir=%s 61 --directories-to-pull=/sdcard/Android/data/org.skia.skqp 62 --timeout 30m 63 %s 64` 65 MODEL_VERSION_TMPL = "--device model=%s,version=%s,orientation=portrait" 66 RESULT_BUCKET = "skia-firebase-test-lab" 67 RESULT_DIR_TMPL = "testruns/%s/%s" 68 RUN_ID_TMPL = "testrun-%d" 69 CMD_AVAILABLE_DEVICES = "gcloud firebase test android models list --format json" 70) 71 72func main() { 73 common.Init() 74 75 // Get the path to the APK. It can be empty if we are dumping the device list. 76 apkPath := flag.Arg(0) 77 if *dumpDevFile == "" && apkPath == "" { 78 sklog.Errorf("Missing APK. The APK file needs to be passed as the positional argument.") 79 os.Exit(1) 80 } 81 82 // Get the available devices. 83 fbDevices, deviceList, err := getAvailableDevices() 84 if err != nil { 85 sklog.Fatalf("Error retrieving available devices: %s", err) 86 } 87 88 // Dump the device list and exit. 89 if *dumpDevFile != "" { 90 if err := writeDeviceList(*dumpDevFile, deviceList); err != nil { 91 sklog.Fatalf("Unable to write devices: %s", err) 92 } 93 return 94 } 95 96 // If no devices are explicitly listed. Use all of them. 97 whiteList := deviceList 98 if *devicesFile != "" { 99 whiteList, err = readDeviceList(*devicesFile) 100 if err != nil { 101 sklog.Fatalf("Error reading device file: %s", err) 102 } 103 } 104 105 // Make sure we can authenticate locally and in the cloud. 106 client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email") 107 if err != nil { 108 sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err) 109 } 110 111 // Filter the devices according the white list and other parameters. 112 devices, ignoredDevices := filterDevices(fbDevices, whiteList, *minAPIVersion, *maxAPIVersion) 113 sklog.Infof("---\nSelected devices:") 114 logDevices(devices) 115 116 if len(devices) == 0 { 117 sklog.Errorf("No devices selected. Not running tests.") 118 os.Exit(1) 119 } 120 121 if err := runTests(apkPath, devices, ignoredDevices, client, *dryRun); err != nil { 122 sklog.Fatalf("Error running tests on Firebase: %s", err) 123 } 124 125 if !*dryRun && (*uploadGCSPath != "") && (*properties != "") { 126 if err := uploadAPK(apkPath, *uploadGCSPath, *properties, client); err != nil { 127 sklog.Fatalf("Error uploading APK to '%s': %s", *uploadGCSPath, err) 128 } 129 } 130} 131 132// getAvailableDevices queries Firebase Testlab for all physical devices that 133// are not deprecated. It returns two lists with the same information. 134// The first contains all device information as returned by Firebase while 135// the second contains the information necessary to use in a whitelist. 136func getAvailableDevices() ([]*DeviceVersions, DeviceList, error) { 137 // Get the list of all devices in JSON format from Firebase testlab. 138 var buf bytes.Buffer 139 var errBuf bytes.Buffer 140 cmd := parseCommand(CMD_AVAILABLE_DEVICES) 141 cmd.Stdout = &buf 142 cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf) 143 if err := cmd.Run(); err != nil { 144 return nil, nil, sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", CMD_AVAILABLE_DEVICES, err, errBuf) 145 } 146 147 // Unmarshal the result. 148 foundDevices := []*DeviceVersions{} 149 bufBytes := buf.Bytes() 150 if err := json.Unmarshal(bufBytes, &foundDevices); err != nil { 151 return nil, nil, sklog.FmtErrorf("Unmarshal of device information failed: %s \nJSON Input: %s\n", err, string(bufBytes)) 152 } 153 154 // Filter the devices and copy them to device list. 155 devList := DeviceList{} 156 ret := make([]*DeviceVersions, 0, len(foundDevices)) 157 for _, foundDev := range foundDevices { 158 // Only consider physical devices and devices that are not deprecated. 159 if (foundDev.Form == "PHYSICAL") && !util.In("deprecated", foundDev.Tags) { 160 ret = append(ret, foundDev) 161 devList = append(devList, &DevInfo{ 162 ID: foundDev.ID, 163 Name: foundDev.Name, 164 RunVersions: foundDev.VersionIDs, 165 }) 166 } 167 } 168 return foundDevices, devList, nil 169} 170 171// filterDevices filters the given devices by ensuring that they are in the white list 172// and within the given api version range. 173// It returns two lists: (accepted_devices, ignored_devices) 174func filterDevices(foundDevices []*DeviceVersions, whiteList DeviceList, minAPIVersion, maxAPIVersion int) ([]*DeviceVersions, []*DeviceVersions) { 175 // iterate over the available devices and partition them. 176 allDevices := make([]*DeviceVersions, 0, len(foundDevices)) 177 ret := make([]*DeviceVersions, 0, len(foundDevices)) 178 ignored := make([]*DeviceVersions, 0, len(foundDevices)) 179 for _, dev := range foundDevices { 180 // Only include devices that are on the whitelist and have versions defined. 181 if targetDev := whiteList.find(dev.ID); targetDev != nil && (len(targetDev.RunVersions) > 0) { 182 versionSet := util.NewStringSet(dev.VersionIDs) 183 reqVersions := util.NewStringSet(filterVersions(targetDev.RunVersions, minAPIVersion, maxAPIVersion)) 184 whiteListVersions := versionSet.Intersect(reqVersions).Keys() 185 ignoredVersions := versionSet.Complement(reqVersions).Keys() 186 sort.Strings(whiteListVersions) 187 sort.Strings(ignoredVersions) 188 if len(whiteListVersions) > 0 { 189 ret = append(ret, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: whiteListVersions}) 190 } 191 if len(ignoredVersions) > 0 { 192 ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: ignoredVersions}) 193 } 194 } else { 195 ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs}) 196 } 197 allDevices = append(allDevices, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs}) 198 } 199 200 sklog.Infof("All devices:") 201 logDevices(allDevices) 202 203 return ret, ignored 204} 205 206// filterVersions returns the elements in versionIDs where minVersion <= element <= maxVersion. 207func filterVersions(versionIDs []string, minVersion, maxVersion int) []string { 208 ret := make([]string, 0, len(versionIDs)) 209 for _, versionID := range versionIDs { 210 id, err := strconv.Atoi(versionID) 211 if err != nil { 212 sklog.Fatalf("Error parsing version id '%s': %s", versionID, err) 213 } 214 if (id >= minVersion) && (id <= maxVersion) { 215 ret = append(ret, versionID) 216 } 217 } 218 return ret 219} 220 221// runTests runs the given apk on the given list of devices. 222func runTests(apk_path string, devices, ignoredDevices []*DeviceVersions, client *http.Client, dryRun bool) error { 223 // Get the model-version we want to test. Assume on average each model has 5 supported versions. 224 modelSelectors := make([]string, 0, len(devices)*5) 225 for _, devRec := range devices { 226 for _, version := range devRec.RunVersions { 227 modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.FirebaseDevice.ID, version)) 228 } 229 } 230 231 now := time.Now() 232 nowMs := now.UnixNano() / int64(time.Millisecond) 233 runID := fmt.Sprintf(RUN_ID_TMPL, nowMs) 234 resultsDir := fmt.Sprintf(RESULT_DIR_TMPL, now.Format("2006/01/02/15"), runID) 235 cmdStr := fmt.Sprintf(RUN_TESTS_TEMPLATE, apk_path, RESULT_BUCKET, resultsDir, strings.Join(modelSelectors, "\n")) 236 cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1)) 237 238 // Run the command. 239 var errBuf bytes.Buffer 240 cmd := parseCommand(cmdStr) 241 cmd.Stdout = os.Stdout 242 cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf) 243 exitCode := 0 244 245 if dryRun { 246 fmt.Printf("[dry run]: Would have run this command: %s\n", cmdStr) 247 return nil 248 } 249 250 if err := cmd.Run(); err != nil { 251 // Get the exit code. 252 if exitError, ok := err.(*exec.ExitError); ok { 253 ws := exitError.Sys().(syscall.WaitStatus) 254 exitCode = ws.ExitStatus() 255 } 256 257 sklog.Errorf("Error running tests: %s", err) 258 sklog.Errorf("Exit code: %d", exitCode) 259 260 // Exit code 10 means triggering on Testlab succeeded, but but some of the 261 // runs on devices failed. We consider it a success for this script. 262 if exitCode != 10 { 263 return sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", cmdStr, err, errBuf) 264 } 265 } 266 267 // Store the result in a meta json file. 268 meta := &TestRunMeta{ 269 ID: runID, 270 TS: nowMs, 271 Devices: devices, 272 IgnoredDevices: ignoredDevices, 273 ExitCode: exitCode, 274 } 275 276 targetPath := fmt.Sprintf("%s/%s/%s", RESULT_BUCKET, resultsDir, META_DATA_FILENAME) 277 if err := meta.writeToGCS(targetPath, client); err != nil { 278 return err 279 } 280 sklog.Infof("Meta data written to gs://%s", targetPath) 281 return nil 282} 283 284// uploadAPK uploads the APK at the given path to the bucket/path in gcsPath. 285// The key-value pairs in propStr are set as custom meta data of the APK. 286func uploadAPK(apkPath, gcsPath, propStr string, client *http.Client) error { 287 properties, err := splitProperties(propStr) 288 if err != nil { 289 return err 290 } 291 apkFile, err := os.Open(apkPath) 292 if err != nil { 293 return err 294 } 295 defer util.Close(apkFile) 296 297 if err := copyReaderToGCS(gcsPath, apkFile, client, "application/vnd.android.package-archive", properties, true, false); err != nil { 298 return err 299 } 300 301 sklog.Infof("APK uploaded to gs://%s", gcsPath) 302 return nil 303} 304 305// splitProperties receives a comma separated list of 'key=value' pairs and 306// returnes them as a map. 307func splitProperties(propStr string) (map[string]string, error) { 308 splitProps := strings.Split(propStr, ",") 309 properties := make(map[string]string, len(splitProps)) 310 for _, oneProp := range splitProps { 311 kv := strings.Split(oneProp, "=") 312 if len(kv) != 2 { 313 return nil, sklog.FmtErrorf("Inavlid porperties format. Unable to parse '%s'", propStr) 314 } 315 properties[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) 316 } 317 return properties, nil 318} 319 320// logDevices logs the given list of devices. 321func logDevices(devices []*DeviceVersions) { 322 sklog.Infof("Found %d devices.", len(devices)) 323 for _, dev := range devices { 324 fbDev := dev.FirebaseDevice 325 sklog.Infof("%-15s %-30s %v / %v", fbDev.ID, fbDev.Name, fbDev.VersionIDs, dev.RunVersions) 326 } 327} 328 329// parseCommad parses a command line and wraps it in an exec.Command instance. 330func parseCommand(cmdStr string) *exec.Cmd { 331 cmdArgs := strings.Split(strings.TrimSpace(cmdStr), " ") 332 for idx := range cmdArgs { 333 cmdArgs[idx] = strings.TrimSpace(cmdArgs[idx]) 334 } 335 return exec.Command(cmdArgs[0], cmdArgs[1:]...) 336} 337 338// DeviceList is a simple list of devices, primarily used to define the 339// whitelist of devices we want to run on. 340type DeviceList []*DevInfo 341 342type DevInfo struct { 343 ID string `json:"id"` 344 Name string `json:"name"` 345 RunVersions []string `json:"runVersions"` 346} 347 348func (d DeviceList) find(id string) *DevInfo { 349 for _, devInfo := range d { 350 if devInfo.ID == id { 351 return devInfo 352 } 353 } 354 return nil 355} 356 357func writeDeviceList(fileName string, devList DeviceList) error { 358 jsonBytes, err := json.MarshalIndent(devList, "", " ") 359 if err != nil { 360 return sklog.FmtErrorf("Unable to encode JSON: %s", err) 361 } 362 363 if err := ioutil.WriteFile(fileName, jsonBytes, 0644); err != nil { 364 sklog.FmtErrorf("Unable to write file '%s': %s", fileName, err) 365 } 366 return nil 367} 368 369func readDeviceList(fileName string) (DeviceList, error) { 370 inFile, err := os.Open(fileName) 371 if err != nil { 372 return nil, sklog.FmtErrorf("Unable to open file '%s': %s", fileName, err) 373 } 374 defer util.Close(inFile) 375 376 var devList DeviceList 377 if err := json.NewDecoder(inFile).Decode(&devList); err != nil { 378 return nil, sklog.FmtErrorf("Unable to decode JSON from '%s': %s", fileName, err) 379 } 380 return devList, nil 381} 382 383// FirebaseDevice contains the information and JSON tags for device information 384// returned by firebase. 385type FirebaseDevice struct { 386 Brand string `json:"brand"` 387 Form string `json:"form"` 388 ID string `json:"id"` 389 Manufacturer string `json:"manufacturer"` 390 Name string `json:"name"` 391 VersionIDs []string `json:"supportedVersionIds"` 392 Tags []string `json:"tags"` 393} 394 395// DeviceVersions combines device information from Firebase Testlab with 396// a selected list of versions. This is used to define a subset of versions 397// used by a devices. 398type DeviceVersions struct { 399 *FirebaseDevice 400 401 // RunVersions contains the version ids of interest contained in Device. 402 RunVersions []string 403} 404 405// TestRunMeta contains the meta data of a complete testrun on firebase. 406type TestRunMeta struct { 407 ID string `json:"id"` 408 TS int64 `json:"timeStamp"` 409 Devices []*DeviceVersions `json:"devices"` 410 IgnoredDevices []*DeviceVersions `json:"ignoredDevices"` 411 ExitCode int `json:"exitCode"` 412} 413 414// writeToGCS writes the meta data as JSON to the given bucket and path in 415// GCS. It assumes that the provided client has permissions to write to the 416// specified location in GCS. 417func (t *TestRunMeta) writeToGCS(gcsPath string, client *http.Client) error { 418 jsonBytes, err := json.Marshal(t) 419 if err != nil { 420 return err 421 } 422 return copyReaderToGCS(gcsPath, bytes.NewReader(jsonBytes), client, "", nil, false, true) 423} 424 425// TODO(stephana): Merge copyReaderToGCS into the go/gcs in 426// the infra repository. 427 428// copyReaderToGCS reads all available content from the given reader and writes 429// it to the given path in GCS. 430func copyReaderToGCS(gcsPath string, reader io.Reader, client *http.Client, contentType string, metaData map[string]string, public bool, gzip bool) error { 431 storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client)) 432 if err != nil { 433 return err 434 } 435 bucket, path := gcs.SplitGSPath(gcsPath) 436 w := storageClient.Bucket(bucket).Object(path).NewWriter(context.Background()) 437 438 // Set the content if requested. 439 if contentType != "" { 440 w.ObjectAttrs.ContentType = contentType 441 } 442 443 // Set the meta data if requested 444 if metaData != nil { 445 w.Metadata = metaData 446 } 447 448 // Make the object public if requested. 449 if public { 450 w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}} 451 } 452 453 // Write the everything the reader can provide to the GCS object. Either 454 // gzip'ed or plain. 455 if gzip { 456 w.ObjectAttrs.ContentEncoding = "gzip" 457 err = util.WithGzipWriter(w, func(w io.Writer) error { 458 _, err := io.Copy(w, reader) 459 return err 460 }) 461 } else { 462 _, err = io.Copy(w, reader) 463 } 464 465 // Make sure we return an error when we close the remote object. 466 if err != nil { 467 _ = w.CloseWithError(err) 468 return err 469 } 470 return w.Close() 471} 472