1// Copyright 2018 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5package main 6 7// This server runs alongside lottiecap.js and istens for POST requests 8// when any test case reports it has output for Gold. 9 10// TODO(kjlubick): Deduplicate with pathkit-aggregator 11// TODO(kjlubick): Handle uninteresting_hash.txt if needed. 12 13import ( 14 "bytes" 15 "crypto/md5" 16 "encoding/base64" 17 "encoding/json" 18 "flag" 19 "fmt" 20 "image" 21 "image/png" 22 "io/ioutil" 23 "log" 24 "net/http" 25 "os" 26 "path" 27 "strings" 28 29 "go.skia.org/infra/golden/go/goldingestion" 30) 31 32// This allows us to use upload_dm_results.py out of the box 33const JSON_FILENAME = "dm.json" 34 35var ( 36 outDir = flag.String("out_dir", "/OUT/", "location to dump the Gold JSON and pngs") 37 port = flag.String("port", "8081", "Port to listen on.") 38 39 botId = flag.String("bot_id", "", "swarming bot id") 40 browser = flag.String("browser", "Chrome", "Browser Key") 41 buildBucketID = flag.Int64("buildbucket_build_id", 0, "Buildbucket build id key") 42 builder = flag.String("builder", "", "Builder, like 'Test-Debian9-EMCC-GCE-CPU-AVX2-wasm-Debug-All-PathKit'") 43 renderer = flag.String("renderer", "lottie-web", "e.g. lottie-web or skottie") 44 config = flag.String("config", "Release", "Configuration (e.g. Debug/Release) key") 45 gitHash = flag.String("git_hash", "-", "The git commit hash of the version being tested") 46 hostOS = flag.String("host_os", "Debian9", "OS Key") 47 issue = flag.Int64("issue", 0, "issue (if tryjob)") 48 patch_storage = flag.String("patch_storage", "", "patch storage (if tryjob)") 49 patchset = flag.Int64("patchset", 0, "patchset (if tryjob)") 50 taskId = flag.String("task_id", "", "swarming task id") 51) 52 53// reportBody is the JSON recieved from the JS side. It represents 54// exactly one unique Gold image/test. 55type reportBody struct { 56 // a base64 encoded PNG image. 57 Data string `json:"data"` 58 // a name describing the test. Should be unique enough to allow use of grep. 59 TestName string `json:"test_name"` 60} 61 62// The keys to be used at the top level for all Results. 63var defaultKeys map[string]string 64 65// contains all the results reported in through report_gold_data 66var results []*goldingestion.Result 67 68func main() { 69 flag.Parse() 70 71 defaultKeys = map[string]string{ 72 "browser": *browser, 73 "renderer": *renderer, 74 "configuration": *config, 75 "cpu_or_gpu": "CPU", 76 "cpu_or_gpu_value": "Browser", 77 "os": *hostOS, 78 "source_type": "lottie", 79 } 80 81 results = []*goldingestion.Result{} 82 83 http.HandleFunc("/report_gold_data", reporter) 84 http.HandleFunc("/dump_json", dumpJSON) 85 86 fmt.Printf("Waiting for gold ingestion on port %s\n", *port) 87 88 log.Fatal(http.ListenAndServe(":"+*port, nil)) 89} 90 91// reporter handles when the client reports a test has Gold output. 92// It writes the corresponding PNG to disk and appends a Result, assuming 93// no errors. 94func reporter(w http.ResponseWriter, r *http.Request) { 95 if r.Method != "POST" { 96 http.Error(w, "Only POST accepted", 400) 97 return 98 } 99 defer r.Body.Close() 100 101 body, err := ioutil.ReadAll(r.Body) 102 if err != nil { 103 http.Error(w, "Malformed body", 400) 104 return 105 } 106 107 testOutput := reportBody{} 108 if err := json.Unmarshal(body, &testOutput); err != nil { 109 fmt.Println(err) 110 http.Error(w, "Could not unmarshal JSON", 400) 111 return 112 } 113 114 hash := "" 115 if hash, err = writeBase64EncodedPNG(testOutput.Data); err != nil { 116 fmt.Println(err) 117 http.Error(w, "Could not write image to disk", 500) 118 return 119 } 120 121 if _, err := w.Write([]byte("Accepted")); err != nil { 122 fmt.Printf("Could not write response: %s\n", err) 123 return 124 } 125 126 results = append(results, &goldingestion.Result{ 127 Digest: hash, 128 Key: map[string]string{ 129 "name": testOutput.TestName, 130 }, 131 Options: map[string]string{ 132 "ext": "png", 133 }, 134 }) 135} 136 137// createOutputFile creates a file and set permissions correctly. 138func createOutputFile(p string) (*os.File, error) { 139 outputFile, err := os.Create(p) 140 if err != nil { 141 return nil, fmt.Errorf("Could not open file %s on disk: %s", p, err) 142 } 143 // Make this accessible (and deletable) by all users 144 if err = outputFile.Chmod(0666); err != nil { 145 return nil, fmt.Errorf("Could not change permissions of file %s: %s", p, err) 146 } 147 return outputFile, nil 148} 149 150// dumpJSON writes out a JSON file with all the results, typically at the end of 151// all the tests. 152func dumpJSON(w http.ResponseWriter, r *http.Request) { 153 if r.Method != "POST" { 154 http.Error(w, "Only POST accepted", 400) 155 return 156 } 157 158 p := path.Join(*outDir, JSON_FILENAME) 159 outputFile, err := createOutputFile(p) 160 defer outputFile.Close() 161 if err != nil { 162 fmt.Println(err) 163 http.Error(w, "Could not open json file on disk", 500) 164 return 165 } 166 167 results := goldingestion.DMResults{ 168 BuildBucketID: *buildBucketID, 169 Builder: *builder, 170 GitHash: *gitHash, 171 Issue: *issue, 172 Key: defaultKeys, 173 PatchStorage: *patch_storage, 174 Patchset: *patchset, 175 Results: results, 176 SwarmingBotID: *botId, 177 SwarmingTaskID: *taskId, 178 } 179 180 enc := json.NewEncoder(outputFile) 181 enc.SetIndent("", " ") // Make it human readable. 182 if err := enc.Encode(&results); err != nil { 183 fmt.Println(err) 184 http.Error(w, "Could not write json to disk", 500) 185 return 186 } 187 fmt.Println("JSON Written") 188} 189 190// writeBase64EncodedPNG writes a PNG to disk and returns the md5 of the 191// decoded PNG bytes and any error. This hash is what will be used as 192// the gold digest and the file name. 193func writeBase64EncodedPNG(data string) (string, error) { 194 // data starts with something like data:image/png;base64,[data] 195 // https://en.wikipedia.org/wiki/Data_URI_scheme 196 start := strings.Index(data, ",") 197 b := bytes.NewBufferString(data[start+1:]) 198 pngReader := base64.NewDecoder(base64.StdEncoding, b) 199 200 pngBytes, err := ioutil.ReadAll(pngReader) 201 if err != nil { 202 return "", fmt.Errorf("Could not decode base 64 encoding %s", err) 203 } 204 205 // compute the hash of the pixel values, like DM does 206 img, err := png.Decode(bytes.NewBuffer(pngBytes)) 207 if err != nil { 208 return "", fmt.Errorf("Not a valid png: %s", err) 209 } 210 hash := "" 211 switch img.(type) { 212 case *image.NRGBA: 213 i := img.(*image.NRGBA) 214 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 215 case *image.RGBA: 216 i := img.(*image.RGBA) 217 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 218 case *image.RGBA64: 219 i := img.(*image.RGBA64) 220 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 221 default: 222 return "", fmt.Errorf("Unknown type of image") 223 } 224 225 p := path.Join(*outDir, hash+".png") 226 outputFile, err := createOutputFile(p) 227 defer outputFile.Close() 228 if err != nil { 229 return "", fmt.Errorf("Could not create png file %s: %s", p, err) 230 } 231 if _, err = outputFile.Write(pngBytes); err != nil { 232 return "", fmt.Errorf("Could not write to file %s: %s", p, err) 233 } 234 return hash, nil 235} 236