1// Copyright 2018 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5package main 6 7// This server runs along side the karma tests and listens for POST requests 8// when any test case reports it has output for Gold. See testReporter.js 9// for the browser side part. 10 11import ( 12 "bytes" 13 "crypto/md5" 14 "encoding/base64" 15 "encoding/json" 16 "flag" 17 "fmt" 18 "image" 19 "image/png" 20 "io/ioutil" 21 "log" 22 "net/http" 23 "os" 24 "path" 25 "strings" 26 27 "go.skia.org/infra/golden/go/goldingestion" 28 "go.skia.org/infra/golden/go/jsonio" 29) 30 31// This allows us to use upload_dm_results.py out of the box 32const JSON_FILENAME = "dm.json" 33 34var ( 35 outDir = flag.String("out_dir", "/OUT/", "location to dump the Gold JSON and pngs") 36 port = flag.String("port", "8081", "Port to listen on.") 37 38 botId = flag.String("bot_id", "", "swarming bot id") 39 browser = flag.String("browser", "Chrome", "Browser Key") 40 buildBucketID = flag.Int64("buildbucket_build_id", 0, "Buildbucket build id key") 41 builder = flag.String("builder", "", "Builder, like 'Test-Debian9-EMCC-GCE-CPU-AVX2-wasm-Debug-All-PathKit'") 42 compiledLanguage = flag.String("compiled_language", "wasm", "wasm or asm.js") 43 config = flag.String("config", "Release", "Configuration (e.g. Debug/Release) key") 44 gitHash = flag.String("git_hash", "-", "The git commit hash of the version being tested") 45 hostOS = flag.String("host_os", "Debian9", "OS Key") 46 issue = flag.Int64("issue", 0, "issue (if tryjob)") 47 patchset = flag.Int64("patchset", 0, "patchset (if tryjob)") 48 taskId = flag.String("task_id", "", "swarming task id") 49 sourceType = flag.String("source_type", "pathkit", "Gold Source type, like pathkit,canvaskit") 50) 51 52// Received from the JS side. 53type reportBody struct { 54 // e.g. "canvas" or "svg" 55 OutputType string `json:"output_type"` 56 // a base64 encoded PNG image. 57 Data string `json:"data"` 58 // a name describing the test. Should be unique enough to allow use of grep. 59 TestName string `json:"test_name"` 60} 61 62// The keys to be used at the top level for all Results. 63var defaultKeys map[string]string 64 65// contains all the results reported in through report_gold_data 66var results []*jsonio.Result 67 68func main() { 69 flag.Parse() 70 71 cpuGPU := "CPU" 72 if strings.Index(*builder, "-GPU-") != -1 { 73 cpuGPU = "GPU" 74 } 75 defaultKeys = map[string]string{ 76 "arch": "WASM", 77 "browser": *browser, 78 "compiled_language": *compiledLanguage, 79 "compiler": "emsdk", 80 "configuration": *config, 81 "cpu_or_gpu": cpuGPU, 82 "cpu_or_gpu_value": "Browser", 83 "os": *hostOS, 84 "source_type": *sourceType, 85 } 86 87 results = []*jsonio.Result{} 88 89 http.HandleFunc("/report_gold_data", reporter) 90 http.HandleFunc("/dump_json", dumpJSON) 91 92 fmt.Printf("Waiting for gold ingestion on port %s\n", *port) 93 94 log.Fatal(http.ListenAndServe(":"+*port, nil)) 95} 96 97// reporter handles when the client reports a test has Gold output. 98// It writes the corresponding PNG to disk and appends a Result, assuming 99// no errors. 100func reporter(w http.ResponseWriter, r *http.Request) { 101 if r.Method != "POST" { 102 http.Error(w, "Only POST accepted", 400) 103 return 104 } 105 defer r.Body.Close() 106 107 body, err := ioutil.ReadAll(r.Body) 108 if err != nil { 109 http.Error(w, "Malformed body", 400) 110 return 111 } 112 113 testOutput := reportBody{} 114 if err := json.Unmarshal(body, &testOutput); err != nil { 115 fmt.Println(err) 116 http.Error(w, "Could not unmarshal JSON", 400) 117 return 118 } 119 120 hash := "" 121 if hash, err = writeBase64EncodedPNG(testOutput.Data); err != nil { 122 fmt.Println(err) 123 http.Error(w, "Could not write image to disk", 500) 124 return 125 } 126 127 if _, err := w.Write([]byte("Accepted")); err != nil { 128 fmt.Printf("Could not write response: %s\n", err) 129 return 130 } 131 132 results = append(results, &jsonio.Result{ 133 Digest: hash, 134 Key: map[string]string{ 135 "name": testOutput.TestName, 136 "config": testOutput.OutputType, 137 }, 138 Options: map[string]string{ 139 "ext": "png", 140 }, 141 }) 142} 143 144// createOutputFile creates a file and set permissions correctly. 145func createOutputFile(p string) (*os.File, error) { 146 outputFile, err := os.Create(p) 147 if err != nil { 148 return nil, fmt.Errorf("Could not open file %s on disk: %s", p, err) 149 } 150 // Make this accessible (and deletable) by all users 151 if err = outputFile.Chmod(0666); err != nil { 152 return nil, fmt.Errorf("Could not change permissions of file %s: %s", p, err) 153 } 154 return outputFile, nil 155} 156 157// dumpJSON writes out a JSON file with all the results, typically at the end of 158// all the tests. 159func dumpJSON(w http.ResponseWriter, r *http.Request) { 160 if r.Method != "POST" { 161 http.Error(w, "Only POST accepted", 400) 162 return 163 } 164 165 p := path.Join(*outDir, JSON_FILENAME) 166 outputFile, err := createOutputFile(p) 167 defer outputFile.Close() 168 if err != nil { 169 fmt.Println(err) 170 http.Error(w, "Could not open json file on disk", 500) 171 return 172 } 173 174 dmresults := goldingestion.DMResults{ 175 GoldResults: &jsonio.GoldResults{ 176 BuildBucketID: *buildBucketID, 177 Builder: *builder, 178 GitHash: *gitHash, 179 Issue: *issue, 180 Key: defaultKeys, 181 Patchset: *patchset, 182 Results: results, 183 SwarmingBotID: *botId, 184 SwarmingTaskID: *taskId, 185 }, 186 } 187 188 enc := json.NewEncoder(outputFile) 189 enc.SetIndent("", " ") // Make it human readable. 190 if err := enc.Encode(&dmresults); err != nil { 191 fmt.Println(err) 192 http.Error(w, "Could not write json to disk", 500) 193 return 194 } 195 fmt.Println("JSON Written") 196} 197 198// writeBase64EncodedPNG writes a PNG to disk and returns the md5 of the 199// decoded PNG bytes and any error. This hash is what will be used as 200// the gold digest and the file name. 201func writeBase64EncodedPNG(data string) (string, error) { 202 // data starts with something like data:image/png;base64,[data] 203 // https://en.wikipedia.org/wiki/Data_URI_scheme 204 start := strings.Index(data, ",") 205 b := bytes.NewBufferString(data[start+1:]) 206 pngReader := base64.NewDecoder(base64.StdEncoding, b) 207 208 pngBytes, err := ioutil.ReadAll(pngReader) 209 if err != nil { 210 return "", fmt.Errorf("Could not decode base 64 encoding %s", err) 211 } 212 213 // compute the hash of the pixel values, like DM does 214 img, err := png.Decode(bytes.NewBuffer(pngBytes)) 215 if err != nil { 216 return "", fmt.Errorf("Not a valid png: %s", err) 217 } 218 hash := "" 219 switch img.(type) { 220 case *image.NRGBA: 221 i := img.(*image.NRGBA) 222 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 223 case *image.RGBA: 224 i := img.(*image.RGBA) 225 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 226 case *image.RGBA64: 227 i := img.(*image.RGBA64) 228 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 229 default: 230 return "", fmt.Errorf("Unknown type of image") 231 } 232 233 p := path.Join(*outDir, hash+".png") 234 outputFile, err := createOutputFile(p) 235 defer outputFile.Close() 236 if err != nil { 237 return "", fmt.Errorf("Could not create png file %s: %s", p, err) 238 } 239 if _, err = outputFile.Write(pngBytes); err != nil { 240 return "", fmt.Errorf("Could not write to file %s: %s", p, err) 241 } 242 return hash, nil 243} 244