1// Copyright 2018 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5package main 6 7// This server runs along side the karma tests and listens for POST requests 8// when any test case reports it has output for Gold. See testReporter.js 9// for the browser side part. 10 11import ( 12 "bytes" 13 "crypto/md5" 14 "encoding/base64" 15 "encoding/json" 16 "flag" 17 "fmt" 18 "image" 19 "image/png" 20 "io/ioutil" 21 "log" 22 "net/http" 23 "os" 24 "path" 25 "strings" 26 "sync" 27 28 "go.skia.org/infra/go/util" 29 "go.skia.org/infra/golden/go/jsonio" 30 "go.skia.org/infra/golden/go/types" 31) 32 33// This allows us to use upload_dm_results.py out of the box 34const JSON_FILENAME = "dm.json" 35 36var ( 37 outDir = flag.String("out_dir", "/OUT/", "location to dump the Gold JSON and pngs") 38 port = flag.String("port", "8081", "Port to listen on.") 39 40 browser = flag.String("browser", "Chrome", "Browser Key") 41 buildBucketID = flag.String("buildbucket_build_id", "", "Buildbucket build id key") 42 builder = flag.String("builder", "", "Builder, like 'Test-Debian9-EMCC-GCE-CPU-AVX2-wasm-Debug-All-PathKit'") 43 compiledLanguage = flag.String("compiled_language", "wasm", "wasm or asm.js") 44 config = flag.String("config", "Release", "Configuration (e.g. Debug/Release) key") 45 gitHash = flag.String("git_hash", "-", "The git commit hash of the version being tested") 46 hostOS = flag.String("host_os", "Debian9", "OS Key") 47 issue = flag.String("issue", "", "ChangelistID (if tryjob)") 48 patchset = flag.Int("patchset", 0, "patchset (if tryjob)") 49 sourceType = flag.String("source_type", "pathkit", "Gold Source type, like pathkit,canvaskit") 50) 51 52// Received from the JS side. 53type reportBody struct { 54 // e.g. "canvas" or "svg" 55 OutputType string `json:"output_type"` 56 // a base64 encoded PNG image. 57 Data string `json:"data"` 58 // a name describing the test. Should be unique enough to allow use of grep. 59 TestName string `json:"test_name"` 60} 61 62// The keys to be used at the top level for all Results. 63var defaultKeys map[string]string 64 65// contains all the results reported in through report_gold_data 66var results []jsonio.Result 67var resultsMutex sync.Mutex 68 69func main() { 70 flag.Parse() 71 72 cpuGPU := "CPU" 73 if strings.Index(*builder, "-GPU-") != -1 { 74 cpuGPU = "GPU" 75 } 76 defaultKeys = map[string]string{ 77 "arch": "WASM", 78 "browser": *browser, 79 "compiled_language": *compiledLanguage, 80 "compiler": "emsdk", 81 "configuration": *config, 82 "cpu_or_gpu": cpuGPU, 83 "cpu_or_gpu_value": "Browser", 84 "os": *hostOS, 85 "source_type": *sourceType, 86 } 87 88 results = []jsonio.Result{} 89 90 http.HandleFunc("/report_gold_data", reporter) 91 http.HandleFunc("/dump_json", dumpJSON) 92 93 fmt.Printf("Waiting for gold ingestion on port %s\n", *port) 94 95 log.Fatal(http.ListenAndServe(":"+*port, nil)) 96} 97 98// reporter handles when the client reports a test has Gold output. 99// It writes the corresponding PNG to disk and appends a Result, assuming 100// no errors. 101func reporter(w http.ResponseWriter, r *http.Request) { 102 if r.Method != "POST" { 103 http.Error(w, "Only POST accepted", 400) 104 return 105 } 106 defer util.Close(r.Body) 107 108 body, err := ioutil.ReadAll(r.Body) 109 if err != nil { 110 http.Error(w, "Malformed body", 400) 111 return 112 } 113 114 testOutput := reportBody{} 115 if err := json.Unmarshal(body, &testOutput); err != nil { 116 fmt.Println(err) 117 http.Error(w, "Could not unmarshal JSON", 400) 118 return 119 } 120 121 hash := "" 122 if hash, err = writeBase64EncodedPNG(testOutput.Data); err != nil { 123 fmt.Println(err) 124 http.Error(w, "Could not write image to disk", 500) 125 return 126 } 127 128 if _, err := w.Write([]byte("Accepted")); err != nil { 129 fmt.Printf("Could not write response: %s\n", err) 130 return 131 } 132 133 resultsMutex.Lock() 134 defer resultsMutex.Unlock() 135 results = append(results, jsonio.Result{ 136 Digest: types.Digest(hash), 137 Key: map[string]string{ 138 "name": testOutput.TestName, 139 "config": testOutput.OutputType, 140 }, 141 Options: map[string]string{ 142 "ext": "png", 143 }, 144 }) 145} 146 147// createOutputFile creates a file and set permissions correctly. 148func createOutputFile(p string) (*os.File, error) { 149 outputFile, err := os.Create(p) 150 if err != nil { 151 return nil, fmt.Errorf("Could not open file %s on disk: %s", p, err) 152 } 153 // Make this accessible (and deletable) by all users 154 if err = outputFile.Chmod(0666); err != nil { 155 return nil, fmt.Errorf("Could not change permissions of file %s: %s", p, err) 156 } 157 return outputFile, nil 158} 159 160// dumpJSON writes out a JSON file with all the results, typically at the end of 161// all the tests. 162func dumpJSON(w http.ResponseWriter, r *http.Request) { 163 if r.Method != "POST" { 164 http.Error(w, "Only POST accepted", 400) 165 return 166 } 167 168 p := path.Join(*outDir, JSON_FILENAME) 169 outputFile, err := createOutputFile(p) 170 if err != nil { 171 fmt.Println(err) 172 http.Error(w, "Could not open json file on disk", 500) 173 return 174 } 175 defer util.Close(outputFile) 176 177 dmresults := jsonio.GoldResults{ 178 GitHash: *gitHash, 179 Key: defaultKeys, 180 Results: results, 181 } 182 183 if *patchset > 0 { 184 dmresults.ChangelistID = *issue 185 dmresults.PatchsetOrder = *patchset 186 dmresults.CodeReviewSystem = "gerrit" 187 dmresults.ContinuousIntegrationSystem = "buildbucket" 188 dmresults.TryJobID = *buildBucketID 189 } 190 191 enc := json.NewEncoder(outputFile) 192 enc.SetIndent("", " ") // Make it human readable. 193 if err := enc.Encode(&dmresults); err != nil { 194 fmt.Println(err) 195 http.Error(w, "Could not write json to disk", 500) 196 return 197 } 198 fmt.Println("JSON Written") 199} 200 201// writeBase64EncodedPNG writes a PNG to disk and returns the md5 of the 202// decoded PNG bytes and any error. This hash is what will be used as 203// the gold digest and the file name. 204func writeBase64EncodedPNG(data string) (string, error) { 205 // data starts with something like data:image/png;base64,[data] 206 // https://en.wikipedia.org/wiki/Data_URI_scheme 207 start := strings.Index(data, ",") 208 b := bytes.NewBufferString(data[start+1:]) 209 pngReader := base64.NewDecoder(base64.StdEncoding, b) 210 211 pngBytes, err := ioutil.ReadAll(pngReader) 212 if err != nil { 213 return "", fmt.Errorf("Could not decode base 64 encoding %s", err) 214 } 215 216 // compute the hash of the pixel values, like DM does 217 img, err := png.Decode(bytes.NewBuffer(pngBytes)) 218 if err != nil { 219 return "", fmt.Errorf("Not a valid png: %s", err) 220 } 221 hash := "" 222 switch img.(type) { 223 case *image.NRGBA: 224 i := img.(*image.NRGBA) 225 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 226 case *image.RGBA: 227 i := img.(*image.RGBA) 228 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 229 case *image.RGBA64: 230 i := img.(*image.RGBA64) 231 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 232 default: 233 return "", fmt.Errorf("Unknown type of image") 234 } 235 236 p := path.Join(*outDir, hash+".png") 237 outputFile, err := createOutputFile(p) 238 if err != nil { 239 return "", fmt.Errorf("Could not create png file %s: %s", p, err) 240 } 241 if _, err = outputFile.Write(pngBytes); err != nil { 242 util.Close(outputFile) 243 return "", fmt.Errorf("Could not write to file %s: %s", p, err) 244 } 245 return hash, outputFile.Close() 246} 247