1// Copyright 2018 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5package main 6 7// This server runs alongside lottiecap.js and istens for POST requests 8// when any test case reports it has output for Gold. 9 10// TODO(kjlubick): Deduplicate with pathkit-aggregator 11// TODO(kjlubick): Handle uninteresting_hash.txt if needed. 12 13import ( 14 "bytes" 15 "crypto/md5" 16 "encoding/base64" 17 "encoding/json" 18 "flag" 19 "fmt" 20 "image" 21 "image/png" 22 "io/ioutil" 23 "log" 24 "net/http" 25 "os" 26 "path" 27 "strings" 28 29 "go.skia.org/infra/golden/go/jsonio" 30 "go.skia.org/infra/golden/go/types" 31) 32 33// This allows us to use upload_dm_results.py out of the box 34const JSON_FILENAME = "dm.json" 35 36var ( 37 outDir = flag.String("out_dir", "/OUT/", "location to dump the Gold JSON and pngs") 38 port = flag.String("port", "8081", "Port to listen on.") 39 40 browser = flag.String("browser", "Chrome", "Browser Key") 41 buildBucketID = flag.String("buildbucket_build_id", "", "Buildbucket build id key") 42 builder = flag.String("builder", "", "Builder, like 'Test-Debian9-EMCC-GCE-CPU-AVX2-wasm-Debug-All-PathKit'") 43 renderer = flag.String("renderer", "lottie-web", "e.g. lottie-web or skottie") 44 config = flag.String("config", "Release", "Configuration (e.g. Debug/Release) key") 45 gitHash = flag.String("git_hash", "-", "The git commit hash of the version being tested") 46 hostOS = flag.String("host_os", "Debian9", "OS Key") 47 issue = flag.String("issue", "", "ChangeListID (if tryjob)") 48 patchset = flag.Int("patchset", 0, "patchset (if tryjob)") 49 taskId = flag.String("task_id", "", "Skia task id") 50) 51 52// reportBody is the JSON recieved from the JS side. It represents 53// exactly one unique Gold image/test. 54type reportBody struct { 55 // a base64 encoded PNG image. 56 Data string `json:"data"` 57 // a name describing the test. Should be unique enough to allow use of grep. 58 TestName string `json:"test_name"` 59} 60 61// The keys to be used at the top level for all Results. 62var defaultKeys map[string]string 63 64// contains all the results reported in through report_gold_data 65var results []jsonio.Result 66 67func main() { 68 flag.Parse() 69 70 defaultKeys = map[string]string{ 71 "browser": *browser, 72 "renderer": *renderer, 73 "configuration": *config, 74 "cpu_or_gpu": "CPU", 75 "cpu_or_gpu_value": "Browser", 76 "os": *hostOS, 77 "source_type": "lottie", 78 } 79 80 results = []jsonio.Result{} 81 82 http.HandleFunc("/report_gold_data", reporter) 83 http.HandleFunc("/dump_json", dumpJSON) 84 85 fmt.Printf("Waiting for gold ingestion on port %s\n", *port) 86 87 log.Fatal(http.ListenAndServe(":"+*port, nil)) 88} 89 90// reporter handles when the client reports a test has Gold output. 91// It writes the corresponding PNG to disk and appends a Result, assuming 92// no errors. 93func reporter(w http.ResponseWriter, r *http.Request) { 94 if r.Method != "POST" { 95 http.Error(w, "Only POST accepted", 400) 96 return 97 } 98 defer r.Body.Close() 99 100 body, err := ioutil.ReadAll(r.Body) 101 if err != nil { 102 http.Error(w, "Malformed body", 400) 103 return 104 } 105 106 testOutput := reportBody{} 107 if err := json.Unmarshal(body, &testOutput); err != nil { 108 fmt.Println(err) 109 http.Error(w, "Could not unmarshal JSON", 400) 110 return 111 } 112 113 hash := "" 114 if hash, err = writeBase64EncodedPNG(testOutput.Data); err != nil { 115 fmt.Println(err) 116 http.Error(w, "Could not write image to disk", 500) 117 return 118 } 119 120 if _, err := w.Write([]byte("Accepted")); err != nil { 121 fmt.Printf("Could not write response: %s\n", err) 122 return 123 } 124 125 results = append(results, jsonio.Result{ 126 Digest: types.Digest(hash), 127 Key: map[string]string{ 128 "name": testOutput.TestName, 129 }, 130 Options: map[string]string{ 131 "ext": "png", 132 }, 133 }) 134} 135 136// createOutputFile creates a file and set permissions correctly. 137func createOutputFile(p string) (*os.File, error) { 138 outputFile, err := os.Create(p) 139 if err != nil { 140 return nil, fmt.Errorf("Could not open file %s on disk: %s", p, err) 141 } 142 // Make this accessible (and deletable) by all users 143 if err = outputFile.Chmod(0666); err != nil { 144 return nil, fmt.Errorf("Could not change permissions of file %s: %s", p, err) 145 } 146 return outputFile, nil 147} 148 149// dumpJSON writes out a JSON file with all the results, typically at the end of 150// all the tests. 151func dumpJSON(w http.ResponseWriter, r *http.Request) { 152 if r.Method != "POST" { 153 http.Error(w, "Only POST accepted", 400) 154 return 155 } 156 157 p := path.Join(*outDir, JSON_FILENAME) 158 outputFile, err := createOutputFile(p) 159 defer outputFile.Close() 160 if err != nil { 161 fmt.Println(err) 162 http.Error(w, "Could not open json file on disk", 500) 163 return 164 } 165 166 results := jsonio.GoldResults{ 167 ChangelistID: *issue, 168 GitHash: *gitHash, 169 CodeReviewSystem: "gerrit", 170 Key: defaultKeys, 171 PatchsetOrder: *patchset, 172 Results: results, 173 TryJobID: *buildBucketID, 174 ContinuousIntegrationSystem: "buildbucket", 175 } 176 177 enc := json.NewEncoder(outputFile) 178 enc.SetIndent("", " ") // Make it human readable. 179 if err := enc.Encode(&results); err != nil { 180 fmt.Println(err) 181 http.Error(w, "Could not write json to disk", 500) 182 return 183 } 184 fmt.Println("JSON Written") 185} 186 187// writeBase64EncodedPNG writes a PNG to disk and returns the md5 of the 188// decoded PNG bytes and any error. This hash is what will be used as 189// the gold digest and the file name. 190func writeBase64EncodedPNG(data string) (string, error) { 191 // data starts with something like data:image/png;base64,[data] 192 // https://en.wikipedia.org/wiki/Data_URI_scheme 193 start := strings.Index(data, ",") 194 b := bytes.NewBufferString(data[start+1:]) 195 pngReader := base64.NewDecoder(base64.StdEncoding, b) 196 197 pngBytes, err := ioutil.ReadAll(pngReader) 198 if err != nil { 199 return "", fmt.Errorf("Could not decode base 64 encoding %s", err) 200 } 201 202 // compute the hash of the pixel values, like DM does 203 img, err := png.Decode(bytes.NewBuffer(pngBytes)) 204 if err != nil { 205 return "", fmt.Errorf("Not a valid png: %s", err) 206 } 207 hash := "" 208 switch img.(type) { 209 case *image.NRGBA: 210 i := img.(*image.NRGBA) 211 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 212 case *image.RGBA: 213 i := img.(*image.RGBA) 214 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 215 case *image.RGBA64: 216 i := img.(*image.RGBA64) 217 hash = fmt.Sprintf("%x", md5.Sum(i.Pix)) 218 default: 219 return "", fmt.Errorf("Unknown type of image") 220 } 221 222 p := path.Join(*outDir, hash+".png") 223 outputFile, err := createOutputFile(p) 224 defer outputFile.Close() 225 if err != nil { 226 return "", fmt.Errorf("Could not create png file %s: %s", p, err) 227 } 228 if _, err = outputFile.Write(pngBytes); err != nil { 229 return "", fmt.Errorf("Could not write to file %s: %s", p, err) 230 } 231 return hash, nil 232} 233