• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2018 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5package main
6
7// This server runs alongside lottiecap.js and istens for POST requests
8// when any test case reports it has output for Gold.
9
10// TODO(kjlubick): Deduplicate with pathkit-aggregator
11// TODO(kjlubick): Handle uninteresting_hash.txt if needed.
12
13import (
14	"bytes"
15	"crypto/md5"
16	"encoding/base64"
17	"encoding/json"
18	"flag"
19	"fmt"
20	"image"
21	"image/png"
22	"io/ioutil"
23	"log"
24	"net/http"
25	"os"
26	"path"
27	"strings"
28
29	"go.skia.org/infra/golden/go/jsonio"
30	"go.skia.org/infra/golden/go/types"
31)
32
33// This allows us to use upload_dm_results.py out of the box
34const JSON_FILENAME = "dm.json"
35
36var (
37	outDir = flag.String("out_dir", "/OUT/", "location to dump the Gold JSON and pngs")
38	port   = flag.String("port", "8081", "Port to listen on.")
39
40	browser       = flag.String("browser", "Chrome", "Browser Key")
41	buildBucketID = flag.String("buildbucket_build_id", "", "Buildbucket build id key")
42	builder       = flag.String("builder", "", "Builder, like 'Test-Debian9-EMCC-GCE-CPU-AVX2-wasm-Debug-All-PathKit'")
43	renderer      = flag.String("renderer", "lottie-web", "e.g. lottie-web or skottie")
44	config        = flag.String("config", "Release", "Configuration (e.g. Debug/Release) key")
45	gitHash       = flag.String("git_hash", "-", "The git commit hash of the version being tested")
46	hostOS        = flag.String("host_os", "Debian9", "OS Key")
47	issue         = flag.String("issue", "", "ChangeListID (if tryjob)")
48	patchset      = flag.Int("patchset", 0, "patchset (if tryjob)")
49	taskId        = flag.String("task_id", "", "Skia task id")
50)
51
52// reportBody is the JSON recieved from the JS side. It represents
53// exactly one unique Gold image/test.
54type reportBody struct {
55	// a base64 encoded PNG image.
56	Data string `json:"data"`
57	// a name describing the test. Should be unique enough to allow use of grep.
58	TestName string `json:"test_name"`
59}
60
61// The keys to be used at the top level for all Results.
62var defaultKeys map[string]string
63
64// contains all the results reported in through report_gold_data
65var results []*jsonio.Result
66
67func main() {
68	flag.Parse()
69
70	defaultKeys = map[string]string{
71		"browser":          *browser,
72		"renderer":         *renderer,
73		"configuration":    *config,
74		"cpu_or_gpu":       "CPU",
75		"cpu_or_gpu_value": "Browser",
76		"os":               *hostOS,
77		"source_type":      "lottie",
78	}
79
80	results = []*jsonio.Result{}
81
82	http.HandleFunc("/report_gold_data", reporter)
83	http.HandleFunc("/dump_json", dumpJSON)
84
85	fmt.Printf("Waiting for gold ingestion on port %s\n", *port)
86
87	log.Fatal(http.ListenAndServe(":"+*port, nil))
88}
89
90// reporter handles when the client reports a test has Gold output.
91// It writes the corresponding PNG to disk and appends a Result, assuming
92// no errors.
93func reporter(w http.ResponseWriter, r *http.Request) {
94	if r.Method != "POST" {
95		http.Error(w, "Only POST accepted", 400)
96		return
97	}
98	defer r.Body.Close()
99
100	body, err := ioutil.ReadAll(r.Body)
101	if err != nil {
102		http.Error(w, "Malformed body", 400)
103		return
104	}
105
106	testOutput := reportBody{}
107	if err := json.Unmarshal(body, &testOutput); err != nil {
108		fmt.Println(err)
109		http.Error(w, "Could not unmarshal JSON", 400)
110		return
111	}
112
113	hash := ""
114	if hash, err = writeBase64EncodedPNG(testOutput.Data); err != nil {
115		fmt.Println(err)
116		http.Error(w, "Could not write image to disk", 500)
117		return
118	}
119
120	if _, err := w.Write([]byte("Accepted")); err != nil {
121		fmt.Printf("Could not write response: %s\n", err)
122		return
123	}
124
125	results = append(results, &jsonio.Result{
126		Digest: types.Digest(hash),
127		Key: map[string]string{
128			"name": testOutput.TestName,
129		},
130		Options: map[string]string{
131			"ext": "png",
132		},
133	})
134}
135
136// createOutputFile creates a file and set permissions correctly.
137func createOutputFile(p string) (*os.File, error) {
138	outputFile, err := os.Create(p)
139	if err != nil {
140		return nil, fmt.Errorf("Could not open file %s on disk: %s", p, err)
141	}
142	// Make this accessible (and deletable) by all users
143	if err = outputFile.Chmod(0666); err != nil {
144		return nil, fmt.Errorf("Could not change permissions of file %s: %s", p, err)
145	}
146	return outputFile, nil
147}
148
149// dumpJSON writes out a JSON file with all the results, typically at the end of
150// all the tests.
151func dumpJSON(w http.ResponseWriter, r *http.Request) {
152	if r.Method != "POST" {
153		http.Error(w, "Only POST accepted", 400)
154		return
155	}
156
157	p := path.Join(*outDir, JSON_FILENAME)
158	outputFile, err := createOutputFile(p)
159	defer outputFile.Close()
160	if err != nil {
161		fmt.Println(err)
162		http.Error(w, "Could not open json file on disk", 500)
163		return
164	}
165
166	results := jsonio.GoldResults{
167		Builder:                     *builder,
168		ChangeListID:                *issue,
169		GitHash:                     *gitHash,
170		CodeReviewSystem:            "gerrit",
171		Key:                         defaultKeys,
172		PatchSetOrder:               *patchset,
173		Results:                     results,
174		TaskID:                      *taskId,
175		TryJobID:                    *buildBucketID,
176		ContinuousIntegrationSystem: "buildbucket",
177	}
178
179	enc := json.NewEncoder(outputFile)
180	enc.SetIndent("", "  ") // Make it human readable.
181	if err := enc.Encode(&results); err != nil {
182		fmt.Println(err)
183		http.Error(w, "Could not write json to disk", 500)
184		return
185	}
186	fmt.Println("JSON Written")
187}
188
189// writeBase64EncodedPNG writes a PNG to disk and returns the md5 of the
190// decoded PNG bytes and any error. This hash is what will be used as
191// the gold digest and the file name.
192func writeBase64EncodedPNG(data string) (string, error) {
193	// data starts with something like data:image/png;base64,[data]
194	// https://en.wikipedia.org/wiki/Data_URI_scheme
195	start := strings.Index(data, ",")
196	b := bytes.NewBufferString(data[start+1:])
197	pngReader := base64.NewDecoder(base64.StdEncoding, b)
198
199	pngBytes, err := ioutil.ReadAll(pngReader)
200	if err != nil {
201		return "", fmt.Errorf("Could not decode base 64 encoding %s", err)
202	}
203
204	// compute the hash of the pixel values, like DM does
205	img, err := png.Decode(bytes.NewBuffer(pngBytes))
206	if err != nil {
207		return "", fmt.Errorf("Not a valid png: %s", err)
208	}
209	hash := ""
210	switch img.(type) {
211	case *image.NRGBA:
212		i := img.(*image.NRGBA)
213		hash = fmt.Sprintf("%x", md5.Sum(i.Pix))
214	case *image.RGBA:
215		i := img.(*image.RGBA)
216		hash = fmt.Sprintf("%x", md5.Sum(i.Pix))
217	case *image.RGBA64:
218		i := img.(*image.RGBA64)
219		hash = fmt.Sprintf("%x", md5.Sum(i.Pix))
220	default:
221		return "", fmt.Errorf("Unknown type of image")
222	}
223
224	p := path.Join(*outDir, hash+".png")
225	outputFile, err := createOutputFile(p)
226	defer outputFile.Close()
227	if err != nil {
228		return "", fmt.Errorf("Could not create png file %s: %s", p, err)
229	}
230	if _, err = outputFile.Write(pngBytes); err != nil {
231		return "", fmt.Errorf("Could not write to file %s: %s", p, err)
232	}
233	return hash, nil
234}
235