• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2022 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5// This executable downloads, verifies, and uploads a given file to the Skia infra Bazel mirror.
6// Users should have gsutil installed, on the PATH and authenticated.
7// There are two modes of use:
8//   - Specify a single file via --url and --sha256.
9//   - Copy a JSON array of objects (or Starlark list of dictionaries) via standard in.
10// This should only need to be called when we add new dependencies or update existing ones. Calling
11// it with already archived files should be fine - the mirror is a CAS, so the update should be a
12// no-op. The files will be uploaded to the mirror with some metadata about where they came from.
13package main
14
15import (
16	"crypto/sha256"
17	"encoding/hex"
18	"flag"
19	"fmt"
20	"io"
21	"net/http"
22	"os"
23	"os/exec"
24	"path/filepath"
25	"strings"
26
27	"github.com/flynn/json5"
28
29	"go.skia.org/infra/go/skerr"
30)
31
32const (
33	gcsBucketAndPrefix = "gs://skia-world-readable/bazel/"
34)
35
36func main() {
37	var (
38		file          = flag.String("file", "", "A local file on disk to upload. --sha256 must be set.")
39		url           = flag.String("url", "", "The single url to mirror. --sha256 must be set.")
40		sha256Hash    = flag.String("sha256", "", "The sha256sum of the url to mirror. --url must also be set.")
41		jsonFromStdin = flag.Bool("json", false, "If set, read JSON from stdin that consists of a list of objects.")
42		noSuffix      = flag.Bool("no_suffix", false, "If true, this is presumed to be a binary which needs no suffix (e.g. executable)")
43		addSuffix     = flag.String("add_suffix", "", "If set, this will be the suffix of the file uploaded")
44	)
45	flag.Parse()
46
47	if (*file != "" && *sha256Hash != "") || (*url != "" && *sha256Hash != "") {
48		// ok
49	} else if *jsonFromStdin {
50		// ok
51	} else {
52		flag.Usage()
53		fatalf("Must specify --url/--file and --sha256 or --json")
54	}
55
56	workDir, err := os.MkdirTemp("", "bazel_gcs")
57	if err != nil {
58		fatalf("Could not make temp directory: %s", err)
59	}
60
61	if *jsonFromStdin {
62		fmt.Println("Waiting for input on std in. Use Ctrl+D (EOF) when done copying and pasting the array.")
63		b, err := io.ReadAll(os.Stdin)
64		if err != nil {
65			fatalf("Error while reading from stdin: %s", err)
66		}
67		if err := processJSON(workDir, b); err != nil {
68			fatalf("Could not process data from stdin: %s", err)
69		}
70	} else if *url != "" {
71		if err := processOneDownload(workDir, *url, *sha256Hash, *addSuffix, *noSuffix); err != nil {
72			fatalf("Error while processing entry: %s", err)
73		}
74		fmt.Printf("https://storage.googleapis.com/skia-world-readable/bazel/%s%s%s\n", *sha256Hash, getSuffix(*url), *addSuffix)
75	} else {
76		if err := processOneLocalFile(*file, *sha256Hash); err != nil {
77			fatalf("Error while processing entry: %s", err)
78		}
79		fmt.Printf("https://storage.googleapis.com/skia-world-readable/bazel/%s%s\n", *sha256Hash, getSuffix(*file))
80	}
81}
82
83type urlEntry struct {
84	SHA256 string `json:"sha256"`
85	URL    string `json:"url"`
86}
87
88func processJSON(workDir string, b []byte) error {
89	// We generally will be copying a list from Bazel files, written with Starlark (i.e. Pythonish).
90	// As a result, we need to turn the almost valid JSON array of objects into actually valid JSON.
91	// It is easier to just do string replacing rather than going line by line to remove the
92	// troublesome comments.
93	cleaned := fixStarlarkComments(b)
94	var entries []urlEntry
95	if err := json5.Unmarshal([]byte(cleaned), &entries); err != nil {
96		return skerr.Wrapf(err, "unmarshalling JSON")
97	}
98	for _, entry := range entries {
99		if err := processOneDownload(workDir, entry.URL, entry.SHA256, "", false); err != nil {
100			return skerr.Wrapf(err, "while processing entry: %+v", entry)
101		}
102	}
103	return nil
104}
105
106// fixStarlarkComments replaces the Starlark comment symbol (#) with a JSON comment symbol (//).
107func fixStarlarkComments(b []byte) string {
108	return strings.ReplaceAll(string(b), "#", "//")
109}
110
111func processOneDownload(workDir, url, hash, addSuffix string, noSuffix bool) error {
112	suf := getSuffix(url) + addSuffix
113	if !noSuffix && suf == "" {
114		return skerr.Fmt("%s is not a supported file type", url)
115	}
116	fmt.Printf("Downloading and verifying %s...\n", url)
117	res, err := http.Get(url)
118	if err != nil {
119		return skerr.Wrapf(err, "downloading %s", url)
120	}
121	contents, err := io.ReadAll(res.Body)
122	if err != nil {
123		return skerr.Wrapf(err, "reading %s", url)
124	}
125	if err := res.Body.Close(); err != nil {
126		return skerr.Wrapf(err, "after reading %s", url)
127	}
128	// Verify
129	h := sha256.Sum256(contents)
130	if actual := hex.EncodeToString(h[:]); actual != hash {
131		return skerr.Fmt("Invalid hash of %s. %s != %s", url, actual, hash)
132	}
133	fmt.Printf("Uploading %s to GCS...\n", url)
134	// Write to disk so gsutil can access it
135	tmpFile := filepath.Join(workDir, hash+suf)
136	if err := os.WriteFile(tmpFile, contents, 0644); err != nil {
137		return skerr.Wrapf(err, "writing %d bytes to %s", len(contents), tmpFile)
138	}
139	// Upload using gsutil (which is assumed to be properly authed)
140	cmd := exec.Command("gsutil",
141		// Add custom metadata so we can figure out what the unrecognizable file name was created
142		// from. Custom metadata values must start with x-goog-meta-
143		"-h", "x-goog-meta-original-url:"+url,
144		"cp", tmpFile, gcsBucketAndPrefix+hash+suf)
145	cmd.Stdout = os.Stdout
146	cmd.Stderr = os.Stderr
147	return skerr.Wrapf(cmd.Run(), "uploading %s to GCS", tmpFile)
148}
149
150func processOneLocalFile(file, hash string) error {
151	file, err := filepath.Abs(file)
152	if err != nil {
153		return skerr.Wrap(err)
154	}
155	suf := getSuffix(file)
156	if suf == "" {
157		return skerr.Fmt("%s is not a supported file type", file)
158	}
159	contents, err := os.ReadFile(file)
160	if err != nil {
161		return skerr.Wrapf(err, "reading %s", file)
162	}
163	// Verify
164	h := sha256.Sum256(contents)
165	if actual := hex.EncodeToString(h[:]); actual != hash {
166		return skerr.Fmt("Invalid hash of %s. %s != %s", file, actual, hash)
167	}
168	fmt.Printf("Uploading %s to GCS...\n", file)
169	// Upload using gsutil (which is assumed to be properly authed)
170	cmd := exec.Command("gsutil",
171		// Add custom metadata so we can figure out what the unrecognizable file name was created
172		// from. Custom metadata values must start with x-goog-meta-
173		"-h", "x-goog-meta-original-file:"+file,
174		"cp", file, gcsBucketAndPrefix+hash+suf)
175	cmd.Stdout = os.Stdout
176	cmd.Stderr = os.Stderr
177	return skerr.Wrapf(cmd.Run(), "uploading %s to GCS", file)
178}
179
180var supportedSuffixes = []string{".tar.gz", ".tgz", ".tar.xz", ".deb", ".zip"}
181
182// getSuffix returns the filetype suffix of the file if it is in the list of supported suffixes.
183// Otherwise, it returns empty string.
184func getSuffix(url string) string {
185	for _, suf := range supportedSuffixes {
186		if strings.HasSuffix(url, suf) {
187			return suf
188		}
189	}
190	return ""
191}
192
193func fatalf(format string, args ...interface{}) {
194	// Ensure there is a newline at the end of the fatal message.
195	format = strings.TrimSuffix(format, "\n") + "\n"
196	fmt.Printf(format, args...)
197	os.Exit(1)
198}
199