• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2022 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5package main
6
7import (
8	"flag"
9	"fmt"
10	"io"
11	"os"
12	"os/exec"
13	"path/filepath"
14	"regexp"
15	"sort"
16	"strings"
17)
18
19type depConfig struct {
20	bazelNameOverride string // Bazel style uses underscores not dashes, so we fix those if needed.
21	needsBazelFile    bool
22	patchCmds         []string
23}
24
25// These are all C++ deps or Rust deps (with a compatible C++ FFI) used by the Bazel build.
26// They are a subset of those listed in DEPS.
27// The key is the name of the repo as specified in DEPS.
28var deps = map[string]depConfig{
29	"abseil-cpp":  {bazelNameOverride: "abseil_cpp"},
30	"brotli":      {},
31	"highway":     {},
32	"spirv-tools": {bazelNameOverride: "spirv_tools"},
33	// This name is important because spirv_tools expects @spirv_headers to exist by that name.
34	"spirv-headers": {bazelNameOverride: "spirv_headers"},
35
36	"dawn":     {needsBazelFile: true},
37	"dng_sdk":  {needsBazelFile: true},
38	"expat":    {needsBazelFile: true},
39	"freetype": {needsBazelFile: true},
40	"harfbuzz": {needsBazelFile: true},
41	"icu": {
42		needsBazelFile: true,
43		patchCmds: []string{`"rm source/i18n/BUILD.bazel"`,
44			`"rm source/common/BUILD.bazel"`,
45			`"rm source/stubdata/BUILD.bazel"`},
46	},
47	"icu4x":                    {needsBazelFile: true},
48	"imgui":                    {needsBazelFile: true},
49	"libavif":                  {needsBazelFile: true},
50	"libgav1":                  {needsBazelFile: true},
51	"libjpeg-turbo":            {bazelNameOverride: "libjpeg_turbo", needsBazelFile: true},
52	"libjxl":                   {needsBazelFile: true},
53	"libpng":                   {needsBazelFile: true},
54	"libwebp":                  {needsBazelFile: true},
55	"libyuv":                   {needsBazelFile: true},
56	"spirv-cross":              {bazelNameOverride: "spirv_cross", needsBazelFile: true},
57	"perfetto":                 {needsBazelFile: true},
58	"piex":                     {needsBazelFile: true},
59	"vello":                    {needsBazelFile: true},
60	"vulkan-headers":           {bazelNameOverride: "vulkan_headers", needsBazelFile: true},
61	"vulkan-tools":             {bazelNameOverride: "vulkan_tools", needsBazelFile: true},
62	"vulkan-utility-libraries": {bazelNameOverride: "vulkan_utility_libraries", needsBazelFile: true},
63	"vulkanmemoryallocator":    {needsBazelFile: true},
64	"wuffs":                    {needsBazelFile: true},
65	// Some other dependency downloads zlib but with their own rules
66	"zlib": {bazelNameOverride: "zlib_skia", needsBazelFile: true},
67}
68
69func main() {
70	var (
71		depsFile      = flag.String("deps_file", "DEPS", "The location of the DEPS file. Usually at the root of the repository")
72		genBzlFile    = flag.String("gen_bzl_file", "bazel/deps.bzl", "The location of the .bzl file that has the generated Bazel repository rules.")
73		workspaceFile = flag.String("workspace_file", "WORKSPACE.bazel", "The location of the WORKSPACE file that should be updated with dep names.")
74		// https://bazel.build/docs/user-manual#running-executables
75		repoDir        = flag.String("repo_dir", os.Getenv("BUILD_WORKSPACE_DIRECTORY"), "The root directory of the repo. Default set by BUILD_WORKSPACE_DIRECTORY env variable.")
76		buildifierPath = flag.String("buildifier", "", "Where to find buildifier. Defaults to Bazel's location")
77	)
78	flag.Parse()
79
80	if *repoDir == "" {
81		fmt.Println(`Must set --repo_dir
82This is done automatically via:
83    bazel run //bazel/deps_parser`)
84		os.Exit(1)
85	}
86
87	buildifier := *buildifierPath
88	if buildifier == "" {
89		// We don't know if this will be buildifier_linux_x64, buildifier_macos_arm64, etc
90		bp, err := filepath.Glob("../buildifier*/file/buildifier")
91		if err != nil || len(bp) != 1 {
92			fmt.Printf("Could not find exactly one buildifier executable %s %v\n", err, bp)
93			os.Exit(1)
94		}
95		buildifier = bp[0]
96	}
97	buildifier, err := filepath.Abs(buildifier)
98	if err != nil {
99		fmt.Printf("Abs path error %s\n", err)
100		os.Exit(1)
101	}
102
103	fmt.Println(os.Environ())
104
105	if *depsFile == "" || *genBzlFile == "" {
106		fmt.Println("Must set --deps_file and --gen_bzl_file")
107		flag.PrintDefaults()
108	}
109
110	if err := os.Chdir(*repoDir); err != nil {
111		fmt.Printf("Could not cd to %s\n", *repoDir)
112		os.Exit(1)
113	}
114
115	b, err := os.ReadFile(*depsFile)
116	if err != nil {
117		fmt.Printf("Could not open %s: %s\n", *depsFile, err)
118		os.Exit(1)
119	}
120	contents := strings.Split(string(b), "\n")
121
122	outputFile, count, err := parseDEPSFile(contents, *workspaceFile)
123	if err != nil {
124		fmt.Printf("Parsing error %s\n", err)
125		os.Exit(1)
126	}
127	if err := exec.Command(buildifier, outputFile).Run(); err != nil {
128		fmt.Printf("Buildifier error %s\n", err)
129		os.Exit(1)
130	}
131	if err := os.Rename(outputFile, *genBzlFile); err != nil {
132		fmt.Printf("Could not write from %s to %s: %s\n", outputFile, *depsFile, err)
133		os.Exit(1)
134	}
135	fmt.Printf("Wrote %d deps\n", count)
136}
137
138func parseDEPSFile(contents []string, workspaceFile string) (string, int, error) {
139	depsLine := regexp.MustCompile(`externals/(\S+)".+"(https.+)@([a-f0-9]+)"`)
140	outputFile, err := os.CreateTemp("", "genbzl")
141	if err != nil {
142		return "", 0, fmt.Errorf("Could not create output file: %s\n", err)
143	}
144	defer outputFile.Close()
145
146	if _, err := outputFile.WriteString(header); err != nil {
147		return "", 0, fmt.Errorf("Could not write header to output file %s: %s\n", outputFile.Name(), err)
148	}
149
150	var nativeRepos []string
151	var providedRepos []string
152
153	count := 0
154	for _, line := range contents {
155		if match := depsLine.FindStringSubmatch(line); len(match) > 0 {
156			id := match[1]
157			repo := match[2]
158			rev := match[3]
159
160			cfg, ok := deps[id]
161			if !ok {
162				continue
163			}
164			if cfg.bazelNameOverride != "" {
165				id = cfg.bazelNameOverride
166			}
167			if cfg.needsBazelFile {
168				if err := writeNewGitRepositoryRule(outputFile, id, repo, rev, cfg.patchCmds); err != nil {
169					return "", 0, fmt.Errorf("Could not write to output file %s: %s\n", outputFile.Name(), err)
170				}
171				workspaceLine := fmt.Sprintf("# @%s - //bazel/external/%s:BUILD.bazel", id, id)
172				providedRepos = append(providedRepos, workspaceLine)
173			} else {
174				if err := writeGitRepositoryRule(outputFile, id, repo, rev); err != nil {
175					return "", 0, fmt.Errorf("Could not write to output file %s: %s\n", outputFile.Name(), err)
176				}
177				workspaceLine := fmt.Sprintf("# @%s - %s", id, repo)
178				nativeRepos = append(nativeRepos, workspaceLine)
179			}
180			count++
181		}
182	}
183	if count != len(deps) {
184		return "", 0, fmt.Errorf("Not enough deps written. Maybe the deps dictionary needs a bazelNameOverride or an old dep needs to be removed?")
185	}
186
187	if _, err := outputFile.WriteString(footer); err != nil {
188		return "", 0, fmt.Errorf("Could not write footer to output file %s: %s\n", outputFile.Name(), err)
189	}
190
191	if newWorkspaceFile, err := writeCommentsToWorkspace(workspaceFile, nativeRepos, providedRepos); err != nil {
192		fmt.Printf("Could not parse workspace file %s: %s\n", workspaceFile, err)
193		os.Exit(1)
194	} else {
195		// Atomically rename temp file to workspace. This should minimize the chance of corruption
196		// or writing a partial file if there is an error or the program is interrupted.
197		if err := os.Rename(newWorkspaceFile, workspaceFile); err != nil {
198			fmt.Printf("Could not write comments in workspace file %s -> %s: %s\n", newWorkspaceFile, workspaceFile, err)
199			os.Exit(1)
200		}
201	}
202	return outputFile.Name(), count, nil
203}
204
205func writeCommentsToWorkspace(workspaceFile string, nativeRepos, providedRepos []string) (string, error) {
206	b, err := os.ReadFile(workspaceFile)
207	if err != nil {
208		return "", fmt.Errorf("Could not open %s: %s\n", workspaceFile, err)
209	}
210	newWorkspace, err := os.CreateTemp("", "workspace")
211	if err != nil {
212		return "", fmt.Errorf("Could not make tempfile: %s\n", err)
213	}
214	defer newWorkspace.Close()
215
216	workspaceContents := strings.Split(string(b), "\n")
217
218	sort.Strings(nativeRepos)
219	sort.Strings(providedRepos)
220	for _, line := range workspaceContents {
221		if _, err := newWorkspace.WriteString(line + "\n"); err != nil {
222			return "", err
223		}
224		if line == startListString {
225			break
226		}
227	}
228	for _, repoLine := range nativeRepos {
229		if _, err := newWorkspace.WriteString(repoLine + "\n"); err != nil {
230			return "", err
231		}
232	}
233	if _, err := newWorkspace.WriteString("#\n"); err != nil {
234		return "", err
235	}
236	for _, repoLine := range providedRepos {
237		if _, err := newWorkspace.WriteString(repoLine + "\n"); err != nil {
238			return "", err
239		}
240	}
241	if _, err := newWorkspace.WriteString(endListString + "\n"); err != nil {
242		return "", err
243	}
244
245	pastEnd := false
246	// Skip the last line, which is blank. We don't want to end with two empty newlines.
247	for _, line := range workspaceContents[:len(workspaceContents)-1] {
248		if line == endListString {
249			pastEnd = true
250			continue
251		}
252		if !pastEnd {
253			continue
254		}
255		if _, err := newWorkspace.WriteString(line + "\n"); err != nil {
256			return "", err
257		}
258	}
259
260	return newWorkspace.Name(), nil
261}
262
263const (
264	startListString = `#### START GENERATED LIST OF THIRD_PARTY DEPS`
265	endListString   = `#### END GENERATED LIST OF THIRD_PARTY DEPS`
266)
267
268const header = `"""
269This file is auto-generated from //bazel/deps_parser
270DO NOT MODIFY BY HAND.
271Instead, do:
272    bazel run //bazel/deps_parser
273"""
274
275load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
276load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
277load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
278load("//bazel:download_config_files.bzl", "download_config_files")
279load("//bazel:gcs_mirror.bzl", "gcs_mirror_url")
280
281def c_plus_plus_deps(ws = "@skia"):
282    """A list of native Bazel git rules to download third party git repositories
283
284       These are in the order they appear in //DEPS.
285        https://bazel.build/rules/lib/repo/git
286
287    Args:
288      ws: The name of the Skia Bazel workspace. The default, "@", may be when used from within the
289          Skia workspace.
290    """`
291
292// If necessary, we can make a new map for bazel deps
293const footer = `
294def bazel_deps():
295    maybe(
296        http_archive,
297        name = "bazel_skylib",
298        sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d",
299        urls = gcs_mirror_url(
300            sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d",
301            url = "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz",
302        ),
303    )
304
305    maybe(
306        http_archive,
307        name = "bazel_toolchains",
308        sha256 = "e52789d4e89c3e2dc0e3446a9684626a626b6bec3fde787d70bae37c6ebcc47f",
309        strip_prefix = "bazel-toolchains-5.1.1",
310        urls = gcs_mirror_url(
311            sha256 = "e52789d4e89c3e2dc0e3446a9684626a626b6bec3fde787d70bae37c6ebcc47f",
312            url = "https://github.com/bazelbuild/bazel-toolchains/archive/refs/tags/v5.1.1.tar.gz",
313        ),
314    )
315
316def header_based_configs():
317    maybe(
318        download_config_files,
319        name = "expat_config",
320        skia_revision = "7b730016006e6b66d24a6f94eefe8bec00ac1674",
321        files = {
322            "BUILD.bazel": "bazel/external/expat/config/BUILD.bazel",
323            "expat_config.h": "third_party/expat/include/expat_config/expat_config.h",
324        },
325    )
326    maybe(
327        download_config_files,
328        name = "freetype_config",
329        skia_revision = "7b730016006e6b66d24a6f94eefe8bec00ac1674",
330        files = {
331            "BUILD.bazel": "bazel/external/freetype/config/BUILD.bazel",
332            "android/freetype/config/ftmodule.h": "third_party/freetype2/include/freetype-android/freetype/config/ftmodule.h",
333            "android/freetype/config/ftoption.h": "third_party/freetype2/include/freetype-android/freetype/config/ftoption.h",
334            "no-type1/freetype/config/ftmodule.h": "third_party/freetype2/include/freetype-no-type1/freetype/config/ftmodule.h",
335            "no-type1/freetype/config/ftoption.h": "third_party/freetype2/include/freetype-no-type1/freetype/config/ftoption.h",
336        },
337    )
338    maybe(
339        download_config_files,
340        name = "harfbuzz_config",
341        skia_revision = "7b730016006e6b66d24a6f94eefe8bec00ac1674",
342        files = {
343            "BUILD.bazel": "bazel/external/harfbuzz/config/BUILD.bazel",
344            "config-override.h": "third_party/harfbuzz/config-override.h",
345        },
346    )
347    maybe(
348        download_config_files,
349        name = "icu_utils",
350        skia_revision = "7b730016006e6b66d24a6f94eefe8bec00ac1674",
351        files = {
352            "BUILD.bazel": "bazel/external/icu/utils/BUILD.bazel",
353            "icu/SkLoadICU.cpp": "third_party/icu/SkLoadICU.cpp",
354            "icu/SkLoadICU.h": "third_party/icu/SkLoadICU.h",
355            "icu/make_data_cpp.py": "third_party/icu/make_data_cpp.py",
356        },
357    )
358`
359
360func writeNewGitRepositoryRule(w io.StringWriter, bazelName, repo, rev string, patchCmds []string) error {
361	if len(patchCmds) == 0 {
362		// TODO(kjlubick) In a newer version of Bazel, new_git_repository can be replaced with just
363		// git_repository
364		_, err := w.WriteString(fmt.Sprintf(`
365    new_git_repository(
366        name = "%s",
367        build_file = ws + "//bazel/external/%s:BUILD.bazel",
368        commit = "%s",
369        remote = "%s",
370    )
371`, bazelName, bazelName, rev, repo))
372		return err
373	}
374	patches := "[" + strings.Join(patchCmds, ",\n") + "]"
375	_, err := w.WriteString(fmt.Sprintf(`
376    new_git_repository(
377        name = "%s",
378        build_file = ws + "//bazel/external/%s:BUILD.bazel",
379        commit = "%s",
380        remote = "%s",
381        patch_cmds = %s,
382    )
383`, bazelName, bazelName, rev, repo, patches))
384	return err
385}
386
387func writeGitRepositoryRule(w io.StringWriter, bazelName, repo, rev string) error {
388	_, err := w.WriteString(fmt.Sprintf(`
389    git_repository(
390        name = "%s",
391        commit = "%s",
392        remote = "%s",
393    )
394`, bazelName, rev, repo))
395	return err
396}
397