• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2022 The ChromiumOS Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Help creating a Rust ebuild with CRATES.
7
8This script is meant to help someone creating a Rust ebuild of the type
9currently used by sys-apps/ripgrep and sys-apps/rust-analyzer.
10
11In these ebuilds, the CRATES variable is used to list all dependencies, rather
12than creating an ebuild for each dependency. This style of ebuild can be used
13for a crate which is only intended for use in the chromiumos SDK, and which has
14many dependencies which otherwise won't be used.
15
16To create such an ebuild, there are essentially two tasks that must be done:
17
181. Determine all transitive dependent crates and version and list them in the
19CRATES variable. Ignore crates that are already included in the main crate's
20repository.
21
222. Find which dependent crates are not already on a chromeos mirror, retrieve
23them from crates.io, and upload them to `gs://chromeos-localmirror/distfiles`.
24
25This script parses the crate's lockfile to list transitive dependent crates,
26and either lists crates to be uploaded or actually uploads them.
27
28Of course these can be done manually instead. If you choose to do these steps
29manually, I recommend *not* using the `cargo download` tool, and instead obtain
30dependent crates at
31`https://crates.io/api/v1/crates/{crate_name}/{crate_version}/download`.
32
33Example usage:
34
35    # Here we instruct the script to ignore crateA and crateB, presumably
36    # because they are already included in the same repository as some-crate.
37    # This will not actually upload any crates to `gs`.
38    python3 crate_ebuild_help.py --lockfile some-crate/Cargo.lock \
39            --ignore crateA --ignore crateB --dry-run
40
41    # Similar to the above, but here we'll actually carry out the uploads.
42    python3 crate_ebuild_help.py --lockfile some-crate/Cargo.lock \
43            --ignore crateA --ignore crateB
44
45See the ebuild files for ripgrep or rust-analyzer for other details.
46"""
47
48import argparse
49import concurrent.futures
50from pathlib import Path
51import subprocess
52import tempfile
53from typing import List, Tuple
54import urllib.request
55
56# Python 3.11 has `tomllib`, so maybe eventually we can switch to that.
57import toml
58
59
60def run(args: List[str]) -> bool:
61    result = subprocess.run(
62        args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False
63    )
64    return result.returncode == 0
65
66
67def run_check(args: List[str]):
68    subprocess.run(
69        args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
70    )
71
72
73def gs_address_exists(address: str) -> bool:
74    # returns False if the file isn't there
75    return run(["gsutil.py", "ls", address])
76
77
78def crate_already_uploaded(crate_name: str, crate_version: str) -> bool:
79    filename = f"{crate_name}-{crate_version}.crate"
80    return gs_address_exists(
81        f"gs://chromeos-localmirror/distfiles/{filename}"
82    ) or gs_address_exists(f"gs://chromeos-mirror/gentoo/distfiles/{filename}")
83
84
85def download_crate(crate_name: str, crate_version: str, localpath: Path):
86    urllib.request.urlretrieve(
87        f"https://crates.io/api/v1/crates/{crate_name}/{crate_version}/download",
88        localpath,
89    )
90
91
92def upload_crate(crate_name: str, crate_version: str, localpath: Path):
93    run_check(
94        [
95            "gsutil.py",
96            "cp",
97            "-n",
98            "-a",
99            "public-read",
100            str(localpath),
101            f"gs://chromeos-localmirror/distfiles/{crate_name}-{crate_version}.crate",
102        ]
103    )
104
105
106def main():
107    parser = argparse.ArgumentParser(
108        description="Help prepare a Rust crate for an ebuild."
109    )
110    parser.add_argument(
111        "--lockfile",
112        type=str,
113        required=True,
114        help="Path to the lockfile of the crate in question.",
115    )
116    parser.add_argument(
117        "--ignore",
118        type=str,
119        action="append",
120        required=False,
121        default=[],
122        help="Ignore the crate by this name (may be used multiple times).",
123    )
124    parser.add_argument(
125        "--dry-run",
126        action="store_true",
127        help="Don't actually download/upload crates, just print their names.",
128    )
129    ns = parser.parse_args()
130
131    to_ignore = set(ns.ignore)
132
133    toml_contents = toml.load(ns.lockfile)
134    packages = toml_contents["package"]
135
136    crates = [
137        (pkg["name"], pkg["version"])
138        for pkg in packages
139        if pkg["name"] not in to_ignore
140    ]
141    crates.sort()
142
143    print("Dependent crates:")
144    for name, version in crates:
145        print(f"{name}-{version}")
146    print()
147
148    if ns.dry_run:
149        print("Crates that would be uploaded (skipping ones already uploaded):")
150    else:
151        print("Uploading crates (skipping ones already uploaded):")
152
153    def maybe_upload(crate: Tuple[str, str]) -> str:
154        name, version = crate
155        if crate_already_uploaded(name, version):
156            return ""
157        if not ns.dry_run:
158            with tempfile.TemporaryDirectory() as temp_dir:
159                path = Path(temp_dir.name, f"{name}-{version}.crate")
160                download_crate(name, version, path)
161                upload_crate(name, version, path)
162        return f"{name}-{version}"
163
164    # Simple benchmarking on my machine with rust-analyzer's Cargo.lock, using
165    # the --dry-run option, gives a wall time of 277 seconds with max_workers=1
166    # and 70 seconds with max_workers=4.
167    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
168        crates_len = len(crates)
169        for i, s in enumerate(executor.map(maybe_upload, crates)):
170            if s:
171                j = i + 1
172                print(f"[{j}/{crates_len}] {s}")
173    print()
174
175
176if __name__ == "__main__":
177    main()
178