1# Copyright 2022 The ChromiumOS Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import contextlib 6from recipe_engine import recipe_api 7 8CROSVM_REPO_URL = "https://chromium.googlesource.com/crosvm/crosvm" 9 10 11class CrosvmApi(recipe_api.RecipeApi): 12 "Crosvm specific functionality shared between recipes." 13 14 @property 15 def source_dir(self): 16 "Where the crosvm source will be checked out." 17 return self.builder_cache.join("crosvm") 18 19 @property 20 def rustup_home(self): 21 "RUSTUP_HOME is cached between runs." 22 return self.builder_cache.join("rustup") 23 24 @property 25 def cargo_home(self): 26 "CARGO_HOME is cached between runs." 27 return self.builder_cache.join("cargo_home") 28 29 @property 30 def cargo_target_dir(self): 31 "CARGO_TARGET_DIR is cleaned up between runs" 32 return self.m.path["cleanup"].join("cargo_target") 33 34 @property 35 def local_bin(self): 36 "Directory used to install local tools required by the build." 37 return self.builder_cache.join("local_bin") 38 39 @property 40 def dev_container_cache(self): 41 return self.builder_cache.join("dev_container") 42 43 @property 44 def builder_cache(self): 45 """ 46 Dedicated cache directory for each builder. 47 48 Luci will try to run each builder on the same bot as previously to keep this cache present. 49 """ 50 return self.m.path["cache"].join("builder") 51 52 def source_context(self): 53 """ 54 Updates the source to the revision to be tested and drops into the source directory. 55 56 Use when no build commands are needed. 57 """ 58 with self.m.context(infra_steps=True): 59 self.__prepare_source() 60 return self.m.context(cwd=self.source_dir) 61 62 def container_build_context(self): 63 """ 64 Prepares source and system to build crosvm via dev container. 65 66 Usage: 67 with api.crosvm.container_build_context(): 68 api.crosvm.step_in_container("build crosvm", ["cargo build"]) 69 """ 70 with self.m.step.nest("Prepare Container Build"): 71 with self.m.context(infra_steps=True): 72 self.__prepare_source() 73 self.__prepare_container() 74 env = { 75 "CROSVM_CONTAINER_CACHE": str(self.dev_container_cache), 76 } 77 return self.m.context(cwd=self.source_dir, env=env) 78 79 def cros_container_build_context(self): 80 """ 81 Prepares source and system to build crosvm via cros container. 82 83 Usage: 84 with api.crosvm.cros_container_build_context(): 85 api.crosvm.step_in_container("build crosvm", ["cargo build"], cros=True) 86 """ 87 with self.m.step.nest("Prepare Cros Container Build"): 88 with self.m.context(infra_steps=True): 89 self.__prepare_source() 90 with self.m.context(cwd=self.source_dir): 91 self.m.step( 92 "Stop existing cros containers", 93 [ 94 "vpython3", 95 self.source_dir.join("tools/dev_container"), 96 "--verbose", 97 "--stop", 98 "--cros", 99 ], 100 ) 101 self.m.step( 102 "Force pull cros_container", 103 [ 104 "vpython3", 105 self.source_dir.join("tools/dev_container"), 106 "--pull", 107 "--cros", 108 ], 109 ) 110 self.m.crosvm.step_in_container("Ensure cros container exists", ["true"], cros=True) 111 return self.m.context(cwd=self.source_dir) 112 113 def host_build_context(self): 114 """ 115 Prepares source and system to build crosvm directly on the host. 116 117 This will install the required rust version via rustup. However no further dependencies 118 are installed. 119 120 Usage: 121 with api.crosvm.host_build_context(): 122 api.step("build crosvm", ["cargo build"]) 123 """ 124 with self.m.step.nest("Prepare Host Build"): 125 with self.m.context(infra_steps=True): 126 self.__prepare_source() 127 env = { 128 "RUSTUP_HOME": str(self.rustup_home), 129 "CARGO_HOME": str(self.cargo_home), 130 "CARGO_TARGET_DIR": str(self.cargo_target_dir), 131 } 132 env_prefixes = { 133 "PATH": [ 134 self.cargo_home.join("bin"), 135 self.local_bin, 136 ], 137 } 138 with self.m.context(env=env, env_prefixes=env_prefixes, cwd=self.source_dir): 139 self.__prepare_rust() 140 self.__prepare_host_depdendencies() 141 142 return self.m.context(env=env, env_prefixes=env_prefixes, cwd=self.source_dir) 143 144 def step_in_container(self, step_name, command, cros=False, **kwargs): 145 """ 146 Runs a luci step inside the crosvm dev container. 147 """ 148 return self.m.step( 149 step_name, 150 [ 151 "vpython3", 152 self.source_dir.join("tools/dev_container"), 153 "--verbose", 154 ] 155 + (["--cros"] if cros else []) 156 + command, 157 **kwargs 158 ) 159 160 def prepare_git(self): 161 with self.m.step.nest("Prepare git"): 162 with self.m.context(cwd=self.m.path["start_dir"]): 163 name = self.m.git.config_get("user.name") 164 email = self.m.git.config_get("user.email") 165 if not name or not email: 166 self.__set_git_config("user.name", "Crosvm Bot") 167 self.__set_git_config( 168 "user.email", "crosvm-bot@crosvm-infra.iam.gserviceaccount.com" 169 ) 170 # Use gcloud for authentication, which will make sure we are interacting with gerrit 171 # using the Luci configured identity. 172 if not self.m.platform.is_win: 173 self.m.step( 174 "Set git config: credential.helper", 175 [ 176 "git", 177 "config", 178 "--global", 179 "--replace-all", 180 "credential.helper", 181 "gcloud.sh", 182 ], 183 ) 184 185 def get_git_sha(self): 186 result = self.m.step( 187 "Get git sha", ["git", "rev-parse", "HEAD"], stdout=self.m.raw_io.output() 188 ) 189 value = result.stdout.strip().decode("utf-8") 190 result.presentation.step_text = value 191 return value 192 193 def upload_coverage(self, filename): 194 with self.m.step.nest("Uploading coverage"): 195 codecov = self.m.cipd.ensure_tool("crosvm/codecov/${platform}", "latest") 196 sha = self.get_git_sha() 197 self.m.step( 198 "Uploading to covecov.io", 199 [ 200 "bash", 201 self.resource("codecov_wrapper.sh"), 202 codecov, 203 "--nonZero", # Enables error codes 204 "--slug=google/crosvm", 205 "--sha=" + sha, 206 "--branch=main", 207 "-X=search", # Don't search for coverage files, just upload the file below. 208 "-f", 209 filename, 210 ], 211 ) 212 213 def __prepare_rust(self): 214 """ 215 Prepares the rust toolchain via rustup. 216 217 Installs rustup-init via CIPD, which is then used to install the rust toolchain version 218 required by the crosvm sources. 219 220 Note: You want to run this after prepare_source to ensure the correct version is installed. 221 """ 222 with self.m.step.nest("Prepare rust"): 223 if not self.m.path.exists( 224 self.cargo_home.join("bin/rustup") 225 ) and not self.m.path.exists(self.cargo_home.join("bin/rustup.exe")): 226 rustup_init = self.m.cipd.ensure_tool("crosvm/rustup-init/${platform}", "latest") 227 self.m.step("Install rustup", [rustup_init, "-y", "--default-toolchain", "none"]) 228 229 if self.m.platform.is_win: 230 self.m.step( 231 "Set rustup default host", 232 ["rustup", "set", "default-host", "x86_64-pc-windows-gnu"], 233 ) 234 235 # Rustup installs a rustc wrapper that will download and use the version specified by 236 # crosvm in the rust-toolchain file. 237 self.m.step("Ensure toolchain is installed", ["rustc", "--version"]) 238 239 def __prepare_host_depdendencies(self): 240 """ 241 Installs additional dependencies of crosvm host-side builds. This is mainly used for 242 builds on windows where the dev container is not available. 243 """ 244 with self.m.step.nest("Prepare host dependencies"): 245 self.m.file.ensure_directory("Ensure local_bin exists", self.local_bin) 246 247 ensure_file = self.m.cipd.EnsureFile() 248 ensure_file.add_package("crosvm/protoc/${platform}", "latest") 249 ensure_file.add_package("crosvm/cargo-nextest/${platform}", "latest") 250 self.m.cipd.ensure(self.local_bin, ensure_file) 251 252 def __sync_submodules(self): 253 with self.m.step.nest("Sync submodules") as sync_step: 254 with self.m.context(cwd=self.source_dir): 255 try: 256 self.m.step( 257 "Init / Update submodules", 258 ["git", "submodule", "update", "--force", "--init"], 259 ) 260 except: 261 # Since the repository is cached between builds, the submodules could be left in 262 # a bad state (e.g. after a previous build is cancelled while syncing). 263 # Repair this by re-initializing the submodules. 264 self.m.step( 265 "De-init submodules", 266 ["git", "submodule", "deinit", "--force", "--all"], 267 ) 268 self.m.step( 269 "Re-init / Update submodules", 270 ["git", "submodule", "update", "--force", "--init"], 271 ) 272 sync_step.step_text = "Repaired submodules." 273 sync_step.status = self.m.step.WARNING 274 275 def __prepare_source(self): 276 """ 277 Prepares the local crosvm source for testing in `self.source_dir` 278 279 CI jobs will check out the revision to be tested, try jobs will check out the gerrit 280 change to be tested. 281 """ 282 self.prepare_git() 283 with self.m.step.nest("Prepare source"): 284 self.m.file.ensure_directory("Ensure builder_cache exists", self.builder_cache) 285 with self.m.context(cwd=self.builder_cache): 286 gclient_config = self.m.gclient.make_config() 287 s = gclient_config.solutions.add() 288 s.url = CROSVM_REPO_URL 289 s.name = "crosvm" 290 gclient_config.got_revision_mapping[s.name] = "got_revision" 291 self.m.bot_update.ensure_checkout(gclient_config=gclient_config) 292 293 self.__sync_submodules() 294 295 # gclient will use a reference to a cache directory, which won't be available inside 296 # the dev container. Repack will make sure all objects are copied into the current 297 # repo. 298 with self.m.context(cwd=self.source_dir): 299 self.m.step("Repack repository", ["git", "repack", "-a"]) 300 301 def __prepare_container(self): 302 with self.m.step.nest("Prepare dev_container"): 303 with self.m.context(cwd=self.source_dir): 304 self.m.step( 305 "Stop existing dev containers", 306 [ 307 "vpython3", 308 self.source_dir.join("tools/dev_container"), 309 "--verbose", 310 "--stop", 311 ], 312 ) 313 self.m.step( 314 "Force pull dev_container", 315 [ 316 "vpython3", 317 self.source_dir.join("tools/dev_container"), 318 "--pull", 319 ], 320 ) 321 self.m.crosvm.step_in_container("Ensure dev container exists", ["true"]) 322 323 def __set_git_config(self, prop, value): 324 self.m.step( 325 "Set git config: %s" % prop, 326 ["git", "config", "--global", prop, value], 327 ) 328