1#!/usr/bin/env python 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""Gcloud setup runner.""" 17 18from __future__ import print_function 19import logging 20import os 21import re 22import subprocess 23 24import six 25 26from acloud import errors 27from acloud.internal.lib import utils 28from acloud.public import config 29from acloud.setup import base_task_runner 30from acloud.setup import google_sdk 31 32 33logger = logging.getLogger(__name__) 34 35# APIs that need to be enabled for GCP project. 36_ANDROID_BUILD_SERVICE = "androidbuildinternal.googleapis.com" 37_ANDROID_BUILD_MSG = ( 38 "This service (%s) help to download images from Android Build. If it isn't " 39 "enabled, acloud only supports local images to create AVD." 40 % _ANDROID_BUILD_SERVICE) 41_COMPUTE_ENGINE_SERVICE = "compute.googleapis.com" 42_COMPUTE_ENGINE_MSG = ( 43 "This service (%s) help to create instance in google cloud platform. If it " 44 "isn't enabled, acloud can't work anymore." % _COMPUTE_ENGINE_SERVICE) 45_OPEN_SERVICE_FAILED_MSG = ( 46 "\n[Open Service Failed]\n" 47 "Service name: %(service_name)s\n" 48 "%(service_msg)s\n") 49 50_BUILD_SERVICE_ACCOUNT = "android-build-prod@system.gserviceaccount.com" 51_BILLING_ENABLE_MSG = "billingEnabled: true" 52_DEFAULT_SSH_FOLDER = os.path.expanduser("~/.ssh") 53_DEFAULT_SSH_KEY = "acloud_rsa" 54_DEFAULT_SSH_PRIVATE_KEY = os.path.join(_DEFAULT_SSH_FOLDER, 55 _DEFAULT_SSH_KEY) 56_DEFAULT_SSH_PUBLIC_KEY = os.path.join(_DEFAULT_SSH_FOLDER, 57 _DEFAULT_SSH_KEY + ".pub") 58_ENV_CLOUDSDK_PYTHON = "CLOUDSDK_PYTHON" 59_GCLOUD_COMPONENT_ALPHA = "alpha" 60# Regular expression to get project/zone information. 61_PROJECT_RE = re.compile(r"^project = (?P<project>.+)") 62_ZONE_RE = re.compile(r"^zone = (?P<zone>.+)") 63 64 65def UpdateConfigFile(config_path, item, value): 66 """Update config data. 67 68 Case A: config file contain this item. 69 In config, "project = A_project". New value is B_project 70 Set config "project = B_project". 71 Case B: config file didn't contain this item. 72 New value is B_project. 73 Setup config as "project = B_project". 74 75 Args: 76 config_path: String, acloud config path. 77 item: String, item name in config file. EX: project, zone 78 value: String, value of item in config file. 79 80 TODO(111574698): Refactor this to minimize writes to the config file. 81 TODO(111574698): Use proto method to update config. 82 """ 83 write_lines = [] 84 find_item = False 85 write_line = item + ": \"" + value + "\"\n" 86 if os.path.isfile(config_path): 87 with open(config_path, "r") as cfg_file: 88 for read_line in cfg_file.readlines(): 89 if read_line.startswith(item + ":"): 90 find_item = True 91 write_lines.append(write_line) 92 else: 93 write_lines.append(read_line) 94 if not find_item: 95 write_lines.append(write_line) 96 with open(config_path, "w") as cfg_file: 97 cfg_file.writelines(write_lines) 98 99 100def SetupSSHKeys(config_path, private_key_path, public_key_path): 101 """Setup the pair of the ssh key for acloud.config. 102 103 User can use the default path: "~/.ssh/acloud_rsa". 104 105 Args: 106 config_path: String, acloud config path. 107 private_key_path: Path to the private key file. 108 e.g. ~/.ssh/acloud_rsa 109 public_key_path: Path to the public key file. 110 e.g. ~/.ssh/acloud_rsa.pub 111 """ 112 private_key_path = os.path.expanduser(private_key_path) 113 if (private_key_path == "" or public_key_path == "" 114 or private_key_path == _DEFAULT_SSH_PRIVATE_KEY): 115 utils.CreateSshKeyPairIfNotExist(_DEFAULT_SSH_PRIVATE_KEY, 116 _DEFAULT_SSH_PUBLIC_KEY) 117 UpdateConfigFile(config_path, "ssh_private_key_path", 118 _DEFAULT_SSH_PRIVATE_KEY) 119 UpdateConfigFile(config_path, "ssh_public_key_path", 120 _DEFAULT_SSH_PUBLIC_KEY) 121 122 123def _InputIsEmpty(input_string): 124 """Check input string is empty. 125 126 Tool requests user to input client ID & client secret. 127 This basic check can detect user input is empty. 128 129 Args: 130 input_string: String, user input string. 131 132 Returns: 133 Boolean: True if input is empty, False otherwise. 134 """ 135 if input_string is None: 136 return True 137 if input_string == "": 138 print("Please enter a non-empty value.") 139 return True 140 return False 141 142 143class GoogleSDKBins(): 144 """Class to run tools in the Google SDK.""" 145 146 def __init__(self, google_sdk_folder): 147 """GoogleSDKBins initialize. 148 149 Args: 150 google_sdk_folder: String, google sdk path. 151 """ 152 self.gcloud_command_path = os.path.join(google_sdk_folder, "gcloud") 153 self.gsutil_command_path = os.path.join(google_sdk_folder, "gsutil") 154 self._env = os.environ.copy() 155 self._env[_ENV_CLOUDSDK_PYTHON] = "python" 156 157 def RunGcloud(self, cmd, **kwargs): 158 """Run gcloud command. 159 160 Args: 161 cmd: String list, command strings. 162 Ex: [config], then this function call "gcloud config". 163 **kwargs: dictionary of keyword based args to pass to func. 164 165 Returns: 166 String, return message after execute gcloud command. 167 """ 168 return utils.CheckOutput([self.gcloud_command_path] + cmd, 169 env=self._env, **kwargs) 170 171 def RunGsutil(self, cmd, **kwargs): 172 """Run gsutil command. 173 174 Args: 175 cmd : String list, command strings. 176 Ex: [list], then this function call "gsutil list". 177 **kwargs: dictionary of keyword based args to pass to func. 178 179 Returns: 180 String, return message after execute gsutil command. 181 """ 182 return utils.CheckOutput([self.gsutil_command_path] + cmd, 183 env=self._env, **kwargs) 184 185 186class GoogleAPIService(): 187 """Class to enable api service in the gcp project.""" 188 189 def __init__(self, service_name, error_msg, required=False): 190 """GoogleAPIService initialize. 191 192 Args: 193 service_name: String, name of api service. 194 error_msg: String, show messages if api service enable failed. 195 required: Boolean, True for service must be enabled for acloud. 196 """ 197 self._name = service_name 198 self._error_msg = error_msg 199 self._required = required 200 201 def EnableService(self, gcloud_runner): 202 """Enable api service. 203 204 Args: 205 gcloud_runner: A GcloudRunner class to run "gcloud" command. 206 """ 207 try: 208 gcloud_runner.RunGcloud(["services", "enable", self._name], 209 stderr=subprocess.STDOUT) 210 except subprocess.CalledProcessError as error: 211 self.ShowFailMessages(error.output) 212 213 def ShowFailMessages(self, error): 214 """Show fail messages. 215 216 Show the fail messages to hint users the impact if the api service 217 isn't enabled. 218 219 Args: 220 error: String of error message when opening api service failed. 221 """ 222 msg_color = (utils.TextColors.FAIL if self._required else 223 utils.TextColors.WARNING) 224 utils.PrintColorString( 225 error + _OPEN_SERVICE_FAILED_MSG % { 226 "service_name": self._name, 227 "service_msg": self._error_msg} 228 , msg_color) 229 230 @property 231 def name(self): 232 """Return name.""" 233 return self._name 234 235 236class GcpTaskRunner(base_task_runner.BaseTaskRunner): 237 """Runner to setup google cloud user information.""" 238 239 WELCOME_MESSAGE_TITLE = "Setup google cloud user information" 240 WELCOME_MESSAGE = ( 241 "This step will walk you through gcloud SDK installation." 242 "Then configure gcloud user information." 243 "Finally enable some gcloud API services.") 244 245 def __init__(self, config_path): 246 """Initialize parameters. 247 248 Load config file to get current values. 249 250 Args: 251 config_path: String, acloud config path. 252 """ 253 # pylint: disable=invalid-name 254 config_mgr = config.AcloudConfigManager(config_path) 255 cfg = config_mgr.Load() 256 self.config_path = config_mgr.user_config_path 257 self.project = cfg.project 258 self.zone = cfg.zone 259 self.ssh_private_key_path = cfg.ssh_private_key_path 260 self.ssh_public_key_path = cfg.ssh_public_key_path 261 self.stable_host_image_name = cfg.stable_host_image_name 262 self.client_id = cfg.client_id 263 self.client_secret = cfg.client_secret 264 self.service_account_name = cfg.service_account_name 265 self.service_account_private_key_path = cfg.service_account_private_key_path 266 self.service_account_json_private_key_path = cfg.service_account_json_private_key_path 267 268 def ShouldRun(self): 269 """Check if we actually need to run GCP setup. 270 271 We'll only do the gcp setup if certain fields in the cfg are empty. 272 273 Returns: 274 True if reqired config fields are empty, False otherwise. 275 """ 276 # We need to ensure the config has the proper auth-related fields set, 277 # so config requires just 1 of the following: 278 # 1. client id/secret 279 # 2. service account name/private key path 280 # 3. service account json private key path 281 if ((not self.client_id or not self.client_secret) 282 and (not self.service_account_name or not self.service_account_private_key_path) 283 and not self.service_account_json_private_key_path): 284 return True 285 286 # If a project isn't set, then we need to run setup. 287 return not self.project 288 289 def _Run(self): 290 """Run GCP setup task.""" 291 self._SetupGcloudInfo() 292 SetupSSHKeys(self.config_path, self.ssh_private_key_path, 293 self.ssh_public_key_path) 294 295 def _SetupGcloudInfo(self): 296 """Setup Gcloud user information. 297 1. Setup Gcloud SDK tools. 298 2. Setup Gcloud project. 299 a. Setup Gcloud project and zone. 300 b. Setup Client ID and Client secret. 301 c. Setup Google Cloud Storage bucket. 302 3. Enable Gcloud API services. 303 """ 304 google_sdk_init = google_sdk.GoogleSDK() 305 try: 306 google_sdk_runner = GoogleSDKBins(google_sdk_init.GetSDKBinPath()) 307 google_sdk_init.InstallGcloudComponent(google_sdk_runner, 308 _GCLOUD_COMPONENT_ALPHA) 309 self._SetupProject(google_sdk_runner) 310 self._EnableGcloudServices(google_sdk_runner) 311 self._CreateStableHostImage() 312 finally: 313 google_sdk_init.CleanUp() 314 315 def _CreateStableHostImage(self): 316 """Create the stable host image.""" 317 # Write default stable_host_image_name with unused value. 318 # TODO(113091773): An additional step to create the host image. 319 if not self.stable_host_image_name: 320 UpdateConfigFile(self.config_path, "stable_host_image_name", "") 321 322 323 def _NeedProjectSetup(self): 324 """Confirm project setup should run or not. 325 326 If the project settings (project name and zone) are blank (either one), 327 we'll run the project setup flow. If they are set, we'll check with 328 the user if they want to update them. 329 330 Returns: 331 Boolean: True if we need to setup the project, False otherwise. 332 """ 333 user_question = ( 334 "Your default Project/Zone settings are:\n" 335 "project:[%s]\n" 336 "zone:[%s]\n" 337 "Would you like to update them?[y/N]: \n") % (self.project, self.zone) 338 339 if not self.project or not self.zone: 340 logger.info("Project or zone is empty. Start to run setup process.") 341 return True 342 return utils.GetUserAnswerYes(user_question) 343 344 def _NeedClientIDSetup(self, project_changed): 345 """Confirm client setup should run or not. 346 347 If project changed, client ID must also have to change. 348 So tool will force to run setup function. 349 If client ID or client secret is empty, tool force to run setup function. 350 If project didn't change and config hold user client ID/secret, tool 351 would skip client ID setup. 352 353 Args: 354 project_changed: Boolean, True for project changed. 355 356 Returns: 357 Boolean: True for run setup function. 358 """ 359 if project_changed: 360 logger.info("Your project changed. Start to run setup process.") 361 return True 362 if not self.client_id or not self.client_secret: 363 logger.info("Client ID or client secret is empty. Start to run setup process.") 364 return True 365 logger.info("Project was unchanged and client ID didn't need to changed.") 366 return False 367 368 def _SetupProject(self, gcloud_runner): 369 """Setup gcloud project information. 370 371 Setup project and zone. 372 Setup client ID and client secret. 373 Make sure billing account enabled in project. 374 375 Args: 376 gcloud_runner: A GcloudRunner class to run "gcloud" command. 377 """ 378 project_changed = False 379 if self._NeedProjectSetup(): 380 project_changed = self._UpdateProject(gcloud_runner) 381 if self._NeedClientIDSetup(project_changed): 382 self._SetupClientIDSecret() 383 self._CheckBillingEnable(gcloud_runner) 384 385 def _UpdateProject(self, gcloud_runner): 386 """Setup gcloud project name and zone name and check project changed. 387 388 Run "gcloud init" to handle gcloud project setup. 389 Then "gcloud list" to get user settings information include "project" & "zone". 390 Record project_changed for next setup steps. 391 392 Args: 393 gcloud_runner: A GcloudRunner class to run "gcloud" command. 394 395 Returns: 396 project_changed: True for project settings changed. 397 """ 398 project_changed = False 399 gcloud_runner.RunGcloud(["init"]) 400 gcp_config_list_out = gcloud_runner.RunGcloud(["config", "list"]) 401 for line in gcp_config_list_out.splitlines(): 402 project_match = _PROJECT_RE.match(line) 403 if project_match: 404 project = project_match.group("project") 405 project_changed = (self.project != project) 406 self.project = project 407 continue 408 zone_match = _ZONE_RE.match(line) 409 if zone_match: 410 self.zone = zone_match.group("zone") 411 continue 412 UpdateConfigFile(self.config_path, "project", self.project) 413 UpdateConfigFile(self.config_path, "zone", self.zone) 414 return project_changed 415 416 def _SetupClientIDSecret(self): 417 """Setup Client ID / Client Secret in config file. 418 419 User can use input new values for Client ID and Client Secret. 420 """ 421 print("Please generate a new client ID/secret by following the instructions here:") 422 print("https://support.google.com/cloud/answer/6158849?hl=en") 423 # TODO: Create markdown readme instructions since the link isn't too helpful. 424 self.client_id = None 425 self.client_secret = None 426 while _InputIsEmpty(self.client_id): 427 self.client_id = str(six.moves.input("Enter Client ID: ").strip()) 428 while _InputIsEmpty(self.client_secret): 429 self.client_secret = str(six.moves.input("Enter Client Secret: ").strip()) 430 UpdateConfigFile(self.config_path, "client_id", self.client_id) 431 UpdateConfigFile(self.config_path, "client_secret", self.client_secret) 432 433 def _CheckBillingEnable(self, gcloud_runner): 434 """Check billing enabled in gcp project. 435 436 The billing info get by gcloud alpha command. Here is one example: 437 $ gcloud alpha billing projects describe project_name 438 billingAccountName: billingAccounts/011BXX-A30XXX-9XXXX 439 billingEnabled: true 440 name: projects/project_name/billingInfo 441 projectId: project_name 442 443 Args: 444 gcloud_runner: A GcloudRunner class to run "gcloud" command. 445 446 Raises: 447 NoBillingError: gcp project doesn't enable billing account. 448 """ 449 billing_info = gcloud_runner.RunGcloud( 450 ["alpha", "billing", "projects", "describe", self.project]) 451 if _BILLING_ENABLE_MSG not in billing_info: 452 raise errors.NoBillingError( 453 "Please set billing account to project(%s) by following the " 454 "instructions here: " 455 "https://cloud.google.com/billing/docs/how-to/modify-project" 456 % self.project) 457 458 @staticmethod 459 def _EnableGcloudServices(gcloud_runner): 460 """Enable 3 Gcloud API services. 461 462 1. Android build service 463 2. Compute engine service 464 To avoid confuse user, we don't show messages for services processing 465 messages. e.g. "Waiting for async operation operations ...." 466 467 Args: 468 gcloud_runner: A GcloudRunner class to run "gcloud" command. 469 """ 470 google_apis = [ 471 GoogleAPIService(_ANDROID_BUILD_SERVICE, _ANDROID_BUILD_MSG), 472 GoogleAPIService(_COMPUTE_ENGINE_SERVICE, _COMPUTE_ENGINE_MSG, required=True) 473 ] 474 enabled_services = gcloud_runner.RunGcloud( 475 ["services", "list", "--enabled", "--format", "value(NAME)"], 476 stderr=subprocess.STDOUT).splitlines() 477 478 for service in google_apis: 479 if service.name not in enabled_services: 480 service.EnableService(gcloud_runner) 481