1#!/usr/bin/env python 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""Gcloud setup runner.""" 17 18from __future__ import print_function 19import logging 20import os 21import re 22import subprocess 23 24import six 25 26from acloud import errors 27from acloud.internal.lib import utils 28from acloud.public import config 29from acloud.setup import base_task_runner 30from acloud.setup import google_sdk 31 32 33logger = logging.getLogger(__name__) 34 35# APIs that need to be enabled for GCP project. 36_ANDROID_BUILD_SERVICE = "androidbuildinternal.googleapis.com" 37_ANDROID_BUILD_MSG = ( 38 "This service (%s) help to download images from Android Build. If it isn't " 39 "enabled, acloud only supports local images to create AVD." 40 % _ANDROID_BUILD_SERVICE) 41_COMPUTE_ENGINE_SERVICE = "compute.googleapis.com" 42_COMPUTE_ENGINE_MSG = ( 43 "This service (%s) help to create instance in google cloud platform. If it " 44 "isn't enabled, acloud can't work anymore." % _COMPUTE_ENGINE_SERVICE) 45_OPEN_SERVICE_FAILED_MSG = ( 46 "\n[Open Service Failed]\n" 47 "Service name: %(service_name)s\n" 48 "%(service_msg)s\n") 49 50_BUILD_SERVICE_ACCOUNT = "android-build-prod@system.gserviceaccount.com" 51_BILLING_ENABLE_MSG = "billingEnabled: true" 52_DEFAULT_SSH_FOLDER = os.path.expanduser("~/.ssh") 53_DEFAULT_SSH_KEY = "acloud_rsa" 54_DEFAULT_SSH_PRIVATE_KEY = os.path.join(_DEFAULT_SSH_FOLDER, 55 _DEFAULT_SSH_KEY) 56_DEFAULT_SSH_PUBLIC_KEY = os.path.join(_DEFAULT_SSH_FOLDER, 57 _DEFAULT_SSH_KEY + ".pub") 58_ENV_CLOUDSDK_PYTHON = "CLOUDSDK_PYTHON" 59_GCLOUD_COMPONENT_ALPHA = "alpha" 60# Regular expression to get project/zone information. 61_PROJECT_RE = re.compile(r"^project = (?P<project>.+)") 62_ZONE_RE = re.compile(r"^zone = (?P<zone>.+)") 63 64 65def UpdateConfigFile(config_path, item, value): 66 """Update config data. 67 68 Case A: config file contain this item. 69 In config, "project = A_project". New value is B_project 70 Set config "project = B_project". 71 Case B: config file didn't contain this item. 72 New value is B_project. 73 Setup config as "project = B_project". 74 75 Args: 76 config_path: String, acloud config path. 77 item: String, item name in config file. EX: project, zone 78 value: String, value of item in config file. 79 80 TODO(111574698): Refactor this to minimize writes to the config file. 81 TODO(111574698): Use proto method to update config. 82 """ 83 write_lines = [] 84 find_item = False 85 write_line = item + ": \"" + value + "\"\n" 86 if os.path.isfile(config_path): 87 with open(config_path, "r") as cfg_file: 88 for read_line in cfg_file.readlines(): 89 if read_line.startswith(item + ":"): 90 find_item = True 91 write_lines.append(write_line) 92 else: 93 write_lines.append(read_line) 94 if not find_item: 95 write_lines.append(write_line) 96 with open(config_path, "w") as cfg_file: 97 cfg_file.writelines(write_lines) 98 99 100def SetupSSHKeys(config_path, private_key_path, public_key_path): 101 """Setup the pair of the ssh key for acloud.config. 102 103 User can use the default path: "~/.ssh/acloud_rsa". 104 105 Args: 106 config_path: String, acloud config path. 107 private_key_path: Path to the private key file. 108 e.g. ~/.ssh/acloud_rsa 109 public_key_path: Path to the public key file. 110 e.g. ~/.ssh/acloud_rsa.pub 111 """ 112 private_key_path = os.path.expanduser(private_key_path) 113 if (private_key_path == "" or public_key_path == "" 114 or private_key_path == _DEFAULT_SSH_PRIVATE_KEY): 115 utils.CreateSshKeyPairIfNotExist(_DEFAULT_SSH_PRIVATE_KEY, 116 _DEFAULT_SSH_PUBLIC_KEY) 117 UpdateConfigFile(config_path, "ssh_private_key_path", 118 _DEFAULT_SSH_PRIVATE_KEY) 119 UpdateConfigFile(config_path, "ssh_public_key_path", 120 _DEFAULT_SSH_PUBLIC_KEY) 121 122 123def _InputIsEmpty(input_string): 124 """Check input string is empty. 125 126 Tool requests user to input client ID & client secret. 127 This basic check can detect user input is empty. 128 129 Args: 130 input_string: String, user input string. 131 132 Returns: 133 Boolean: True if input is empty, False otherwise. 134 """ 135 if input_string is None: 136 return True 137 if input_string == "": 138 print("Please enter a non-empty value.") 139 return True 140 return False 141 142 143class GoogleSDKBins(object): 144 """Class to run tools in the Google SDK.""" 145 146 def __init__(self, google_sdk_folder): 147 """GoogleSDKBins initialize. 148 149 Args: 150 google_sdk_folder: String, google sdk path. 151 """ 152 self.gcloud_command_path = os.path.join(google_sdk_folder, "gcloud") 153 self.gsutil_command_path = os.path.join(google_sdk_folder, "gsutil") 154 # TODO(137195528): Remove python2 environment after acloud support python3. 155 self._env = os.environ.copy() 156 self._env[_ENV_CLOUDSDK_PYTHON] = "python2" 157 158 def RunGcloud(self, cmd, **kwargs): 159 """Run gcloud command. 160 161 Args: 162 cmd: String list, command strings. 163 Ex: [config], then this function call "gcloud config". 164 **kwargs: dictionary of keyword based args to pass to func. 165 166 Returns: 167 String, return message after execute gcloud command. 168 """ 169 return utils.CheckOutput([self.gcloud_command_path] + cmd, 170 env=self._env, **kwargs) 171 172 def RunGsutil(self, cmd, **kwargs): 173 """Run gsutil command. 174 175 Args: 176 cmd : String list, command strings. 177 Ex: [list], then this function call "gsutil list". 178 **kwargs: dictionary of keyword based args to pass to func. 179 180 Returns: 181 String, return message after execute gsutil command. 182 """ 183 return utils.CheckOutput([self.gsutil_command_path] + cmd, 184 env=self._env, **kwargs) 185 186 187class GoogleAPIService(object): 188 """Class to enable api service in the gcp project.""" 189 190 def __init__(self, service_name, error_msg, required=False): 191 """GoogleAPIService initialize. 192 193 Args: 194 service_name: String, name of api service. 195 error_msg: String, show messages if api service enable failed. 196 required: Boolean, True for service must be enabled for acloud. 197 """ 198 self._name = service_name 199 self._error_msg = error_msg 200 self._required = required 201 202 def EnableService(self, gcloud_runner): 203 """Enable api service. 204 205 Args: 206 gcloud_runner: A GcloudRunner class to run "gcloud" command. 207 """ 208 try: 209 gcloud_runner.RunGcloud(["services", "enable", self._name], 210 stderr=subprocess.STDOUT) 211 except subprocess.CalledProcessError as error: 212 self.ShowFailMessages(error.output) 213 214 def ShowFailMessages(self, error): 215 """Show fail messages. 216 217 Show the fail messages to hint users the impact if the api service 218 isn't enabled. 219 220 Args: 221 error: String of error message when opening api service failed. 222 """ 223 msg_color = (utils.TextColors.FAIL if self._required else 224 utils.TextColors.WARNING) 225 utils.PrintColorString( 226 error + _OPEN_SERVICE_FAILED_MSG % { 227 "service_name": self._name, 228 "service_msg": self._error_msg} 229 , msg_color) 230 231 @property 232 def name(self): 233 """Return name.""" 234 return self._name 235 236 237class GcpTaskRunner(base_task_runner.BaseTaskRunner): 238 """Runner to setup google cloud user information.""" 239 240 WELCOME_MESSAGE_TITLE = "Setup google cloud user information" 241 WELCOME_MESSAGE = ( 242 "This step will walk you through gcloud SDK installation." 243 "Then configure gcloud user information." 244 "Finally enable some gcloud API services.") 245 246 def __init__(self, config_path): 247 """Initialize parameters. 248 249 Load config file to get current values. 250 251 Args: 252 config_path: String, acloud config path. 253 """ 254 # pylint: disable=invalid-name 255 config_mgr = config.AcloudConfigManager(config_path) 256 cfg = config_mgr.Load() 257 self.config_path = config_mgr.user_config_path 258 self.project = cfg.project 259 self.zone = cfg.zone 260 self.ssh_private_key_path = cfg.ssh_private_key_path 261 self.ssh_public_key_path = cfg.ssh_public_key_path 262 self.stable_host_image_name = cfg.stable_host_image_name 263 self.client_id = cfg.client_id 264 self.client_secret = cfg.client_secret 265 self.service_account_name = cfg.service_account_name 266 self.service_account_private_key_path = cfg.service_account_private_key_path 267 self.service_account_json_private_key_path = cfg.service_account_json_private_key_path 268 269 def ShouldRun(self): 270 """Check if we actually need to run GCP setup. 271 272 We'll only do the gcp setup if certain fields in the cfg are empty. 273 274 Returns: 275 True if reqired config fields are empty, False otherwise. 276 """ 277 # We need to ensure the config has the proper auth-related fields set, 278 # so config requires just 1 of the following: 279 # 1. client id/secret 280 # 2. service account name/private key path 281 # 3. service account json private key path 282 if ((not self.client_id or not self.client_secret) 283 and (not self.service_account_name or not self.service_account_private_key_path) 284 and not self.service_account_json_private_key_path): 285 return True 286 287 # If a project isn't set, then we need to run setup. 288 return not self.project 289 290 def _Run(self): 291 """Run GCP setup task.""" 292 self._SetupGcloudInfo() 293 SetupSSHKeys(self.config_path, self.ssh_private_key_path, 294 self.ssh_public_key_path) 295 296 def _SetupGcloudInfo(self): 297 """Setup Gcloud user information. 298 1. Setup Gcloud SDK tools. 299 2. Setup Gcloud project. 300 a. Setup Gcloud project and zone. 301 b. Setup Client ID and Client secret. 302 c. Setup Google Cloud Storage bucket. 303 3. Enable Gcloud API services. 304 """ 305 google_sdk_init = google_sdk.GoogleSDK() 306 try: 307 google_sdk_runner = GoogleSDKBins(google_sdk_init.GetSDKBinPath()) 308 google_sdk_init.InstallGcloudComponent(google_sdk_runner, 309 _GCLOUD_COMPONENT_ALPHA) 310 self._SetupProject(google_sdk_runner) 311 self._EnableGcloudServices(google_sdk_runner) 312 self._CreateStableHostImage() 313 finally: 314 google_sdk_init.CleanUp() 315 316 def _CreateStableHostImage(self): 317 """Create the stable host image.""" 318 # Write default stable_host_image_name with unused value. 319 # TODO(113091773): An additional step to create the host image. 320 if not self.stable_host_image_name: 321 UpdateConfigFile(self.config_path, "stable_host_image_name", "") 322 323 324 def _NeedProjectSetup(self): 325 """Confirm project setup should run or not. 326 327 If the project settings (project name and zone) are blank (either one), 328 we'll run the project setup flow. If they are set, we'll check with 329 the user if they want to update them. 330 331 Returns: 332 Boolean: True if we need to setup the project, False otherwise. 333 """ 334 user_question = ( 335 "Your default Project/Zone settings are:\n" 336 "project:[%s]\n" 337 "zone:[%s]\n" 338 "Would you like to update them?[y/N]: \n") % (self.project, self.zone) 339 340 if not self.project or not self.zone: 341 logger.info("Project or zone is empty. Start to run setup process.") 342 return True 343 return utils.GetUserAnswerYes(user_question) 344 345 def _NeedClientIDSetup(self, project_changed): 346 """Confirm client setup should run or not. 347 348 If project changed, client ID must also have to change. 349 So tool will force to run setup function. 350 If client ID or client secret is empty, tool force to run setup function. 351 If project didn't change and config hold user client ID/secret, tool 352 would skip client ID setup. 353 354 Args: 355 project_changed: Boolean, True for project changed. 356 357 Returns: 358 Boolean: True for run setup function. 359 """ 360 if project_changed: 361 logger.info("Your project changed. Start to run setup process.") 362 return True 363 elif not self.client_id or not self.client_secret: 364 logger.info("Client ID or client secret is empty. Start to run setup process.") 365 return True 366 logger.info("Project was unchanged and client ID didn't need to changed.") 367 return False 368 369 def _SetupProject(self, gcloud_runner): 370 """Setup gcloud project information. 371 372 Setup project and zone. 373 Setup client ID and client secret. 374 Make sure billing account enabled in project. 375 376 Args: 377 gcloud_runner: A GcloudRunner class to run "gcloud" command. 378 """ 379 project_changed = False 380 if self._NeedProjectSetup(): 381 project_changed = self._UpdateProject(gcloud_runner) 382 if self._NeedClientIDSetup(project_changed): 383 self._SetupClientIDSecret() 384 self._CheckBillingEnable(gcloud_runner) 385 386 def _UpdateProject(self, gcloud_runner): 387 """Setup gcloud project name and zone name and check project changed. 388 389 Run "gcloud init" to handle gcloud project setup. 390 Then "gcloud list" to get user settings information include "project" & "zone". 391 Record project_changed for next setup steps. 392 393 Args: 394 gcloud_runner: A GcloudRunner class to run "gcloud" command. 395 396 Returns: 397 project_changed: True for project settings changed. 398 """ 399 project_changed = False 400 gcloud_runner.RunGcloud(["init"]) 401 gcp_config_list_out = gcloud_runner.RunGcloud(["config", "list"]) 402 for line in gcp_config_list_out.splitlines(): 403 project_match = _PROJECT_RE.match(line) 404 if project_match: 405 project = project_match.group("project") 406 project_changed = (self.project != project) 407 self.project = project 408 continue 409 zone_match = _ZONE_RE.match(line) 410 if zone_match: 411 self.zone = zone_match.group("zone") 412 continue 413 UpdateConfigFile(self.config_path, "project", self.project) 414 UpdateConfigFile(self.config_path, "zone", self.zone) 415 return project_changed 416 417 def _SetupClientIDSecret(self): 418 """Setup Client ID / Client Secret in config file. 419 420 User can use input new values for Client ID and Client Secret. 421 """ 422 print("Please generate a new client ID/secret by following the instructions here:") 423 print("https://support.google.com/cloud/answer/6158849?hl=en") 424 # TODO: Create markdown readme instructions since the link isn't too helpful. 425 self.client_id = None 426 self.client_secret = None 427 while _InputIsEmpty(self.client_id): 428 self.client_id = str(six.moves.input("Enter Client ID: ").strip()) 429 while _InputIsEmpty(self.client_secret): 430 self.client_secret = str(six.moves.input("Enter Client Secret: ").strip()) 431 UpdateConfigFile(self.config_path, "client_id", self.client_id) 432 UpdateConfigFile(self.config_path, "client_secret", self.client_secret) 433 434 def _CheckBillingEnable(self, gcloud_runner): 435 """Check billing enabled in gcp project. 436 437 The billing info get by gcloud alpha command. Here is one example: 438 $ gcloud alpha billing projects describe project_name 439 billingAccountName: billingAccounts/011BXX-A30XXX-9XXXX 440 billingEnabled: true 441 name: projects/project_name/billingInfo 442 projectId: project_name 443 444 Args: 445 gcloud_runner: A GcloudRunner class to run "gcloud" command. 446 447 Raises: 448 NoBillingError: gcp project doesn't enable billing account. 449 """ 450 billing_info = gcloud_runner.RunGcloud( 451 ["alpha", "billing", "projects", "describe", self.project]) 452 if _BILLING_ENABLE_MSG not in billing_info: 453 raise errors.NoBillingError( 454 "Please set billing account to project(%s) by following the " 455 "instructions here: " 456 "https://cloud.google.com/billing/docs/how-to/modify-project" 457 % self.project) 458 459 @staticmethod 460 def _EnableGcloudServices(gcloud_runner): 461 """Enable 3 Gcloud API services. 462 463 1. Android build service 464 2. Compute engine service 465 To avoid confuse user, we don't show messages for services processing 466 messages. e.g. "Waiting for async operation operations ...." 467 468 Args: 469 gcloud_runner: A GcloudRunner class to run "gcloud" command. 470 """ 471 google_apis = [ 472 GoogleAPIService(_ANDROID_BUILD_SERVICE, _ANDROID_BUILD_MSG), 473 GoogleAPIService(_COMPUTE_ENGINE_SERVICE, _COMPUTE_ENGINE_MSG, required=True) 474 ] 475 enabled_services = gcloud_runner.RunGcloud( 476 ["services", "list", "--enabled", "--format", "value(NAME)"], 477 stderr=subprocess.STDOUT).splitlines() 478 479 for service in google_apis: 480 if service.name not in enabled_services: 481 service.EnableService(gcloud_runner) 482