#!/usr/bin/env python # # Copyright 2018 - The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Gcloud setup runner.""" from __future__ import print_function import logging import os import re import subprocess from acloud import errors from acloud.internal.lib import utils from acloud.public import config from acloud.setup import base_task_runner from acloud.setup import google_sdk # APIs that need to be enabled for GCP project. _ANDROID_BUILD_SERVICE = "androidbuildinternal.googleapis.com" _COMPUTE_ENGINE_SERVICE = "compute.googleapis.com" _GOOGLE_CLOUD_STORAGE_SERVICE = "storage-component.googleapis.com" _GOOGLE_APIS = [ _GOOGLE_CLOUD_STORAGE_SERVICE, _ANDROID_BUILD_SERVICE, _COMPUTE_ENGINE_SERVICE ] _BUILD_SERVICE_ACCOUNT = "android-build-prod@system.gserviceaccount.com" _DEFAULT_SSH_FOLDER = os.path.expanduser("~/.ssh") _DEFAULT_SSH_KEY = "acloud_rsa" _DEFAULT_SSH_PRIVATE_KEY = os.path.join(_DEFAULT_SSH_FOLDER, _DEFAULT_SSH_KEY) _DEFAULT_SSH_PUBLIC_KEY = os.path.join(_DEFAULT_SSH_FOLDER, _DEFAULT_SSH_KEY + ".pub") # Bucket naming parameters _BUCKET_HEADER = "gs://" _BUCKET_LENGTH_LIMIT = 63 _DEFAULT_BUCKET_HEADER = "acloud" _DEFAULT_BUCKET_REGION = "US" _INVALID_BUCKET_NAME_END_CHARS = "_-" _PROJECT_SEPARATOR = ":" # Regular expression to get project/zone/bucket information. _BUCKET_RE = re.compile(r"^gs://(?P.+)/") _BUCKET_REGION_RE = re.compile(r"^Location constraint:(?P.+)") _PROJECT_RE = re.compile(r"^project = (?P.+)") _ZONE_RE = re.compile(r"^zone = (?P.+)") logger = logging.getLogger(__name__) def UpdateConfigFile(config_path, item, value): """Update config data. Case A: config file contain this item. In config, "project = A_project". New value is B_project Set config "project = B_project". Case B: config file didn't contain this item. New value is B_project. Setup config as "project = B_project". Args: config_path: String, acloud config path. item: String, item name in config file. EX: project, zone value: String, value of item in config file. TODO(111574698): Refactor this to minimize writes to the config file. TODO(111574698): Use proto method to update config. """ write_lines = [] find_item = False write_line = item + ": \"" + value + "\"\n" if os.path.isfile(config_path): with open(config_path, "r") as cfg_file: for read_line in cfg_file.readlines(): if read_line.startswith(item + ":"): find_item = True write_lines.append(write_line) else: write_lines.append(read_line) if not find_item: write_lines.append(write_line) with open(config_path, "w") as cfg_file: cfg_file.writelines(write_lines) def SetupSSHKeys(config_path, private_key_path, public_key_path): """Setup the pair of the ssh key for acloud.config. User can use the default path: "~/.ssh/acloud_rsa". Args: config_path: String, acloud config path. private_key_path: Path to the private key file. e.g. ~/.ssh/acloud_rsa public_key_path: Path to the public key file. e.g. ~/.ssh/acloud_rsa.pub """ private_key_path = os.path.expanduser(private_key_path) if (private_key_path == "" or public_key_path == "" or private_key_path == _DEFAULT_SSH_PRIVATE_KEY): utils.CreateSshKeyPairIfNotExist(_DEFAULT_SSH_PRIVATE_KEY, _DEFAULT_SSH_PUBLIC_KEY) UpdateConfigFile(config_path, "ssh_private_key_path", _DEFAULT_SSH_PRIVATE_KEY) UpdateConfigFile(config_path, "ssh_public_key_path", _DEFAULT_SSH_PUBLIC_KEY) def _InputIsEmpty(input_string): """Check input string is empty. Tool requests user to input client ID & client secret. This basic check can detect user input is empty. Args: input_string: String, user input string. Returns: Boolean: True if input is empty, False otherwise. """ if input_string is None: return True if input_string == "": print("Please enter a non-empty value.") return True return False class GoogleSDKBins(object): """Class to run tools in the Google SDK.""" def __init__(self, google_sdk_folder): """GoogleSDKBins initialize. Args: google_sdk_folder: String, google sdk path. """ self.gcloud_command_path = os.path.join(google_sdk_folder, "gcloud") self.gsutil_command_path = os.path.join(google_sdk_folder, "gsutil") def RunGcloud(self, cmd, **kwargs): """Run gcloud command. Args: cmd: String list, command strings. Ex: [config], then this function call "gcloud config". **kwargs: dictionary of keyword based args to pass to func. Returns: String, return message after execute gcloud command. """ return subprocess.check_output([self.gcloud_command_path] + cmd, **kwargs) def RunGsutil(self, cmd, **kwargs): """Run gsutil command. Args: cmd : String list, command strings. Ex: [list], then this function call "gsutil list". **kwargs: dictionary of keyword based args to pass to func. Returns: String, return message after execute gsutil command. """ return subprocess.check_output([self.gsutil_command_path] + cmd, **kwargs) class GcpTaskRunner(base_task_runner.BaseTaskRunner): """Runner to setup google cloud user information.""" WELCOME_MESSAGE_TITLE = "Setup google cloud user information" WELCOME_MESSAGE = ( "This step will walk you through gcloud SDK installation." "Then configure gcloud user information." "Finally enable some gcloud API services.") def __init__(self, config_path): """Initialize parameters. Load config file to get current values. Args: config_path: String, acloud config path. """ # pylint: disable=invalid-name config_mgr = config.AcloudConfigManager(config_path) cfg = config_mgr.Load() self.config_path = config_mgr.user_config_path self.project = cfg.project self.zone = cfg.zone self.storage_bucket_name = cfg.storage_bucket_name self.ssh_private_key_path = cfg.ssh_private_key_path self.ssh_public_key_path = cfg.ssh_public_key_path self.stable_host_image_name = cfg.stable_host_image_name self.client_id = cfg.client_id self.client_secret = cfg.client_secret self.service_account_name = cfg.service_account_name self.service_account_private_key_path = cfg.service_account_private_key_path self.service_account_json_private_key_path = cfg.service_account_json_private_key_path def ShouldRun(self): """Check if we actually need to run GCP setup. We'll only do the gcp setup if certain fields in the cfg are empty. Returns: True if reqired config fields are empty, False otherwise. """ # We need to ensure the config has the proper auth-related fields set, # so config requires just 1 of the following: # 1. client id/secret # 2. service account name/private key path # 3. service account json private key path if ((not self.client_id or not self.client_secret) and (not self.service_account_name or not self.service_account_private_key_path) and not self.service_account_json_private_key_path): return True # If a project isn't set, then we need to run setup. return not self.project def _Run(self): """Run GCP setup task.""" self._SetupGcloudInfo() SetupSSHKeys(self.config_path, self.ssh_private_key_path, self.ssh_public_key_path) def _SetupGcloudInfo(self): """Setup Gcloud user information. 1. Setup Gcloud SDK tools. 2. Setup Gcloud project. a. Setup Gcloud project and zone. b. Setup Client ID and Client secret. c. Setup Google Cloud Storage bucket. 3. Enable Gcloud API services. """ google_sdk_init = google_sdk.GoogleSDK() try: google_sdk_runner = GoogleSDKBins(google_sdk_init.GetSDKBinPath()) self._SetupProject(google_sdk_runner) self._EnableGcloudServices(google_sdk_runner) self._CreateStableHostImage() finally: google_sdk_init.CleanUp() def _CreateStableHostImage(self): """Create the stable host image.""" # Write default stable_host_image_name with dummy value. # TODO(113091773): An additional step to create the host image. if not self.stable_host_image_name: UpdateConfigFile(self.config_path, "stable_host_image_name", "") def _NeedProjectSetup(self): """Confirm project setup should run or not. If the project settings (project name and zone) are blank (either one), we'll run the project setup flow. If they are set, we'll check with the user if they want to update them. Returns: Boolean: True if we need to setup the project, False otherwise. """ user_question = ( "Your default Project/Zone settings are:\n" "project:[%s]\n" "zone:[%s]\n" "Would you like to update them?[y/N]: \n") % (self.project, self.zone) if not self.project or not self.zone: logger.info("Project or zone is empty. Start to run setup process.") return True return utils.GetUserAnswerYes(user_question) def _NeedClientIDSetup(self, project_changed): """Confirm client setup should run or not. If project changed, client ID must also have to change. So tool will force to run setup function. If client ID or client secret is empty, tool force to run setup function. If project didn't change and config hold user client ID/secret, tool would skip client ID setup. Args: project_changed: Boolean, True for project changed. Returns: Boolean: True for run setup function. """ if project_changed: logger.info("Your project changed. Start to run setup process.") return True elif not self.client_id or not self.client_secret: logger.info("Client ID or client secret is empty. Start to run setup process.") return True logger.info("Project was unchanged and client ID didn't need to changed.") return False def _SetupProject(self, gcloud_runner): """Setup gcloud project information. Setup project and zone. Setup client ID and client secret. Setup Google Cloud Storage bucket. Args: gcloud_runner: A GcloudRunner class to run "gcloud" command. """ project_changed = False if self._NeedProjectSetup(): project_changed = self._UpdateProject(gcloud_runner) if self._NeedClientIDSetup(project_changed): self._SetupClientIDSecret() self._SetupStorageBucket(gcloud_runner) def _UpdateProject(self, gcloud_runner): """Setup gcloud project name and zone name and check project changed. Run "gcloud init" to handle gcloud project setup. Then "gcloud list" to get user settings information include "project" & "zone". Record project_changed for next setup steps. Args: gcloud_runner: A GcloudRunner class to run "gcloud" command. Returns: project_changed: True for project settings changed. """ project_changed = False gcloud_runner.RunGcloud(["init"]) gcp_config_list_out = gcloud_runner.RunGcloud(["config", "list"]) for line in gcp_config_list_out.splitlines(): project_match = _PROJECT_RE.match(line) if project_match: project = project_match.group("project") project_changed = (self.project != project) self.project = project continue zone_match = _ZONE_RE.match(line) if zone_match: self.zone = zone_match.group("zone") continue UpdateConfigFile(self.config_path, "project", self.project) UpdateConfigFile(self.config_path, "zone", self.zone) return project_changed def _SetupClientIDSecret(self): """Setup Client ID / Client Secret in config file. User can use input new values for Client ID and Client Secret. """ print("Please generate a new client ID/secret by following the instructions here:") print("https://support.google.com/cloud/answer/6158849?hl=en") # TODO: Create markdown readme instructions since the link isn't too helpful. self.client_id = None self.client_secret = None while _InputIsEmpty(self.client_id): self.client_id = str(raw_input("Enter Client ID: ").strip()) while _InputIsEmpty(self.client_secret): self.client_secret = str(raw_input("Enter Client Secret: ").strip()) UpdateConfigFile(self.config_path, "client_id", self.client_id) UpdateConfigFile(self.config_path, "client_secret", self.client_secret) def _SetupStorageBucket(self, gcloud_runner): """Setup storage_bucket_name in config file. We handle the following cases: 1. Bucket set in the config && bucket is valid. - Configure the bucket. 2. Bucket set in the config && bucket is invalid. - Create a default acloud bucket and configure it 3. Bucket is not set in the config. - Create a default acloud bucket and configure it. Args: gcloud_runner: A GcloudRunner class to run "gsutil" command. """ if (not self.storage_bucket_name or not self._BucketIsValid(self.storage_bucket_name, gcloud_runner)): self.storage_bucket_name = self._CreateDefaultBucket(gcloud_runner) self._ConfigureBucket(gcloud_runner) UpdateConfigFile(self.config_path, "storage_bucket_name", self.storage_bucket_name) logger.info("Storage bucket name set to [%s]", self.storage_bucket_name) def _ConfigureBucket(self, gcloud_runner): """Setup write access right for Android Build service account. To avoid confuse user, we don't show messages for processing messages. e.g. "No changes to gs://acloud-bucket/" Args: gcloud_runner: A GcloudRunner class to run "gsutil" command. """ gcloud_runner.RunGsutil([ "acl", "ch", "-u", "%s:W" % (_BUILD_SERVICE_ACCOUNT), "%s" % (_BUCKET_HEADER + self.storage_bucket_name) ], stderr=subprocess.STDOUT) def _BucketIsValid(self, bucket_name, gcloud_runner): """Check bucket is valid or not. If bucket exists and region is in default region, then this bucket is valid. Args: bucket_name: String, name of storage bucket. gcloud_runner: A GcloudRunner class to run "gsutil" command. Returns: Boolean: True if bucket is valid, otherwise False. """ return (self._BucketExists(bucket_name, gcloud_runner) and self._BucketInDefaultRegion(bucket_name, gcloud_runner)) def _CreateDefaultBucket(self, gcloud_runner): """Setup bucket to default bucket name. Default bucket name is "acloud-{project}". If default bucket exist and its region is not "US", then default bucket name is changed as "acloud-{project}-us" If default bucket didn't exist, tool will create it. Args: gcloud_runner: A GcloudRunner class to run "gsutil" command. Returns: String: string of bucket name. """ bucket_name = self._GenerateBucketName(self.project) if (self._BucketExists(bucket_name, gcloud_runner) and not self._BucketInDefaultRegion(bucket_name, gcloud_runner)): bucket_name += ("-" + _DEFAULT_BUCKET_REGION.lower()) if not self._BucketExists(bucket_name, gcloud_runner): self._CreateBucket(bucket_name, gcloud_runner) return bucket_name @staticmethod def _GenerateBucketName(project_name): """Generate GCS bucket name that meets the naming guidelines. Naming guidelines: https://cloud.google.com/storage/docs/naming 1. Filter out organization name. 2. Filter out illegal characters. 3. Length limit. 4. Name must end with a number or letter. Args: project_name: String, name of project. Returns: String: GCS bucket name compliant with naming guidelines. """ # Sanitize the project name by filtering out the org name (e.g. # AOSP:fake_project -> fake_project) if _PROJECT_SEPARATOR in project_name: _, project_name = project_name.split(_PROJECT_SEPARATOR) bucket_name = "%s-%s" % (_DEFAULT_BUCKET_HEADER, project_name) # Rule 1: A bucket name can contain lowercase alphanumeric characters, # hyphens, and underscores. bucket_name = re.sub("[^a-zA-Z_/-]+", "", bucket_name).lower() # Rule 2: Bucket names must limit to 63 characters. if len(bucket_name) > _BUCKET_LENGTH_LIMIT: bucket_name = bucket_name[:_BUCKET_LENGTH_LIMIT] # Rule 3: Bucket names must end with a letter, strip out any ending # "-" or "_" at the end of the name. bucket_name = bucket_name.rstrip(_INVALID_BUCKET_NAME_END_CHARS) return bucket_name @staticmethod def _BucketExists(bucket_name, gcloud_runner): """Confirm bucket exist in project or not. Args: bucket_name: String, name of storage bucket. gcloud_runner: A GcloudRunner class to run "gsutil" command. Returns: Boolean: True for bucket exist in project. """ output = gcloud_runner.RunGsutil(["list"]) for output_line in output.splitlines(): match = _BUCKET_RE.match(output_line) if match.group("bucket") == bucket_name: return True return False @staticmethod def _BucketInDefaultRegion(bucket_name, gcloud_runner): """Confirm bucket region settings is "US" or not. Args: bucket_name: String, name of storage bucket. gcloud_runner: A GcloudRunner class to run "gsutil" command. Returns: Boolean: True for bucket region is in default region. Raises: errors.SetupError: For parsing bucket region information error. """ output = gcloud_runner.RunGsutil( ["ls", "-L", "-b", "%s" % (_BUCKET_HEADER + bucket_name)]) for region_line in output.splitlines(): region_match = _BUCKET_REGION_RE.match(region_line.strip()) if region_match: region = region_match.group("region").strip() logger.info("Bucket[%s] is in %s (checking for %s)", bucket_name, region, _DEFAULT_BUCKET_REGION) if region == _DEFAULT_BUCKET_REGION: return True return False raise errors.ParseBucketRegionError("Could not determine bucket region.") @staticmethod def _CreateBucket(bucket_name, gcloud_runner): """Create new storage bucket in project. Args: bucket_name: String, name of storage bucket. gcloud_runner: A GcloudRunner class to run "gsutil" command. """ gcloud_runner.RunGsutil(["mb", "%s" % (_BUCKET_HEADER + bucket_name)]) logger.info("Create bucket [%s].", bucket_name) @staticmethod def _EnableGcloudServices(gcloud_runner): """Enable 3 Gcloud API services. 1. Android build service 2. Compute engine service 3. Google cloud storage service To avoid confuse user, we don't show messages for services processing messages. e.g. "Waiting for async operation operations ...." Args: gcloud_runner: A GcloudRunner class to run "gcloud" command. """ for service in _GOOGLE_APIS: gcloud_runner.RunGcloud(["services", "enable", service], stderr=subprocess.STDOUT)