#!/usr/bin/env python # # Copyright 2018 - The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Common operations between managing GCE and Cuttlefish devices. This module provides the common operations between managing GCE (device_driver) and Cuttlefish (create_cuttlefish_action) devices. Should not be called directly. """ from __future__ import print_function import getpass import logging import os import subprocess from acloud import errors from acloud.public import avd from acloud.public import report from acloud.internal import constants from acloud.internal.lib import utils logger = logging.getLogger(__name__) def CreateSshKeyPairIfNecessary(cfg): """Create ssh key pair if necessary. Args: cfg: An Acloudconfig instance. Raises: error.DriverError: If it falls into an unexpected condition. """ if not cfg.ssh_public_key_path: logger.warning( "ssh_public_key_path is not specified in acloud config. " "Project-wide public key will " "be used when creating AVD instances. " "Please ensure you have the correct private half of " "a project-wide public key if you want to ssh into the " "instances after creation.") elif cfg.ssh_public_key_path and not cfg.ssh_private_key_path: logger.warning( "Only ssh_public_key_path is specified in acloud config, " "but ssh_private_key_path is missing. " "Please ensure you have the correct private half " "if you want to ssh into the instances after creation.") elif cfg.ssh_public_key_path and cfg.ssh_private_key_path: utils.CreateSshKeyPairIfNotExist(cfg.ssh_private_key_path, cfg.ssh_public_key_path) else: # Should never reach here. raise errors.DriverError( "Unexpected error in CreateSshKeyPairIfNecessary") class DevicePool(object): """A class that manages a pool of virtual devices. Attributes: devices: A list of devices in the pool. """ def __init__(self, device_factory, devices=None): """Constructs a new DevicePool. Args: device_factory: A device factory capable of producing a goldfish or cuttlefish device. The device factory must expose an attribute with the credentials that can be used to retrieve information from the constructed device. devices: List of devices managed by this pool. """ self._devices = devices or [] self._device_factory = device_factory self._compute_client = device_factory.GetComputeClient() def _CollectAdbLogcats(self, output_dir): """Collect Adb logcats. Args: output_dir: String, the output file directory to store adb logcats. Returns: The file information dictionary with file path and file name. """ file_dict = {} for device in self._devices: if not device.adb_port: # If device adb tunnel is not established, do not do adb logcat continue file_name = "%s_adb_logcat.log" % device.instance_name full_file_path = os.path.join(output_dir, file_name) logger.info("Get adb %s:%s logcat for instance %s", constants.LOCALHOST, device.adb_port, device.instance_name) try: subprocess.check_call( ["adb -s %s:%s logcat -b all -d > %s" % ( constants.LOCALHOST, device.adb_port, full_file_path)], shell=True) file_dict[full_file_path] = file_name except subprocess.CalledProcessError: logging.error("Failed to get adb logcat for %s for instance %s", device.serial_number, device.instance_name) return file_dict def CreateDevices(self, num): """Creates |num| devices for given build_target and build_id. Args: num: Number of devices to create. """ # Create host instances for cuttlefish/goldfish device. # Currently one instance supports only 1 device. for _ in range(num): instance = self._device_factory.CreateInstance() ip = self._compute_client.GetInstanceIP(instance) self.devices.append( avd.AndroidVirtualDevice(ip=ip, instance_name=instance)) @utils.TimeExecute(function_description="Waiting for AVD(s) to boot up", result_evaluator=utils.BootEvaluator) def WaitForBoot(self): """Waits for all devices to boot up. Returns: A dictionary that contains all the failures. The key is the name of the instance that fails to boot, and the value is an errors.DeviceBootError object. """ failures = {} for device in self._devices: try: self._compute_client.WaitForBoot(device.instance_name) except errors.DeviceBootError as e: failures[device.instance_name] = e return failures def PullLogs(self, source_files, output_dir, user=None, ssh_rsa_path=None): """Tar logs from GCE instance into output_dir. Args: source_files: List of file names to be pulled. output_dir: String. The output file dirtory user: String, the ssh username to access GCE ssh_rsa_path: String, the ssh rsa key path to access GCE Returns: The file dictionary with file_path and file_name """ file_dict = {} for device in self._devices: if isinstance(source_files, basestring): source_files = [source_files] for source_file in source_files: file_name = "%s_%s" % (device.instance_name, os.path.basename(source_file)) dst_file = os.path.join(output_dir, file_name) logger.info("Pull %s for instance %s with user %s to %s", source_file, device.instance_name, user, dst_file) try: utils.ScpPullFile(source_file, dst_file, device.ip, user_name=user, rsa_key_file=ssh_rsa_path) file_dict[dst_file] = file_name except errors.DeviceConnectionError as e: logger.warning("Failed to pull %s from instance %s: %s", source_file, device.instance_name, e) return file_dict def CollectSerialPortLogs(self, output_file, port=constants.DEFAULT_SERIAL_PORT): """Tar the instance serial logs into specified output_file. Args: output_file: String, the output tar file path port: The serial port number to be collected """ # For emulator, the serial log is the virtual host serial log. # For GCE AVD device, the serial log is the AVD device serial log. with utils.TempDir() as tempdir: src_dict = {} for device in self._devices: logger.info("Store instance %s serial port %s output to %s", device.instance_name, port, output_file) serial_log = self._compute_client.GetSerialPortOutput( instance=device.instance_name, port=port) file_name = "%s_serial_%s.log" % (device.instance_name, port) file_path = os.path.join(tempdir, file_name) src_dict[file_path] = file_name with open(file_path, "w") as f: f.write(serial_log.encode("utf-8")) utils.MakeTarFile(src_dict, output_file) def CollectLogcats(self, output_file, ssh_user, ssh_rsa_path): """Tar the instances' logcat and other logs into specified output_file. Args: output_file: String, the output tar file path ssh_user: The ssh user name ssh_rsa_path: The ssh rsa key path """ with utils.TempDir() as tempdir: file_dict = {} if getattr(self._device_factory, "LOG_FILES", None): file_dict = self.PullLogs( self._device_factory.LOG_FILES, tempdir, user=ssh_user, ssh_rsa_path=ssh_rsa_path) # If the device is auto-connected, get adb logcat for file_path, file_name in self._CollectAdbLogcats( tempdir).items(): file_dict[file_path] = file_name utils.MakeTarFile(file_dict, output_file) @property def devices(self): """Returns a list of devices in the pool. Returns: A list of devices in the pool. """ return self._devices # TODO: Delete unused-argument when b/119614469 is resolved. # pylint: disable=unused-argument # pylint: disable=too-many-locals def CreateDevices(command, cfg, device_factory, num, avd_type, report_internal_ip=False, autoconnect=False, serial_log_file=None, logcat_file=None): """Create a set of devices using the given factory. Main jobs in create devices. 1. Create GCE instance: Launch instance in GCP(Google Cloud Platform). 2. Starting up AVD: Wait device boot up. Args: command: The name of the command, used for reporting. cfg: An AcloudConfig instance. device_factory: A factory capable of producing a single device. num: The number of devices to create. avd_type: String, the AVD type(cuttlefish, goldfish...). report_internal_ip: Boolean to report the internal ip instead of external ip. serial_log_file: String, the file path to tar the serial logs. logcat_file: String, the file path to tar the logcats. autoconnect: Boolean, whether to auto connect to device. Raises: errors: Create instance fail. Returns: A Report instance. """ reporter = report.Report(command=command) try: CreateSshKeyPairIfNecessary(cfg) device_pool = DevicePool(device_factory) device_pool.CreateDevices(num) failures = device_pool.WaitForBoot() if failures: reporter.SetStatus(report.Status.BOOT_FAIL) else: reporter.SetStatus(report.Status.SUCCESS) # Collect logs if serial_log_file: device_pool.CollectSerialPortLogs( serial_log_file, port=constants.DEFAULT_SERIAL_PORT) # TODO(b/119614469): Refactor CollectLogcats into a utils lib and # turn it on inside the reporting loop. # if logcat_file: # device_pool.CollectLogcats(logcat_file, ssh_user, ssh_rsa_path) # Write result to report. for device in device_pool.devices: ip = (device.ip.internal if report_internal_ip else device.ip.external) device_dict = { "ip": ip, "instance_name": device.instance_name } for attr in ("branch", "build_target", "build_id", "kernel_branch", "kernel_build_target", "kernel_build_id", "emulator_branch", "emulator_build_target", "emulator_build_id"): if getattr(device_factory, "_%s" % attr, None): device_dict[attr] = getattr(device_factory, "_%s" % attr) if autoconnect: forwarded_ports = utils.AutoConnect( ip, cfg.ssh_private_key_path, utils.AVD_PORT_DICT[avd_type].vnc_port, utils.AVD_PORT_DICT[avd_type].adb_port, getpass.getuser()) device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port device_dict[constants.ADB_PORT] = forwarded_ports.adb_port if device.instance_name in failures: reporter.AddData(key="devices_failing_boot", value=device_dict) reporter.AddError(str(failures[device.instance_name])) else: reporter.AddData(key="devices", value=device_dict) except errors.DriverError as e: reporter.AddError(str(e)) reporter.SetStatus(report.Status.FAIL) return reporter