• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2018 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""Common operations between managing GCE and Cuttlefish devices.
17
18This module provides the common operations between managing GCE (device_driver)
19and Cuttlefish (create_cuttlefish_action) devices. Should not be called
20directly.
21"""
22
23from __future__ import print_function
24import getpass
25import logging
26import os
27import subprocess
28
29from acloud import errors
30from acloud.public import avd
31from acloud.public import report
32from acloud.internal import constants
33from acloud.internal.lib import utils
34
35logger = logging.getLogger(__name__)
36
37
38def CreateSshKeyPairIfNecessary(cfg):
39    """Create ssh key pair if necessary.
40
41    Args:
42        cfg: An Acloudconfig instance.
43
44    Raises:
45        error.DriverError: If it falls into an unexpected condition.
46    """
47    if not cfg.ssh_public_key_path:
48        logger.warning(
49            "ssh_public_key_path is not specified in acloud config. "
50            "Project-wide public key will "
51            "be used when creating AVD instances. "
52            "Please ensure you have the correct private half of "
53            "a project-wide public key if you want to ssh into the "
54            "instances after creation.")
55    elif cfg.ssh_public_key_path and not cfg.ssh_private_key_path:
56        logger.warning(
57            "Only ssh_public_key_path is specified in acloud config, "
58            "but ssh_private_key_path is missing. "
59            "Please ensure you have the correct private half "
60            "if you want to ssh into the instances after creation.")
61    elif cfg.ssh_public_key_path and cfg.ssh_private_key_path:
62        utils.CreateSshKeyPairIfNotExist(cfg.ssh_private_key_path,
63                                         cfg.ssh_public_key_path)
64    else:
65        # Should never reach here.
66        raise errors.DriverError(
67            "Unexpected error in CreateSshKeyPairIfNecessary")
68
69
70class DevicePool(object):
71    """A class that manages a pool of virtual devices.
72
73    Attributes:
74        devices: A list of devices in the pool.
75    """
76
77    def __init__(self, device_factory, devices=None):
78        """Constructs a new DevicePool.
79
80        Args:
81            device_factory: A device factory capable of producing a goldfish or
82                cuttlefish device. The device factory must expose an attribute with
83                the credentials that can be used to retrieve information from the
84                constructed device.
85            devices: List of devices managed by this pool.
86        """
87        self._devices = devices or []
88        self._device_factory = device_factory
89        self._compute_client = device_factory.GetComputeClient()
90
91    def _CollectAdbLogcats(self, output_dir):
92        """Collect Adb logcats.
93
94        Args:
95            output_dir: String, the output file directory to store adb logcats.
96
97        Returns:
98            The file information dictionary with file path and file name.
99        """
100        file_dict = {}
101        for device in self._devices:
102            if not device.adb_port:
103                # If device adb tunnel is not established, do not do adb logcat
104                continue
105            file_name = "%s_adb_logcat.log" % device.instance_name
106            full_file_path = os.path.join(output_dir, file_name)
107            logger.info("Get adb %s:%s logcat for instance %s",
108                        constants.LOCALHOST, device.adb_port,
109                        device.instance_name)
110            try:
111                subprocess.check_call(
112                    ["adb -s %s:%s logcat -b all -d > %s" % (
113                        constants.LOCALHOST, device.adb_port, full_file_path)],
114                    shell=True)
115                file_dict[full_file_path] = file_name
116            except subprocess.CalledProcessError:
117                logging.error("Failed to get adb logcat for %s for instance %s",
118                              device.serial_number, device.instance_name)
119        return file_dict
120
121    def CreateDevices(self, num):
122        """Creates |num| devices for given build_target and build_id.
123
124        Args:
125            num: Number of devices to create.
126        """
127        # Create host instances for cuttlefish/goldfish device.
128        # Currently one instance supports only 1 device.
129        for _ in range(num):
130            instance = self._device_factory.CreateInstance()
131            ip = self._compute_client.GetInstanceIP(instance)
132            self.devices.append(
133                avd.AndroidVirtualDevice(ip=ip, instance_name=instance))
134
135    @utils.TimeExecute(function_description="Waiting for AVD(s) to boot up",
136                       result_evaluator=utils.BootEvaluator)
137    def WaitForBoot(self):
138        """Waits for all devices to boot up.
139
140        Returns:
141            A dictionary that contains all the failures.
142            The key is the name of the instance that fails to boot,
143            and the value is an errors.DeviceBootError object.
144        """
145        failures = {}
146        for device in self._devices:
147            try:
148                self._compute_client.WaitForBoot(device.instance_name)
149            except errors.DeviceBootError as e:
150                failures[device.instance_name] = e
151        return failures
152
153    def PullLogs(self, source_files, output_dir, user=None, ssh_rsa_path=None):
154        """Tar logs from GCE instance into output_dir.
155
156        Args:
157            source_files: List of file names to be pulled.
158            output_dir: String. The output file dirtory
159            user: String, the ssh username to access GCE
160            ssh_rsa_path: String, the ssh rsa key path to access GCE
161
162        Returns:
163            The file dictionary with file_path and file_name
164        """
165
166        file_dict = {}
167        for device in self._devices:
168            if isinstance(source_files, basestring):
169                source_files = [source_files]
170            for source_file in source_files:
171                file_name = "%s_%s" % (device.instance_name,
172                                       os.path.basename(source_file))
173                dst_file = os.path.join(output_dir, file_name)
174                logger.info("Pull %s for instance %s with user %s to %s",
175                            source_file, device.instance_name, user, dst_file)
176                try:
177                    utils.ScpPullFile(source_file, dst_file, device.ip,
178                                      user_name=user, rsa_key_file=ssh_rsa_path)
179                    file_dict[dst_file] = file_name
180                except errors.DeviceConnectionError as e:
181                    logger.warning("Failed to pull %s from instance %s: %s",
182                                   source_file, device.instance_name, e)
183        return file_dict
184
185    def CollectSerialPortLogs(self, output_file,
186                              port=constants.DEFAULT_SERIAL_PORT):
187        """Tar the instance serial logs into specified output_file.
188
189        Args:
190            output_file: String, the output tar file path
191            port: The serial port number to be collected
192        """
193        # For emulator, the serial log is the virtual host serial log.
194        # For GCE AVD device, the serial log is the AVD device serial log.
195        with utils.TempDir() as tempdir:
196            src_dict = {}
197            for device in self._devices:
198                logger.info("Store instance %s serial port %s output to %s",
199                            device.instance_name, port, output_file)
200                serial_log = self._compute_client.GetSerialPortOutput(
201                    instance=device.instance_name, port=port)
202                file_name = "%s_serial_%s.log" % (device.instance_name, port)
203                file_path = os.path.join(tempdir, file_name)
204                src_dict[file_path] = file_name
205                with open(file_path, "w") as f:
206                    f.write(serial_log.encode("utf-8"))
207            utils.MakeTarFile(src_dict, output_file)
208
209    def CollectLogcats(self, output_file, ssh_user, ssh_rsa_path):
210        """Tar the instances' logcat and other logs into specified output_file.
211
212        Args:
213            output_file: String, the output tar file path
214            ssh_user: The ssh user name
215            ssh_rsa_path: The ssh rsa key path
216        """
217        with utils.TempDir() as tempdir:
218            file_dict = {}
219            if getattr(self._device_factory, "LOG_FILES", None):
220                file_dict = self.PullLogs(
221                    self._device_factory.LOG_FILES, tempdir, user=ssh_user,
222                    ssh_rsa_path=ssh_rsa_path)
223            # If the device is auto-connected, get adb logcat
224            for file_path, file_name in self._CollectAdbLogcats(
225                    tempdir).items():
226                file_dict[file_path] = file_name
227            utils.MakeTarFile(file_dict, output_file)
228
229    @property
230    def devices(self):
231        """Returns a list of devices in the pool.
232
233        Returns:
234            A list of devices in the pool.
235        """
236        return self._devices
237
238# TODO: Delete unused-argument when b/119614469 is resolved.
239# pylint: disable=unused-argument
240# pylint: disable=too-many-locals
241def CreateDevices(command, cfg, device_factory, num, avd_type,
242                  report_internal_ip=False, autoconnect=False,
243                  serial_log_file=None, logcat_file=None):
244    """Create a set of devices using the given factory.
245
246    Main jobs in create devices.
247        1. Create GCE instance: Launch instance in GCP(Google Cloud Platform).
248        2. Starting up AVD: Wait device boot up.
249
250    Args:
251        command: The name of the command, used for reporting.
252        cfg: An AcloudConfig instance.
253        device_factory: A factory capable of producing a single device.
254        num: The number of devices to create.
255        avd_type: String, the AVD type(cuttlefish, goldfish...).
256        report_internal_ip: Boolean to report the internal ip instead of
257                            external ip.
258        serial_log_file: String, the file path to tar the serial logs.
259        logcat_file: String, the file path to tar the logcats.
260        autoconnect: Boolean, whether to auto connect to device.
261
262    Raises:
263        errors: Create instance fail.
264
265    Returns:
266        A Report instance.
267    """
268    reporter = report.Report(command=command)
269    try:
270        CreateSshKeyPairIfNecessary(cfg)
271        device_pool = DevicePool(device_factory)
272        device_pool.CreateDevices(num)
273        failures = device_pool.WaitForBoot()
274        if failures:
275            reporter.SetStatus(report.Status.BOOT_FAIL)
276        else:
277            reporter.SetStatus(report.Status.SUCCESS)
278
279        # Collect logs
280        if serial_log_file:
281            device_pool.CollectSerialPortLogs(
282                serial_log_file, port=constants.DEFAULT_SERIAL_PORT)
283        # TODO(b/119614469): Refactor CollectLogcats into a utils lib and
284        #                    turn it on inside the reporting loop.
285        # if logcat_file:
286        #     device_pool.CollectLogcats(logcat_file, ssh_user, ssh_rsa_path)
287
288        # Write result to report.
289        for device in device_pool.devices:
290            ip = (device.ip.internal if report_internal_ip
291                  else device.ip.external)
292            device_dict = {
293                "ip": ip,
294                "instance_name": device.instance_name
295            }
296            for attr in ("branch", "build_target", "build_id", "kernel_branch",
297                         "kernel_build_target", "kernel_build_id",
298                         "emulator_branch", "emulator_build_target",
299                         "emulator_build_id"):
300                if getattr(device_factory, "_%s" % attr, None):
301                    device_dict[attr] = getattr(device_factory, "_%s" % attr)
302            if autoconnect:
303                forwarded_ports = utils.AutoConnect(
304                    ip, cfg.ssh_private_key_path,
305                    utils.AVD_PORT_DICT[avd_type].vnc_port,
306                    utils.AVD_PORT_DICT[avd_type].adb_port,
307                    getpass.getuser())
308                device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port
309                device_dict[constants.ADB_PORT] = forwarded_ports.adb_port
310            if device.instance_name in failures:
311                reporter.AddData(key="devices_failing_boot", value=device_dict)
312                reporter.AddError(str(failures[device.instance_name]))
313            else:
314                reporter.AddData(key="devices", value=device_dict)
315    except errors.DriverError as e:
316        reporter.AddError(str(e))
317        reporter.SetStatus(report.Status.FAIL)
318    return reporter
319