1# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import re 8import time 9 10import common 11from autotest_lib.client.common_lib import error, global_config 12from autotest_lib.client.common_lib.cros import retry 13from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 14from autotest_lib.server.hosts import cros_host 15from autotest_lib.server.hosts import cros_repair 16 17 18AUTOTEST_INSTALL_DIR = global_config.global_config.get_config_value( 19 'SCHEDULER', 'drone_installation_directory') 20 21ENABLE_SSH_TUNNEL_FOR_MOBLAB = global_config.global_config.get_config_value( 22 'CROS', 'enable_ssh_tunnel_for_moblab', type=bool, default=False) 23 24#'/usr/local/autotest' 25SHADOW_CONFIG_PATH = '%s/shadow_config.ini' % AUTOTEST_INSTALL_DIR 26ATEST_PATH = '%s/cli/atest' % AUTOTEST_INSTALL_DIR 27SUBNET_DUT_SEARCH_RE = ( 28 r'/?.*\((?P<ip>192.168.231.*)\) at ' 29 '(?P<mac>[0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])') 30MOBLAB_IMAGE_STORAGE = '/mnt/moblab/static' 31MOBLAB_HOME = '/home/moblab' 32MOBLAB_BOTO_LOCATION = '%s/.boto' % MOBLAB_HOME 33MOBLAB_LAUNCH_CONTROL_KEY_LOCATION = '%s/.launch_control_key' % MOBLAB_HOME 34MOBLAB_SERVICE_ACCOUNT_LOCATION = '%s/.service_account.json' % MOBLAB_HOME 35MOBLAB_AUTODIR = '/usr/local/autodir' 36DHCPD_LEASE_FILE = '/var/lib/dhcp/dhcpd.leases' 37MOBLAB_SERVICES = ['moblab-scheduler-init', 38 'moblab-database-init', 39 'moblab-devserver-init', 40 'moblab-gsoffloader-init', 41 'moblab-gsoffloader_s-init'] 42MOBLAB_PROCESSES = ['apache2', 'dhcpd'] 43DUT_VERIFY_SLEEP_SECS = 5 44DUT_VERIFY_TIMEOUT = 15 * 60 45MOBLAB_TMP_DIR = '/mnt/moblab/tmp' 46MOBLAB_PORT = 80 47 48 49class MoblabHost(cros_host.CrosHost): 50 """Moblab specific host class.""" 51 52 53 def _initialize_frontend_rpcs(self, timeout_min): 54 """Initialize frontends for AFE and TKO for a moblab host. 55 56 AFE and TKO are initialized differently based on |_use_tunnel|, 57 which indicates that whether to use ssh tunnel to connect to moblab. 58 59 @param timeout_min: The timeout minuties for AFE services. 60 """ 61 if self._use_tunnel: 62 self.web_address = self.rpc_server_tracker.tunnel_connect( 63 MOBLAB_PORT) 64 # Pass timeout_min to self.afe 65 self.afe = frontend_wrappers.RetryingAFE(timeout_min=timeout_min, 66 user='moblab', 67 server=self.web_address) 68 # Use default timeout_min of MoblabHost for self.tko 69 self.tko = frontend_wrappers.RetryingTKO(timeout_min=self.timeout_min, 70 user='moblab', 71 server=self.web_address) 72 73 74 def _initialize(self, *args, **dargs): 75 super(MoblabHost, self)._initialize(*args, **dargs) 76 # TODO(jrbarnette): Our superclass already initialized 77 # _repair_strategy, and now we're re-initializing it here. 78 # That's awkward, if not actually wrong. 79 self._repair_strategy = cros_repair.create_moblab_repair_strategy() 80 81 # Clear the Moblab Image Storage so that staging an image is properly 82 # tested. 83 if dargs.get('retain_image_storage') is not True: 84 self.run('rm -rf %s/*' % MOBLAB_IMAGE_STORAGE) 85 self.web_address = dargs.get('web_address', self.hostname) 86 self._use_tunnel = (ENABLE_SSH_TUNNEL_FOR_MOBLAB and 87 self.web_address == self.hostname) 88 self.timeout_min = dargs.get('rpc_timeout_min', 1) 89 self._initialize_frontend_rpcs(self.timeout_min) 90 91 92 @staticmethod 93 def check_host(host, timeout=10): 94 """ 95 Check if the given host is an moblab host. 96 97 @param host: An ssh host representing a device. 98 @param timeout: The timeout for the run command. 99 100 101 @return: True if the host device has adb. 102 103 @raises AutoservRunError: If the command failed. 104 @raises AutoservSSHTimeout: Ssh connection has timed out. 105 """ 106 try: 107 result = host.run( 108 'grep -q moblab /etc/lsb-release && ' 109 '! test -f /mnt/stateful_partition/.android_tester', 110 ignore_status=True, timeout=timeout) 111 except (error.AutoservRunError, error.AutoservSSHTimeout): 112 return False 113 return result.exit_status == 0 114 115 116 def install_boto_file(self, boto_path=''): 117 """Install a boto file on the Moblab device. 118 119 @param boto_path: Path to the boto file to install. If None, sends the 120 boto file in the current HOME directory. 121 122 @raises error.TestError if the boto file does not exist. 123 """ 124 if not boto_path: 125 boto_path = os.path.join(os.getenv('HOME'), '.boto') 126 if not os.path.exists(boto_path): 127 raise error.TestError('Boto File:%s does not exist.' % boto_path) 128 self.send_file(boto_path, MOBLAB_BOTO_LOCATION) 129 self.run('chown moblab:moblab %s' % MOBLAB_BOTO_LOCATION) 130 131 132 def get_autodir(self): 133 """Return the directory to install autotest for client side tests.""" 134 return self.autodir or MOBLAB_AUTODIR 135 136 137 def run_as_moblab(self, command, **kwargs): 138 """Moblab commands should be ran as the moblab user not root. 139 140 @param command: Command to run as user moblab. 141 """ 142 command = "su - moblab -c '%s'" % command 143 return self.run(command, **kwargs) 144 145 146 def reboot(self, **dargs): 147 """Reboot the Moblab Host and wait for its services to restart.""" 148 super(MoblabHost, self).reboot(**dargs) 149 # In general after a reboot, we want to wait till the web frontend 150 # and other Autotest services are up before executing. However should 151 # something be wrong with these services, repair needs to be able 152 # to continue and reimage the device. 153 try: 154 self.wait_afe_up() 155 except Exception as e: 156 logging.error('DUT has rebooted but AFE has failed to load.: %s', 157 e) 158 159 160 def wait_afe_up(self, timeout_min=5): 161 """Wait till the AFE is up and loaded. 162 163 Attempt to reach the Moblab's AFE and database through its RPC 164 interface. 165 166 @param timeout_min: Minutes to wait for the AFE to respond. Default is 167 5 minutes. 168 169 @raises urllib2.HTTPError if AFE does not respond within the timeout. 170 """ 171 # Use moblabhost's own AFE object with a longer timeout to wait for the 172 # AFE to load. Also re-create the ssh tunnel for connections to moblab. 173 # Set the timeout_min to be longer than self.timeout_min for rebooting. 174 self._initialize_frontend_rpcs(timeout_min) 175 # Verify the AFE can handle a simple request. 176 self._check_afe() 177 # Reset the timeout_min after rebooting checks for afe services. 178 self.afe.set_timeout(self.timeout_min) 179 180 181 def _wake_devices(self): 182 """Search the subnet and attempt to ping any available duts. 183 184 Fills up the arp table with entries about devices on the subnet. 185 186 Either uses fping or directly pings devices listed in the dhcpd lease 187 file. 188 """ 189 fping_result = self.run(('fping -g 192.168.231.100 192.168.231.110 ' 190 '-a -c 10 -p 30 -q'), 191 ignore_status=True) 192 # If fping is not on the system, ping entries in the dhcpd lease file. 193 if fping_result.exit_status == 127: 194 leases = set(self.run('grep ^lease %s' % DHCPD_LEASE_FILE, 195 ignore_status=True).stdout.splitlines()) 196 for lease in leases: 197 ip = re.match('lease (?P<ip>.*) {', lease).groups('ip') 198 self.run('ping %s -w 1' % ip, ignore_status=True) 199 200 201 def add_dut(self, hostname): 202 """Add a DUT hostname to the AFE. 203 204 @param hostname: DUT hostname to add. 205 """ 206 result = self.run_as_moblab('%s host create %s' % (ATEST_PATH, 207 hostname)) 208 logging.debug('atest host create output for host %s:\n%s', 209 hostname, result.stdout) 210 211 212 def find_and_add_duts(self): 213 """Discover DUTs on the testing subnet and add them to the AFE. 214 215 Runs 'arp -a' on the Moblab host and parses the output to discover DUTs 216 and if they are not already in the AFE, adds them. 217 """ 218 self._wake_devices() 219 existing_hosts = [host.hostname for host in self.afe.get_hosts()] 220 arp_command = self.run('arp -a') 221 for line in arp_command.stdout.splitlines(): 222 match = re.match(SUBNET_DUT_SEARCH_RE, line) 223 if match: 224 dut_hostname = match.group('ip') 225 if dut_hostname in existing_hosts: 226 break 227 # SSP package ip's start at 150 for the moblab, so it is not 228 # a DUT 229 if int(dut_hostname.split('.')[-1]) > 150: 230 break 231 self.add_dut(dut_hostname) 232 233 234 def verify_software(self): 235 """Verify working software on a Chrome OS system. 236 237 Tests for the following conditions: 238 1. All conditions tested by the parent version of this 239 function. 240 2. Ensures that Moblab services are running. 241 3. Ensures that both DUTs successfully run Verify. 242 243 """ 244 # In case cleanup or powerwash wiped the autodir, create an empty 245 # directory. 246 self.run('mkdir -p %s' % MOBLAB_AUTODIR) 247 super(MoblabHost, self).verify_software() 248 self._verify_moblab_services() 249 self._verify_duts() 250 251 252 @retry.retry(error.AutoservError, timeout_min=2, delay_sec=10) 253 def _verify_upstart_service(self, service): 254 """Retry to verify the required moblab services are up and running. 255 256 Regarding crbug.com/649811, moblab services takes longer to restart 257 under the new provision framework. This is a fix to retry the service 258 check until all services are successfully restarted. 259 260 @param service: the moblab upstart service. 261 262 @return True if this service is started and running, otherwise False. 263 """ 264 return self.upstart_status(service) 265 266 267 def _verify_moblab_services(self): 268 """Verify the required Moblab services are up and running. 269 270 @raises AutoservError if any moblab service is not running. 271 """ 272 for service in MOBLAB_SERVICES: 273 if not self._verify_upstart_service(service): 274 raise error.AutoservError('Moblab service: %s is not running.' 275 % service) 276 for process in MOBLAB_PROCESSES: 277 try: 278 self.run('pgrep %s' % process) 279 except error.AutoservRunError: 280 raise error.AutoservError('Moblab process: %s is not running.' 281 % process) 282 283 284 def _check_afe(self): 285 """Verify whether afe of moblab works before verify its DUTs. 286 287 Verifying moblab sometimes happens after a successful provision, in 288 which case moblab is restarted but tunnel of afe is not re-connected. 289 This func is used to check whether afe is working now. 290 291 @return True if afe works, otherwise, raise urllib2.HTTPError. 292 """ 293 try: 294 self.afe.get_hosts() 295 except: 296 logging.debug('AFE is not responding') 297 raise 298 299 return True 300 301 302 def _verify_duts(self): 303 """Verify the Moblab DUTs are up and running. 304 305 @raises AutoservError if no DUTs are in the Ready State. 306 """ 307 # Check whether afe is well connected, if not, restart it. 308 try: 309 self._check_afe() 310 except: 311 self.wait_afe_up() 312 313 # Add the DUTs if they have not yet been added. 314 self.find_and_add_duts() 315 # Ensure a boto file is installed in case this Moblab was wiped in 316 # repair. 317 self.install_boto_file() 318 hosts = self.afe.reverify_hosts() 319 logging.debug('DUTs scheduled for reverification: %s', hosts) 320 # Wait till all pending special tasks are completed. 321 total_time = 0 322 while (self.afe.get_special_tasks(is_complete=False) and 323 total_time < DUT_VERIFY_TIMEOUT): 324 total_time = total_time + DUT_VERIFY_SLEEP_SECS 325 time.sleep(DUT_VERIFY_SLEEP_SECS) 326 if not self.afe.get_hosts(status='Ready'): 327 for host in self.afe.get_hosts(): 328 logging.error('DUT: %s Status: %s', host, host.status) 329 raise error.AutoservError('Moblab has 0 Ready DUTs') 330 331 332 def get_platform(self): 333 """Determine the correct platform label for this host. 334 335 For Moblab devices '_moblab' is appended. 336 337 @returns a string representing this host's platform. 338 """ 339 return super(MoblabHost, self).get_platform() + '_moblab' 340 341 342 def make_tmp_dir(self, base=MOBLAB_TMP_DIR): 343 """Creates a temporary directory. 344 345 @param base: The directory where it should be created. 346 347 @return Path to a newly created temporary directory. 348 """ 349 self.run('mkdir -p %s' % base) 350 return self.run('mktemp -d -p %s' % base).stdout.strip() 351 352 353 def get_os_type(self): 354 return 'moblab' 355