• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Function tests of lxc module. To be able to run this test, following setup
6is required:
7  1. lxc is installed.
8  2. Autotest code exists in /usr/local/autotest, with site-packages installed.
9     (run utils/build_externals.py)
10  3. The user runs the test should have sudo access. Run the test with sudo.
11Note that the test does not require Autotest database and frontend.
12"""
13
14
15import argparse
16import logging
17import os
18import tempfile
19import time
20
21import common
22from autotest_lib.client.bin import utils
23from autotest_lib.site_utils import lxc
24from autotest_lib.site_utils.lxc import unittest_logging
25
26
27TEST_JOB_ID = 123
28TEST_JOB_FOLDER = '123-debug_user'
29# Create a temp directory for functional tests. The directory is not under /tmp
30# for Moblab to be able to run the test.
31TEMP_DIR = tempfile.mkdtemp(dir=lxc.DEFAULT_CONTAINER_PATH,
32                            prefix='container_test_')
33RESULT_PATH = os.path.join(TEMP_DIR, 'results', str(TEST_JOB_ID))
34# Link to download a test package of autotest server package.
35# Ideally the test should stage a build on devserver and download the
36# autotest_server_package from devserver. This test is focused on testing
37# container, so it's prefered to avoid dependency on devserver.
38AUTOTEST_SERVER_PKG = ('http://storage.googleapis.com/abci-ssp/'
39                       'autotest-containers/autotest_server_package.tar.bz2')
40
41# Test log file to be created in result folder, content is `test`.
42TEST_LOG = 'test.log'
43# Name of test script file to run in container.
44TEST_SCRIPT = 'test.py'
45# Test script to run in container to verify autotest code setup.
46TEST_SCRIPT_CONTENT = """
47import socket
48import sys
49
50# Test import
51import common
52import chromite
53
54# This test has to be before the import of autotest_lib, because ts_mon requires
55# httplib2 module in chromite/third_party. The one in Autotest site-packages is
56# out dated.
57%(ts_mon_test)s
58
59from autotest_lib.server import utils
60from autotest_lib.site_utils import lxc
61
62with open(sys.argv[1], 'w') as f:
63    f.write('test')
64
65# Confirm hostname starts with `test-`
66if not socket.gethostname().startswith('test-'):
67    raise Exception('The container\\\'s hostname must start with `test-`.')
68
69# Test installing packages
70lxc.install_packages(['atop'], ['acora'])
71
72"""
73
74TEST_SCRIPT_CONTENT_TS_MON = """
75# Test ts_mon metrics can be set up.
76from chromite.lib import ts_mon_config
77ts_mon_config.SetupTsMonGlobalState('some_test', suppress_exception=False)
78"""
79
80CREATE_FAKE_TS_MON_CONFIG_SCRIPT = 'create_fake_key.py'
81
82CREATE_FAKE_TS_MON_CONFIG_SCRIPT_CONTENT = """
83import os
84import rsa
85
86EXPECTED_TS_MON_CONFIG_NAME = '/etc/chrome-infra/ts-mon.json'
87
88FAKE_TS_MON_CONFIG_CONTENT = '''
89    {
90        "credentials":"/tmp/service_account_prodx_mon.json",
91        "endpoint":"https://xxx.googleapis.com/v1:insert",
92        "use_new_proto": true
93    }'''
94
95FAKE_SERVICE_ACCOUNT_CRED_JSON = '''
96    {
97        "type": "service_account",
98        "project_id": "test_project",
99        "private_key_id": "aaa",
100        "private_key": "%s",
101        "client_email": "xxx",
102        "client_id": "111",
103        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
104        "token_uri": "https://accounts.google.com/o/oauth2/token",
105        "auth_provider_x509_cert_url":
106                "https://www.googleapis.com/oauth2/v1/certs",
107        "client_x509_cert_url":
108                "https://www.googleapis.com/robot/v1/metadata/x509/xxx"
109    }'''
110
111
112TEST_KEY = '''------BEGIN PRIVATE KEY-----
113MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCzg4K2SXqf9LAM
11452a/t2HfpY5y49sbrgRb1llP6c8RVWhUX/pGdjbcIM97+1CJEWBN8Vmraoe4+71o
1151idTPehJfHRNeyXQUnro8CmnSxE9tLHtdKj0pzvO+yqT66O6Iw1aUAIX+dG4Us9Q
116Z22ypFHaJ74lKw9JFwAFTJ/TF1rXUXqgufYTNNqP3Ra7wCHF8BmtjwRYAlvsR9CO
117c4eVC1+qhq/8/EOMCgF/rsbZW93r/nz5xgsSX0k6WkAz5WX2mniHfmBFpmr039jZ
1180eI1mEMGDAYuUn05++dNveo/ZOZj3wBlFzyfNSeeWJB5SdKPTvN3H/Iu0Aw+Rtb6
119szwNClaFAgMBAAECggEAHZ8cjVRUJ/tiJorzlTyfKZ6hwhsPv4JIRVg6LhnceZWA
120jPW2cHSWyl2epyx55lhH7iyeeY7vXOqrX1aBMDb1stSWw2dH/tdxYSkqEmksa+R6
121fL6kl5RV5epjpPt77Z3VmPq9UbP/M310qKWcgB8lw4wN0AfKMqsZLYauk9BVhNRu
122Bgah9O7BmcXS+mp49w0Xyfo1UBvzW8R6UnBhHbf9aOY8ObMD0Jj/wDjlYMqSSIKR
1239/8GZWQEKe6q0PyRRdNNtdzbpBrR0fIw6/T9pfDR2fBAcpNvD50eJk2jRiRDTWFJ
124rVSc0bvZFb74Rc3LbMSXW/6Kb7I2IG1XsWw7nxp92QKBgQDgzdIxZrkNZ3Tbuzng
125SG4atjnaCXoekOHK7VZVYd30S0AAizeGu1sjpUVQgsf+qkFskXAQp2/2f+Wiuq2G
126+nJYvXwZ/r9IcUs/oD3Fa2ezCVz1N/HOSPFAZK9XZuZbL8sXEYIPGJWH5F8Sanmb
127xNp9IUynlpwgM2JlZNeTCkv4PQKBgQDMbL/AF3LSpKvwi+QvYVkX/gChQmNMr4pP
128TM/GI4D03tNrzsut3oerKMUw0c5MxonkAJpuACN6baRyBOBxRYQSt8wWkORg9iqy
129a7aHnQqIGRafydW1/Snhr2DJSSaViHfO0oaA1r61zgMUTnSGb3UjyxJQp65dvPac
130BhpR9wpz6QKBgQDR2S/CL8rEqXObfi1roREu3DYqw7f8enBb1qtFrsLbPbd0CoD9
131wz0zjB6lJj/9CP9jkmwTD8njR8ab3jkIDBfboJ4NQhFbVW7R6QpglH9L0Iy2189g
132KhUScCqBoyubqYSidxR6dQ94uATLkxsL/nmaXxBITL5XDMBoN/dIak86XQKBgDqa
133oo4LKtvAYZpgQFZk7gm2w693PMhrOpdpSddfrkSE7M9nRXTe6r3ivkU0oJPaBwXa
134Nmt6lrEuZYpaY42VhDtpfZSqjQ5PBAaKYpWWK8LAjn/YeO/nV+5fPLv3wJv1t4MP
135T4f4CExOdwuHQliX81kDioicyZwN5BTumvUMgW6hAoGAF29kI1KthKaHN9P1DchI
136qqoHb9FPdZ5I6HDQpn6fr9ut7+9kVqexUrQ2AMvcVei6gDWW6P3yDCdTKcV9qtts
1371JOP2aSmXvibflx/bNfnhu988qJDhJ3CCjfc79fjwntUIXNPsFmwC9W5lnlSMKHM
138rH4RdmnjeCIG1PZ35m/yUSU=
139-----END PRIVATE KEY-----'''
140
141if not os.path.exists(EXPECTED_TS_MON_CONFIG_NAME):
142    try:
143        os.makedirs(os.path.dirname(EXPECTED_TS_MON_CONFIG_NAME))
144    except OSError:
145        # Directory already exists.
146        pass
147
148    with open(EXPECTED_TS_MON_CONFIG_NAME, 'w') as f:
149        f.write(FAKE_TS_MON_CONFIG_CONTENT)
150    with open ('/tmp/service_account_prodx_mon.json', 'w') as f:
151        f.write(FAKE_SERVICE_ACCOUNT_CRED_JSON % repr(TEST_KEY)[2:-1])
152"""
153
154# Name of the test control file.
155TEST_CONTROL_FILE = 'attach.1'
156TEST_DUT = '172.27.213.193'
157TEST_RESULT_PATH = lxc.RESULT_DIR_FMT % TEST_JOB_FOLDER
158# Test autoserv command.
159AUTOSERV_COMMAND = (('/usr/bin/python -u /usr/local/autotest/server/autoserv '
160                     '-p -r %(result_path)s/%(test_dut)s -m %(test_dut)s '
161                     '-u debug_user -l test -s -P %(job_id)s-debug_user/'
162                     '%(test_dut)s -n %(result_path)s/%(test_control_file)s '
163                     '--verify_job_repo_url') %
164                     {'job_id': TEST_JOB_ID,
165                      'result_path': TEST_RESULT_PATH,
166                      'test_dut': TEST_DUT,
167                      'test_control_file': TEST_CONTROL_FILE})
168# Content of the test control file.
169TEST_CONTROL_CONTENT = """
170def run(machine):
171    job.run_test('dummy_PassServer',
172                 host=hosts.create_host(machine))
173
174parallel_simple(run, machines)
175"""
176
177
178def setup_base(bucket):
179    """Test setup base container works.
180
181    @param bucket: ContainerBucket to interact with containers.
182    """
183    logging.info('Rebuild base container in folder %s.', bucket.container_path)
184    bucket.setup_base()
185    containers = bucket.get_all()
186    logging.info('Containers created: %s', containers.keys())
187
188
189def setup_test(bucket, name, skip_cleanup):
190    """Test container can be created from base container.
191
192    @param bucket: ContainerBucket to interact with containers.
193    @param name: Name of the test container.
194    @param skip_cleanup: Set to True to skip cleanup, used to troubleshoot
195                         container failures.
196
197    @return: A Container object created for the test container.
198    """
199    logging.info('Create test container.')
200    os.makedirs(RESULT_PATH)
201    container = bucket.setup_test(name, TEST_JOB_ID, AUTOTEST_SERVER_PKG,
202                                  RESULT_PATH, skip_cleanup=skip_cleanup,
203                                  job_folder=TEST_JOB_FOLDER,
204                                  dut_name='192.168.0.3')
205
206    # Inject "AUTOSERV/testing_mode: True" in shadow config to test autoserv.
207    container.attach_run('echo $\'[AUTOSERV]\ntesting_mode: True\' >>'
208                         ' /usr/local/autotest/shadow_config.ini')
209
210    if not utils.is_moblab():
211        # Create fake '/etc/chrome-infra/ts-mon.json' if it doesn't exist.
212        create_key_script = os.path.join(
213                RESULT_PATH, CREATE_FAKE_TS_MON_CONFIG_SCRIPT)
214        with open(create_key_script, 'w') as script:
215            script.write(CREATE_FAKE_TS_MON_CONFIG_SCRIPT_CONTENT)
216        container_result_path = lxc.RESULT_DIR_FMT % TEST_JOB_FOLDER
217        container_create_key_script = os.path.join(
218                container_result_path, CREATE_FAKE_TS_MON_CONFIG_SCRIPT)
219        container.attach_run('python %s' % container_create_key_script)
220
221    return container
222
223
224def test_share(container):
225    """Test container can share files with the host.
226
227    @param container: The test container.
228    """
229    logging.info('Test files written to result directory can be accessed '
230                 'from the host running the container..')
231    host_test_script = os.path.join(RESULT_PATH, TEST_SCRIPT)
232    with open(host_test_script, 'w') as script:
233        if utils.is_moblab():
234            script.write(TEST_SCRIPT_CONTENT)
235        else:
236            script.write(TEST_SCRIPT_CONTENT %
237                         {'ts_mon_test': TEST_SCRIPT_CONTENT_TS_MON})
238
239    container_result_path = lxc.RESULT_DIR_FMT % TEST_JOB_FOLDER
240    container_test_script = os.path.join(container_result_path, TEST_SCRIPT)
241    container_test_script_dest = os.path.join('/usr/local/autotest/utils/',
242                                              TEST_SCRIPT)
243    container_test_log = os.path.join(container_result_path, TEST_LOG)
244    host_test_log = os.path.join(RESULT_PATH, TEST_LOG)
245    # Move the test script out of result folder as it needs to import common.
246    container.attach_run('mv %s %s' % (container_test_script,
247                                       container_test_script_dest))
248    container.attach_run('python %s %s' % (container_test_script_dest,
249                                           container_test_log))
250    if not os.path.exists(host_test_log):
251        raise Exception('Results created in container can not be accessed from '
252                        'the host.')
253    with open(host_test_log, 'r') as log:
254        if log.read() != 'test':
255            raise Exception('Failed to read the content of results in '
256                            'container.')
257
258
259def test_autoserv(container):
260    """Test container can run autoserv command.
261
262    @param container: The test container.
263    """
264    logging.info('Test autoserv command.')
265    logging.info('Create test control file.')
266    host_control_file = os.path.join(RESULT_PATH, TEST_CONTROL_FILE)
267    with open(host_control_file, 'w') as control_file:
268        control_file.write(TEST_CONTROL_CONTENT)
269
270    logging.info('Run autoserv command.')
271    container.attach_run(AUTOSERV_COMMAND)
272
273    logging.info('Confirm results are available from host.')
274    # Read status.log to check the content is not empty.
275    container_status_log = os.path.join(TEST_RESULT_PATH, TEST_DUT,
276                                        'status.log')
277    status_log = container.attach_run(command='cat %s' % container_status_log
278                                      ).stdout
279    if len(status_log) < 10:
280        raise Exception('Failed to read status.log in container.')
281
282
283def test_package_install(container):
284    """Test installing package in container.
285
286    @param container: The test container.
287    """
288    # Packages are installed in TEST_SCRIPT_CONTENT. Verify the packages in
289    # this method.
290    container.attach_run('which atop')
291    container.attach_run('python -c "import acora"')
292
293
294def test_ssh(container, remote):
295    """Test container can run ssh to remote server.
296
297    @param container: The test container.
298    @param remote: The remote server to ssh to.
299
300    @raise: error.CmdError if container can't ssh to remote server.
301    """
302    logging.info('Test ssh to %s.', remote)
303    container.attach_run('ssh %s -a -x -o StrictHostKeyChecking=no '
304                         '-o BatchMode=yes -o UserKnownHostsFile=/dev/null '
305                         '-p 22 "true"' % remote)
306
307
308def parse_options():
309    """Parse command line inputs.
310    """
311    parser = argparse.ArgumentParser()
312    parser.add_argument('-d', '--dut', type=str,
313                        help='Test device to ssh to.',
314                        default=None)
315    parser.add_argument('-r', '--devserver', type=str,
316                        help='Test devserver to ssh to.',
317                        default=None)
318    parser.add_argument('-v', '--verbose', action='store_true',
319                        default=False,
320                        help='Print out ALL entries.')
321    parser.add_argument('-s', '--skip_cleanup', action='store_true',
322                        default=False,
323                        help='Skip deleting test containers.')
324    return parser.parse_args()
325
326
327def main(options):
328    """main script.
329
330    @param options: Options to run the script.
331    """
332    # Force to run the test as superuser.
333    # TODO(dshi): crbug.com/459344 Set remove this enforcement when test
334    # container can be unprivileged container.
335    if utils.sudo_require_password():
336        logging.warn('SSP requires root privilege to run commands, please '
337                     'grant root access to this process.')
338        utils.run('sudo true')
339
340    log_level=(logging.DEBUG if options.verbose else logging.INFO)
341    unittest_logging.setup(log_level)
342
343    bucket = lxc.ContainerBucket(TEMP_DIR)
344
345    setup_base(bucket)
346    container_test_name = (lxc.TEST_CONTAINER_NAME_FMT %
347                           (TEST_JOB_ID, time.time(), os.getpid()))
348    container = setup_test(bucket, container_test_name, options.skip_cleanup)
349    test_share(container)
350    test_autoserv(container)
351    if options.dut:
352        test_ssh(container, options.dut)
353    if options.devserver:
354        test_ssh(container, options.devserver)
355    # Packages are installed in TEST_SCRIPT, verify the packages are installed.
356    test_package_install(container)
357    logging.info('All tests passed.')
358
359
360if __name__ == '__main__':
361    options = parse_options()
362    try:
363        main(options)
364    finally:
365        if not options.skip_cleanup:
366            logging.info('Cleaning up temporary directory %s.', TEMP_DIR)
367            try:
368                lxc.ContainerBucket(TEMP_DIR).destroy_all()
369            finally:
370                utils.run('sudo rm -rf "%s"' % TEMP_DIR)
371