• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import os, logging, datetime, glob, shutil
2from autotest_lib.client.bin import utils, os_dep
3from autotest_lib.client.common_lib import error
4import virt_utils, virt_installer
5
6
7def kill_qemu_processes():
8    """
9    Kills all qemu processes, also kills all processes holding /dev/kvm down.
10    """
11    logging.debug("Killing any qemu processes that might be left behind")
12    utils.system("pkill qemu", ignore_status=True)
13    # Let's double check to see if some other process is holding /dev/kvm
14    if os.path.isfile("/dev/kvm"):
15        utils.system("fuser -k /dev/kvm", ignore_status=True)
16
17
18def create_symlinks(test_bindir, prefix=None, bin_list=None, unittest=None):
19    """
20    Create symbolic links for the appropriate qemu and qemu-img commands on
21    the kvm test bindir.
22
23    @param test_bindir: KVM test bindir
24    @param prefix: KVM prefix path
25    @param bin_list: List of qemu binaries to link
26    @param unittest: Path to configuration file unittests.cfg
27    """
28    qemu_path = os.path.join(test_bindir, "qemu")
29    qemu_img_path = os.path.join(test_bindir, "qemu-img")
30    qemu_unittest_path = os.path.join(test_bindir, "unittests")
31    if os.path.lexists(qemu_path):
32        os.unlink(qemu_path)
33    if os.path.lexists(qemu_img_path):
34        os.unlink(qemu_img_path)
35    if unittest and os.path.lexists(qemu_unittest_path):
36        os.unlink(qemu_unittest_path)
37
38    logging.debug("Linking qemu binaries")
39
40    if bin_list:
41        for bin in bin_list:
42            if os.path.basename(bin) == 'qemu-kvm':
43                os.symlink(bin, qemu_path)
44            elif os.path.basename(bin) == 'qemu-img':
45                os.symlink(bin, qemu_img_path)
46
47    elif prefix:
48        kvm_qemu = os.path.join(prefix, "bin", "qemu-system-x86_64")
49        if not os.path.isfile(kvm_qemu):
50            raise error.TestError('Invalid qemu path')
51        kvm_qemu_img = os.path.join(prefix, "bin", "qemu-img")
52        if not os.path.isfile(kvm_qemu_img):
53            raise error.TestError('Invalid qemu-img path')
54        os.symlink(kvm_qemu, qemu_path)
55        os.symlink(kvm_qemu_img, qemu_img_path)
56
57    if unittest:
58        logging.debug("Linking unittest dir")
59        os.symlink(unittest, qemu_unittest_path)
60
61
62def install_roms(rom_dir, prefix):
63    logging.debug("Path to roms specified. Copying roms to install prefix")
64    rom_dst_dir = os.path.join(prefix, 'share', 'qemu')
65    for rom_src in glob.glob('%s/*.bin' % rom_dir):
66        rom_dst = os.path.join(rom_dst_dir, os.path.basename(rom_src))
67        logging.debug("Copying rom file %s to %s", rom_src, rom_dst)
68        shutil.copy(rom_src, rom_dst)
69
70
71class KvmInstallException(Exception):
72    pass
73
74
75class FailedKvmInstall(KvmInstallException):
76    pass
77
78
79class KvmNotInstalled(KvmInstallException):
80    pass
81
82
83class BaseInstaller(object):
84    def __init__(self, mode=None):
85        self.install_mode = mode
86        self._full_module_list = None
87
88    def set_install_params(self, test, params):
89        self.params = params
90
91        load_modules = params.get('load_modules', 'no')
92        if not load_modules or load_modules == 'yes':
93            self.should_load_modules = True
94        elif load_modules == 'no':
95            self.should_load_modules = False
96        default_extra_modules = str(None)
97        self.extra_modules = eval(params.get("extra_modules",
98                                             default_extra_modules))
99
100        self.cpu_vendor = virt_utils.get_cpu_vendor()
101
102        self.srcdir = test.srcdir
103        if not os.path.isdir(self.srcdir):
104            os.makedirs(self.srcdir)
105
106        self.test_bindir = test.bindir
107        self.results_dir = test.resultsdir
108
109        # KVM build prefix, for the modes that do need it
110        prefix = os.path.join(test.bindir, 'build')
111        self.prefix = os.path.abspath(prefix)
112
113        # Current host kernel directory
114        default_host_kernel_source = '/lib/modules/%s/build' % os.uname()[2]
115        self.host_kernel_srcdir = params.get('host_kernel_source',
116                                             default_host_kernel_source)
117
118        # Extra parameters that can be passed to the configure script
119        self.extra_configure_options = params.get('extra_configure_options',
120                                                  None)
121
122        # Do we want to save the result of the build on test.resultsdir?
123        self.save_results = True
124        save_results = params.get('save_results', 'no')
125        if save_results == 'no':
126            self.save_results = False
127
128        self._full_module_list = list(self._module_list())
129
130
131    def install_unittests(self):
132        userspace_srcdir = os.path.join(self.srcdir, "kvm_userspace")
133        test_repo = self.params.get("test_git_repo")
134        test_branch = self.params.get("test_branch", "master")
135        test_commit = self.params.get("test_commit", None)
136        test_lbranch = self.params.get("test_lbranch", "master")
137
138        if test_repo:
139            test_srcdir = os.path.join(self.srcdir, "kvm-unit-tests")
140            virt_utils.get_git_branch(test_repo, test_branch, test_srcdir,
141                                     test_commit, test_lbranch)
142            unittest_cfg = os.path.join(test_srcdir, 'x86',
143                                        'unittests.cfg')
144            self.test_srcdir = test_srcdir
145        else:
146            unittest_cfg = os.path.join(userspace_srcdir, 'kvm', 'test', 'x86',
147                                        'unittests.cfg')
148        self.unittest_cfg = None
149        if os.path.isfile(unittest_cfg):
150            self.unittest_cfg = unittest_cfg
151        else:
152            if test_repo:
153                logging.error("No unittest config file %s found, skipping "
154                              "unittest build", self.unittest_cfg)
155
156        self.unittest_prefix = None
157        if self.unittest_cfg:
158            logging.info("Building and installing unittests")
159            os.chdir(os.path.dirname(os.path.dirname(self.unittest_cfg)))
160            utils.system('./configure --prefix=%s' % self.prefix)
161            utils.system('make')
162            utils.system('make install')
163            self.unittest_prefix = os.path.join(self.prefix, 'share', 'qemu',
164                                                'tests')
165
166
167    def full_module_list(self):
168        """Return the module list used by the installer
169
170        Used by the module_probe test, to avoid using utils.unload_module().
171        """
172        if self._full_module_list is None:
173            raise KvmNotInstalled("KVM modules not installed yet (installer: %s)" % (type(self)))
174        return self._full_module_list
175
176
177    def _module_list(self):
178        """Generate the list of modules that need to be loaded
179        """
180        yield 'kvm'
181        yield 'kvm-%s' % (self.cpu_vendor)
182        if self.extra_modules:
183            for module in self.extra_modules:
184                yield module
185
186
187    def _load_modules(self, mod_list):
188        """
189        Load the KVM modules
190
191        May be overridden by subclasses.
192        """
193        logging.info("Loading KVM modules")
194        for module in mod_list:
195            utils.system("modprobe %s" % module)
196
197
198    def load_modules(self, mod_list=None):
199        if mod_list is None:
200            mod_list = self.full_module_list()
201        self._load_modules(mod_list)
202
203
204    def _unload_modules(self, mod_list=None):
205        """
206        Just unload the KVM modules, without trying to kill Qemu
207        """
208        if mod_list is None:
209            mod_list = self.full_module_list()
210        logging.info("Unloading previously loaded KVM modules")
211        for module in reversed(mod_list):
212            utils.unload_module(module)
213
214
215    def unload_modules(self, mod_list=None):
216        """
217        Kill Qemu and unload the KVM modules
218        """
219        kill_qemu_processes()
220        self._unload_modules(mod_list)
221
222
223    def reload_modules(self):
224        """
225        Reload the KVM modules after killing Qemu and unloading the current modules
226        """
227        self.unload_modules()
228        self.load_modules()
229
230
231    def reload_modules_if_needed(self):
232        if self.should_load_modules:
233            self.reload_modules()
234
235
236class YumInstaller(BaseInstaller):
237    """
238    Class that uses yum to install and remove packages.
239    """
240    def set_install_params(self, test, params):
241        super(YumInstaller, self).set_install_params(test, params)
242        # Checking if all required dependencies are available
243        os_dep.command("rpm")
244        os_dep.command("yum")
245
246        default_pkg_list = str(['qemu-kvm', 'qemu-kvm-tools'])
247        default_qemu_bin_paths = str(['/usr/bin/qemu-kvm', '/usr/bin/qemu-img'])
248        default_pkg_path_list = str(None)
249        self.pkg_list = eval(params.get("pkg_list", default_pkg_list))
250        self.pkg_path_list = eval(params.get("pkg_path_list",
251                                             default_pkg_path_list))
252        self.qemu_bin_paths = eval(params.get("qemu_bin_paths",
253                                              default_qemu_bin_paths))
254
255
256    def _clean_previous_installs(self):
257        kill_qemu_processes()
258        removable_packages = ""
259        for pkg in self.pkg_list:
260            removable_packages += " %s" % pkg
261
262        utils.system("yum remove -y %s" % removable_packages)
263
264
265    def _get_packages(self):
266        for pkg in self.pkg_path_list:
267            utils.get_file(pkg, os.path.join(self.srcdir,
268                                             os.path.basename(pkg)))
269
270
271    def _install_packages(self):
272        """
273        Install all downloaded packages.
274        """
275        os.chdir(self.srcdir)
276        utils.system("yum install --nogpgcheck -y *.rpm")
277
278
279    def install(self):
280        self.install_unittests()
281        self._clean_previous_installs()
282        self._get_packages()
283        self._install_packages()
284        create_symlinks(test_bindir=self.test_bindir,
285                        bin_list=self.qemu_bin_paths,
286                        unittest=self.unittest_prefix)
287        self.reload_modules_if_needed()
288        if self.save_results:
289            virt_utils.archive_as_tarball(self.srcdir, self.results_dir)
290
291
292class KojiInstaller(YumInstaller):
293    """
294    Class that handles installing KVM from the fedora build service, koji.
295
296    It uses yum to install and remove packages. Packages are specified
297    according to the syntax defined in the PkgSpec class.
298    """
299    def set_install_params(self, test, params):
300        """
301        Gets parameters and initializes the package downloader.
302
303        @param test: kvm test object
304        @param params: Dictionary with test arguments
305        """
306        super(KojiInstaller, self).set_install_params(test, params)
307        self.tag = params.get("koji_tag", None)
308        self.koji_cmd = params.get("koji_cmd", None)
309        if self.tag is not None:
310            virt_utils.set_default_koji_tag(self.tag)
311        self.koji_pkgs = eval(params.get("koji_pkgs", "[]"))
312
313
314    def _get_packages(self):
315        """
316        Downloads the specific arch RPMs for the specific build name.
317        """
318        koji_client = virt_utils.KojiClient(cmd=self.koji_cmd)
319        for pkg_text in self.koji_pkgs:
320            pkg = virt_utils.KojiPkgSpec(pkg_text)
321            if pkg.is_valid():
322                koji_client.get_pkgs(pkg, dst_dir=self.srcdir)
323            else:
324                logging.error('Package specification (%s) is invalid: %s', pkg,
325                              pkg.describe_invalid())
326
327
328    def _clean_previous_installs(self):
329        kill_qemu_processes()
330        removable_packages = " ".join(self._get_rpm_names())
331        utils.system("yum -y remove %s" % removable_packages)
332
333
334    def install(self):
335        self._clean_previous_installs()
336        self._get_packages()
337        self._install_packages()
338        self.install_unittests()
339        create_symlinks(test_bindir=self.test_bindir,
340                        bin_list=self.qemu_bin_paths,
341                        unittest=self.unittest_prefix)
342        self.reload_modules_if_needed()
343        if self.save_results:
344            virt_utils.archive_as_tarball(self.srcdir, self.results_dir)
345
346
347    def _get_rpm_names(self):
348        all_rpm_names = []
349        koji_client = virt_utils.KojiClient(cmd=self.koji_cmd)
350        for pkg_text in self.koji_pkgs:
351            pkg = virt_utils.KojiPkgSpec(pkg_text)
352            rpm_names = koji_client.get_pkg_rpm_names(pkg)
353            all_rpm_names += rpm_names
354        return all_rpm_names
355
356
357    def _get_rpm_file_names(self):
358        all_rpm_file_names = []
359        koji_client = virt_utils.KojiClient(cmd=self.koji_cmd)
360        for pkg_text in self.koji_pkgs:
361            pkg = virt_utils.KojiPkgSpec(pkg_text)
362            rpm_file_names = koji_client.get_pkg_rpm_file_names(pkg)
363            all_rpm_file_names += rpm_file_names
364        return all_rpm_file_names
365
366
367    def _install_packages(self):
368        """
369        Install all downloaded packages.
370        """
371        os.chdir(self.srcdir)
372        rpm_file_names = " ".join(self._get_rpm_file_names())
373        utils.system("yum --nogpgcheck -y localinstall %s" % rpm_file_names)
374
375
376class SourceDirInstaller(BaseInstaller):
377    """
378    Class that handles building/installing KVM directly from a tarball or
379    a single source code dir.
380    """
381    def set_install_params(self, test, params):
382        """
383        Initializes class attributes, and retrieves KVM code.
384
385        @param test: kvm test object
386        @param params: Dictionary with test arguments
387        """
388        super(SourceDirInstaller, self).set_install_params(test, params)
389
390        self.mod_install_dir = os.path.join(self.prefix, 'modules')
391
392        srcdir = params.get("srcdir", None)
393        self.path_to_roms = params.get("path_to_rom_images", None)
394
395        if self.install_mode == 'localsrc':
396            if srcdir is None:
397                raise error.TestError("Install from source directory specified"
398                                      "but no source directory provided on the"
399                                      "control file.")
400            else:
401                shutil.copytree(srcdir, self.srcdir)
402
403        elif self.install_mode == 'localtar':
404            tarball = params.get("tarball")
405            if not tarball:
406                raise error.TestError("KVM Tarball install specified but no"
407                                      " tarball provided on control file.")
408            logging.info("Installing KVM from a local tarball")
409            logging.info("Using tarball %s")
410            tarball = utils.unmap_url("/", params.get("tarball"), "/tmp")
411            utils.extract_tarball_to_dir(tarball, self.srcdir)
412
413        if self.install_mode in ['localtar', 'srcdir']:
414            self.repo_type = virt_utils.check_kvm_source_dir(self.srcdir)
415            p = os.path.join(self.srcdir, 'configure')
416            self.configure_options = virt_installer.check_configure_options(p)
417
418
419    def _build(self):
420        make_jobs = utils.count_cpus()
421        os.chdir(self.srcdir)
422        # For testing purposes, it's better to build qemu binaries with
423        # debugging symbols, so we can extract more meaningful stack traces.
424        cfg = "./configure --prefix=%s" % self.prefix
425        if "--disable-strip" in self.configure_options:
426            cfg += " --disable-strip"
427        steps = [cfg, "make clean", "make -j %s" % make_jobs]
428        logging.info("Building KVM")
429        for step in steps:
430            utils.system(step)
431
432
433    def _install(self):
434        os.chdir(self.srcdir)
435        logging.info("Installing KVM userspace")
436        if self.repo_type == 1:
437            utils.system("make -C qemu install")
438        elif self.repo_type == 2:
439            utils.system("make install")
440        if self.path_to_roms:
441            install_roms(self.path_to_roms, self.prefix)
442        self.install_unittests()
443        create_symlinks(test_bindir=self.test_bindir,
444                        prefix=self.prefix,
445                        unittest=self.unittest_prefix)
446
447
448    def install(self):
449        self._build()
450        self._install()
451        self.reload_modules_if_needed()
452        if self.save_results:
453            virt_utils.archive_as_tarball(self.srcdir, self.results_dir)
454
455class GitRepo(object):
456    def __init__(self, installer, prefix,
457            srcdir, build_steps=[], repo_param=None):
458        params = installer.params
459        self.installer = installer
460        self.repo = params.get(repo_param or (prefix + '_repo'))
461        self.branch = params.get(prefix + '_branch', 'master')
462        self.lbranch = params.get(prefix + '_lbranch', 'master')
463        self.commit = params.get(prefix + '_commit', None)
464        # The config system yields strings, which have to be evalued
465        self.patches = eval(params.get(prefix + '_patches', "[]"))
466        self.build_steps = build_steps
467        self.srcdir = os.path.join(self.installer.srcdir, srcdir)
468
469
470    def fetch_and_patch(self):
471        if not self.repo:
472            return
473        virt_utils.get_git_branch(self.repo, self.branch, self.srcdir,
474                                 self.commit, self.lbranch)
475        os.chdir(self.srcdir)
476        for patch in self.patches:
477            utils.get_file(patch, os.path.join(self.srcdir,
478                                               os.path.basename(patch)))
479            utils.system('patch -p1 < %s' % os.path.basename(patch))
480
481
482    def build(self):
483        os.chdir(self.srcdir)
484        for step in self.build_steps:
485            logging.info(step)
486            utils.run(step)
487
488
489class GitInstaller(SourceDirInstaller):
490    def _pull_code(self):
491        """
492        Retrieves code from git repositories.
493        """
494        params = self.params
495        make_jobs = utils.count_cpus()
496        cfg = 'PKG_CONFIG_PATH="%s/lib/pkgconfig:%s/share/pkgconfig" ./configure' % (
497            self.prefix, self.prefix)
498
499        self.spice_protocol = GitRepo(installer=self, prefix='spice_protocol',
500            srcdir='spice-protocol',
501            build_steps= ['./autogen.sh',
502                          './configure --prefix=%s' % self.prefix,
503                          'make clean',
504                          'make -j %s' % (make_jobs),
505                          'make install'])
506
507        self.spice = GitRepo(installer=self, prefix='spice', srcdir='spice',
508            build_steps= ['PKG_CONFIG_PATH="%s/lib/pkgconfig:%s/share/pkgconfig" CXXFLAGS=-Wl,--add-needed ./autogen.sh --prefix=%s' % (self.prefix, self.prefix, self.prefix),
509                          'make clean',
510                          'make -j %s' % (make_jobs),
511                          'make install'])
512
513        self.userspace = GitRepo(installer=self, prefix='user',
514            repo_param='user_git_repo', srcdir='kvm_userspace')
515
516        p = os.path.join(self.userspace.srcdir, 'configure')
517        self.configure_options = virt_installer.check_configure_options(p)
518
519        cfg = cfg + ' --prefix=%s' % self.prefix
520        if "--disable-strip" in self.configure_options:
521            cfg += ' --disable-strip'
522        if self.extra_configure_options:
523            cfg += ' %s' % self.extra_configure_options
524
525        self.userspace.build_steps=[cfg, 'make clean', 'make -j %s' % make_jobs]
526
527        if not self.userspace.repo:
528            message = "KVM user git repository path not specified"
529            logging.error(message)
530            raise error.TestError(message)
531
532        for repo in [self.userspace, self.spice_protocol, self.spice]:
533            if not repo.repo:
534                continue
535            repo.fetch_and_patch()
536
537    def _build(self):
538        if self.spice_protocol.repo:
539            logging.info('Building Spice-protocol')
540            self.spice_protocol.build()
541
542        if self.spice.repo:
543            logging.info('Building Spice')
544            self.spice.build()
545
546        logging.info('Building KVM userspace code')
547        self.userspace.build()
548
549
550    def _install(self):
551        os.chdir(self.userspace.srcdir)
552        utils.system('make install')
553
554        if self.path_to_roms:
555            install_roms(self.path_to_roms, self.prefix)
556        self.install_unittests()
557        create_symlinks(test_bindir=self.test_bindir, prefix=self.prefix,
558                        bin_list=None,
559                        unittest=self.unittest_prefix)
560
561
562    def install(self):
563        self._pull_code()
564        self._build()
565        self._install()
566        self.reload_modules_if_needed()
567        if self.save_results:
568            virt_utils.archive_as_tarball(self.srcdir, self.results_dir)
569
570
571class PreInstalledKvm(BaseInstaller):
572    def install(self):
573        logging.info("Expecting KVM to be already installed. Doing nothing")
574
575
576class FailedInstaller:
577    """
578    Class used to be returned instead of the installer if a installation fails
579
580    Useful to make sure no installer object is used if KVM installation fails.
581    """
582    def __init__(self, msg="KVM install failed"):
583        self._msg = msg
584
585
586    def load_modules(self):
587        """Will refuse to load the KVM modules as install failed"""
588        raise FailedKvmInstall("KVM modules not available. reason: %s" % (self._msg))
589
590
591installer_classes = {
592    'localsrc': SourceDirInstaller,
593    'localtar': SourceDirInstaller,
594    'git': GitInstaller,
595    'yum': YumInstaller,
596    'koji': KojiInstaller,
597    'preinstalled': PreInstalledKvm,
598}
599
600
601def _installer_class(install_mode):
602    c = installer_classes.get(install_mode)
603    if c is None:
604        raise error.TestError('Invalid or unsupported'
605                              ' install mode: %s' % install_mode)
606    return c
607
608
609def make_installer(params):
610    # priority:
611    # - 'install_mode' param
612    # - 'mode' param
613    mode = params.get("install_mode", params.get("mode"))
614    klass = _installer_class(mode)
615    return klass(mode)
616