• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2#
3# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""CrOS suite scheduler.  Will schedule suites based on configured triggers.
8
9The Scheduler understands two main primitives: Events and Tasks.  Each stanza
10in the config file specifies a Task that triggers on a given Event.
11
12Events:
13  The scheduler supports two kinds of Events: timed events, and
14  build system events -- like a particular build artifact becoming available.
15  Every Event has a set of Tasks that get run whenever the event happens.
16
17Tasks:
18  Basically, event handlers.  A Task is specified in the config file like so:
19  [NightlyPower]
20  suite: power
21  run_on: nightly
22  pool: remote_power
23  branch_specs: >=R20,factory
24
25  This specifies a Task that gets run whenever the 'nightly' event occurs.
26  The Task schedules a suite of tests called 'power' on the pool of machines
27  called 'remote_power', for both the factory branch and all active release
28  branches from R20 on.
29
30
31On startup, the scheduler reads in a config file that provides a few
32parameters for certain supported Events (the time/day of the 'weekly'
33and 'nightly' triggers, for example), and configures all the Tasks
34that will be in play.
35"""
36
37import getpass, logging, logging.handlers, optparse, os, re, signal, sys
38import traceback
39import common
40import board_enumerator, deduping_scheduler, driver, forgiving_config_parser
41import manifest_versions, sanity
42from autotest_lib.client.common_lib import global_config
43from autotest_lib.client.common_lib import logging_config, logging_manager
44from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
45try:
46    from autotest_lib.frontend import setup_django_environment
47    # server_manager_utils depend on django which
48    # may not be available when people run checks with --sanity
49    from autotest_lib.site_utils import server_manager_utils
50except ImportError:
51    server_manager_utils = None
52    logging.debug('Could not load server_manager_utils module, expected '
53                  'if you are running sanity check or pre-submit hook')
54
55
56CONFIG_SECTION = 'SCHEDULER'
57
58CONFIG_SECTION_SERVER = 'SERVER'
59
60
61def signal_handler(signal, frame):
62    """Singnal hanlder to exit gracefully.
63
64    @param signal: signum
65    @param frame: stack frame object
66    """
67    logging.info('Signal %d received.  Exiting gracefully...', signal)
68    sys.exit(0)
69
70
71class SeverityFilter(logging.Filter):
72    """Filters out messages of anything other than self._level"""
73    def __init__(self, level):
74        self._level = level
75
76
77    def filter(self, record):
78        """Causes only messages of |self._level| severity to be logged."""
79        return record.levelno == self._level
80
81
82class SchedulerLoggingConfig(logging_config.LoggingConfig):
83    """Configure loggings for scheduler, e.g., email setup."""
84    def __init__(self):
85        super(SchedulerLoggingConfig, self).__init__()
86        self._from_address = global_config.global_config.get_config_value(
87                CONFIG_SECTION, "notify_email_from", default=getpass.getuser())
88
89        self._notify_address = global_config.global_config.get_config_value(
90                CONFIG_SECTION, "notify_email",
91                default='chromeos-lab-admins@google.com')
92
93        self._smtp_server = global_config.global_config.get_config_value(
94                CONFIG_SECTION_SERVER, "smtp_server", default='localhost')
95
96        self._smtp_port = global_config.global_config.get_config_value(
97                CONFIG_SECTION_SERVER, "smtp_port", default=None)
98
99        self._smtp_user = global_config.global_config.get_config_value(
100                CONFIG_SECTION_SERVER, "smtp_user", default='')
101
102        self._smtp_password = global_config.global_config.get_config_value(
103                CONFIG_SECTION_SERVER, "smtp_password", default='')
104
105
106    @classmethod
107    def get_log_name(cls):
108        """Get timestamped log name of suite_scheduler, e.g.,
109        suite_scheduler.log.2013-2-1-02-05-06.
110
111        @param cls: class
112        """
113        return cls.get_timestamped_log_name('suite_scheduler')
114
115
116    def add_smtp_handler(self, subject, level=logging.ERROR):
117        """Add smtp handler to logging handler to trigger email when logging
118        occurs.
119
120        @param subject: email subject.
121        @param level: level of logging to trigger smtp handler.
122        """
123        if not self._smtp_user or not self._smtp_password:
124            creds = None
125        else:
126            creds = (self._smtp_user, self._smtp_password)
127        server = self._smtp_server
128        if self._smtp_port:
129            server = (server, self._smtp_port)
130
131        handler = logging.handlers.SMTPHandler(server,
132                                               self._from_address,
133                                               [self._notify_address],
134                                               subject,
135                                               creds)
136        handler.setLevel(level)
137        # We want to send mail for the given level, and only the given level.
138        # One can add more handlers to send messages for other levels.
139        handler.addFilter(SeverityFilter(level))
140        handler.setFormatter(
141            logging.Formatter('%(asctime)s %(levelname)-5s %(message)s'))
142        self.logger.addHandler(handler)
143        return handler
144
145
146    def configure_logging(self, log_dir=None):
147        super(SchedulerLoggingConfig, self).configure_logging(use_console=True)
148
149        if not log_dir:
150            return
151        base = self.get_log_name()
152
153        self.add_file_handler(base + '.DEBUG', logging.DEBUG, log_dir=log_dir)
154        self.add_file_handler(base + '.INFO', logging.INFO, log_dir=log_dir)
155        self.add_smtp_handler('Suite scheduler ERROR', logging.ERROR)
156        self.add_smtp_handler('Suite scheduler WARNING', logging.WARN)
157
158
159def parse_options():
160    """Parse commandline options."""
161    usage = "usage: %prog [options]"
162    parser = optparse.OptionParser(usage=usage)
163    parser.add_option('-f', '--config_file', dest='config_file',
164                      metavar='/path/to/config', default='suite_scheduler.ini',
165                      help='Scheduler config. Defaults to suite_scheduler.ini')
166    parser.add_option('-e', '--events', dest='events',
167                      metavar='list,of,events',
168                      help='Handle listed events once each, then exit.  '\
169                        'Must also specify a build to test.')
170    parser.add_option('-i', '--build', dest='build',
171                      help='If handling a list of events, the build to test.'\
172                        ' Ignored otherwise.')
173    parser.add_option('-d', '--log_dir', dest='log_dir',
174                      help='Log to a file in the specified directory.')
175    parser.add_option('-l', '--list_events', dest='list',
176                      action='store_true', default=False,
177                      help='List supported events and exit.')
178    parser.add_option('-r', '--repo_dir', dest='tmp_repo_dir', default=None,
179                      help=('Path to a tmpdir containing manifest versions. '
180                            'This option is only used for testing.'))
181    parser.add_option('-t', '--sanity', dest='sanity', action='store_true',
182                      default=False,
183                      help='Check the config file for any issues.')
184    parser.add_option('-b', '--file_bug', dest='file_bug', action='store_true',
185                      default=False,
186                      help='File bugs for known suite scheduling exceptions.')
187
188
189    options, args = parser.parse_args()
190    return parser, options, args
191
192
193def main():
194    """Entry point for suite_scheduler.py"""
195    signal.signal(signal.SIGINT, signal_handler)
196    signal.signal(signal.SIGHUP, signal_handler)
197    signal.signal(signal.SIGTERM, signal_handler)
198
199    parser, options, args = parse_options()
200    if args or options.events and not options.build:
201        parser.print_help()
202        return 1
203
204    if options.config_file and not os.path.exists(options.config_file):
205        logging.error('Specified config file %s does not exist.',
206                      options.config_file)
207        return 1
208
209    config = forgiving_config_parser.ForgivingConfigParser()
210    config.read(options.config_file)
211
212    if options.list:
213        print 'Supported events:'
214        for event_class in driver.Driver.EVENT_CLASSES:
215            print '  ', event_class.KEYWORD
216        return 0
217
218    # If we're just sanity checking, we can stop after we've parsed the
219    # config file.
220    if options.sanity:
221        # config_file_getter generates a high amount of noise at DEBUG level
222        logging.getLogger().setLevel(logging.WARNING)
223        d = driver.Driver(None, None, True)
224        d.SetUpEventsAndTasks(config, None)
225        tasks_per_event = d.TasksFromConfig(config)
226        # flatten [[a]] -> [a]
227        tasks = [x for y in tasks_per_event.values() for x in y]
228        control_files_exist = sanity.CheckControlFileExistance(tasks)
229        return control_files_exist
230
231    logging_manager.configure_logging(SchedulerLoggingConfig(),
232                                      log_dir=options.log_dir)
233    if not options.log_dir:
234        logging.info('Not logging to a file, as --log_dir was not passed.')
235
236    # If server database is enabled, check if the server has role
237    # `suite_scheduler`. If the server does not have suite_scheduler role,
238    # exception will be raised and suite scheduler will not continue to run.
239    if not server_manager_utils:
240        raise ImportError(
241            'Could not import autotest_lib.site_utils.server_manager_utils')
242    if server_manager_utils.use_server_db():
243        server_manager_utils.confirm_server_has_role(hostname='localhost',
244                                                     role='suite_scheduler')
245
246    afe_server = global_config.global_config.get_config_value(
247                CONFIG_SECTION_SERVER, "suite_scheduler_afe", default=None)
248
249    afe = frontend_wrappers.RetryingAFE(
250            server=afe_server, timeout_min=10, delay_sec=5, debug=False)
251    logging.info('Connecting to: %s' , afe.server)
252    enumerator = board_enumerator.BoardEnumerator(afe)
253    scheduler = deduping_scheduler.DedupingScheduler(afe, options.file_bug)
254    mv = manifest_versions.ManifestVersions(options.tmp_repo_dir)
255    d = driver.Driver(scheduler, enumerator)
256    d.SetUpEventsAndTasks(config, mv)
257
258    try:
259        if options.events:
260            # Act as though listed events have just happened.
261            keywords = re.split('\s*,\s*', options.events)
262            if not options.tmp_repo_dir:
263                logging.warn('To run a list of events, you may need to use '
264                             '--repo_dir to specify a folder that already has '
265                             'manifest repo set up. This is needed for suites '
266                             'requiring firmware update.')
267            logging.info('Forcing events: %r', keywords)
268            d.ForceEventsOnceForBuild(keywords, options.build)
269        else:
270            if not options.tmp_repo_dir:
271                mv.Initialize()
272            d.RunForever(config, mv)
273    except Exception as e:
274        logging.error('Fatal exception in suite_scheduler: %r\n%s', e,
275                      traceback.format_exc())
276        return 1
277
278if __name__ == "__main__":
279    sys.exit(main())
280