• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import contextlib
6import logging
7import time
8from multiprocessing import pool
9
10import base_event, board_enumerator, build_event, deduping_scheduler
11import error
12import task, timed_event
13
14import common
15from autotest_lib.client.common_lib import utils
16from autotest_lib.server import utils
17
18try:
19    from chromite.lib import metrics
20except ImportError:
21    metrics = utils.metrics_mock
22
23
24POOL_SIZE = 32
25
26BOARD_WHITELIST_SECTION = 'board_lists'
27PRE_SECTIONS = [BOARD_WHITELIST_SECTION]
28
29class Driver(object):
30    """Implements the main loop of the suite_scheduler.
31
32    @var EVENT_CLASSES: list of the event classes Driver supports.
33    @var _LOOP_INTERVAL_SECONDS: seconds to wait between loop iterations.
34
35    @var _scheduler: a DedupingScheduler, used to schedule jobs with the AFE.
36    @var _enumerator: a BoardEnumerator, used to list plaforms known to
37                      the AFE
38    @var _events: dict of BaseEvents to be handled each time through main loop.
39    """
40
41    EVENT_CLASSES = [timed_event.Nightly, timed_event.Weekly,
42                     build_event.NewBuild]
43    _LOOP_INTERVAL_SECONDS = 5 * 60
44
45    # Cache for known ChromeOS boards. The cache helps to avoid unnecessary
46    # repeated calls to Launch Control API.
47    _cros_boards = set()
48
49    def __init__(self, scheduler, enumerator, is_sanity=False):
50        """Constructor
51
52        @param scheduler: an instance of deduping_scheduler.DedupingScheduler.
53        @param enumerator: an instance of board_enumerator.BoardEnumerator.
54        @param is_sanity: Set to True if the driver is created for sanity check.
55                          Default is set to False.
56        """
57        self._scheduler = scheduler
58        self._enumerator = enumerator
59        task.TotMilestoneManager.is_sanity = is_sanity
60
61
62    def RereadAndReprocessConfig(self, config, mv):
63        """Re-read config, re-populate self._events and recreate task lists.
64
65        @param config: an instance of ForgivingConfigParser.
66        @param mv: an instance of ManifestVersions.
67        """
68        config.reread()
69        new_events = self._CreateEventsWithTasks(config, mv)
70        for keyword, event in self._events.iteritems():
71            event.Merge(new_events[keyword])
72
73
74    def SetUpEventsAndTasks(self, config, mv):
75        """Populate self._events and create task lists from config.
76
77        @param config: an instance of ForgivingConfigParser.
78        @param mv: an instance of ManifestVersions.
79        """
80        self._events = self._CreateEventsWithTasks(config, mv)
81
82
83    def _ReadBoardWhitelist(self, config):
84        """Read board whitelist from config and save as dict.
85
86        @param config: an instance of ForgivingConfigParser.
87        """
88        board_lists = {}
89        if BOARD_WHITELIST_SECTION not in config.sections():
90            return board_lists
91
92        for option in config.options(BOARD_WHITELIST_SECTION):
93            if option in board_lists:
94                raise error.MalformedConfigEntry(
95                        'Board list name must be unique.')
96            else:
97                board_lists[option] = config.getstring(
98                        BOARD_WHITELIST_SECTION, option)
99
100        return board_lists
101
102
103    def _CreateEventsWithTasks(self, config, mv):
104        """Create task lists from config, and assign to newly-minted events.
105
106        Calling multiple times should start afresh each time.
107
108        @param config: an instance of ForgivingConfigParser.
109        @param mv: an instance of ManifestVersions.
110        """
111        events = {}
112        for klass in self.EVENT_CLASSES:
113            events[klass.KEYWORD] = klass.CreateFromConfig(config, mv)
114
115        tasks = self.TasksFromConfig(config)
116        for keyword, task_list in tasks.iteritems():
117            if keyword in events:
118                events[keyword].tasks = task_list
119            else:
120                logging.warning('%s, is an unknown keyword.', keyword)
121        return events
122
123
124    def TasksFromConfig(self, config):
125        """Generate a dict of {event_keyword: [tasks]} mappings from |config|.
126
127        For each section in |config| that encodes a Task, instantiate a Task
128        object.  Determine the event that Task is supposed to run_on and
129        append the object to a list associated with the appropriate event
130        keyword.  Return a dictionary of these keyword: list of task mappings.
131
132        @param config: a ForgivingConfigParser containing tasks to be parsed.
133        @return dict of {event_keyword: [tasks]} mappings.
134        @raise MalformedConfigEntry on a task parsing error.
135        """
136        board_lists = self._ReadBoardWhitelist(config)
137        tasks = {}
138        for section in config.sections():
139            if (not base_event.HonoredSection(section) and
140                section not in PRE_SECTIONS):
141                try:
142                    keyword, new_task = task.Task.CreateFromConfigSection(
143                            config, section, board_lists=board_lists)
144                except error.MalformedConfigEntry as e:
145                    logging.warning('%s is malformed: %s', section, str(e))
146                    continue
147                tasks.setdefault(keyword, []).append(new_task)
148        return tasks
149
150
151    def RunForever(self, config, mv):
152        """Main loop of the scheduler.  Runs til the process is killed.
153
154        @param config: an instance of ForgivingConfigParser.
155        @param mv: an instance of manifest_versions.ManifestVersions.
156        """
157        for event in self._events.itervalues():
158            event.Prepare()
159        while True:
160            try:
161                self.HandleEventsOnce(mv)
162            except board_enumerator.EnumeratorException as e:
163                logging.warning('Failed to enumerate boards: %r', e)
164            mv.Update()
165            task.TotMilestoneManager().refresh()
166            time.sleep(self._LOOP_INTERVAL_SECONDS)
167            self.RereadAndReprocessConfig(config, mv)
168            metrics.Counter('chromeos/autotest/suite_scheduler/'
169                            'handle_events_tick').increment()
170
171
172    @staticmethod
173    def HandleBoard(inputs):
174        """Handle event based on given inputs.
175
176        @param inputs: A dictionary of the arguments needed to handle an event.
177            Keys include:
178            scheduler: a DedupingScheduler, used to schedule jobs with the AFE.
179            event: An event object to be handled.
180            board: Name of the board.
181        """
182        scheduler = inputs['scheduler']
183        event = inputs['event']
184        board = inputs['board']
185
186        # Try to get builds from LaunchControl first. If failed, the board could
187        # be ChromeOS. Use the cache Driver._cros_boards to avoid unnecessary
188        # repeated call to LaunchControl API.
189        launch_control_builds = None
190        if board not in Driver._cros_boards:
191            launch_control_builds = event.GetLaunchControlBuildsForBoard(board)
192        if launch_control_builds:
193            event.Handle(scheduler, branch_builds=None, board=board,
194                         launch_control_builds=launch_control_builds)
195        else:
196            branch_builds = event.GetBranchBuildsForBoard(board)
197            if branch_builds:
198                Driver._cros_boards.add(board)
199                logging.info('Found ChromeOS build for board %s. This should '
200                             'be a ChromeOS board.', board)
201            event.Handle(scheduler, branch_builds, board)
202        logging.info('Finished handling %s event for board %s', event.keyword,
203                     board)
204
205    @metrics.SecondsTimerDecorator('chromeos/autotest/suite_scheduler/'
206                                   'handle_events_once_duration')
207    def HandleEventsOnce(self, mv):
208        """One turn through the loop.  Separated out for unit testing.
209
210        @param mv: an instance of manifest_versions.ManifestVersions.
211        @raise EnumeratorException if we can't enumerate any supported boards.
212        """
213        boards = self._enumerator.Enumerate()
214        logging.info('%d boards currently in the lab: %r', len(boards), boards)
215        thread_pool = pool.ThreadPool(POOL_SIZE)
216        with contextlib.closing(thread_pool):
217            for e in self._events.itervalues():
218                if not e.ShouldHandle():
219                    continue
220                # Reset the value of delay_minutes, as this is the beginning of
221                # handling an event for all boards.
222                self._scheduler.delay_minutes = 0
223                self._scheduler.delay_minutes_interval = (
224                        deduping_scheduler.DELAY_MINUTES_INTERVAL)
225                logging.info('Handling %s event for %d boards', e.keyword,
226                             len(boards))
227                args = []
228                for board in boards:
229                    args.append({'scheduler': self._scheduler,
230                                 'event': e,
231                                 'board': board})
232                thread_pool.map(self.HandleBoard, args)
233                logging.info('Finished handling %s event for %d boards',
234                             e.keyword, len(boards))
235                e.UpdateCriteria()
236
237
238    def ForceEventsOnceForBuild(self, keywords, build_name,
239                                os_type=task.OS_TYPE_CROS):
240        """Force events with provided keywords to happen, with given build.
241
242        @param keywords: iterable of event keywords to force
243        @param build_name: instead of looking up builds to test, test this one.
244        @param os_type: Type of the OS to test, default to cros.
245        """
246        branch_builds = None
247        launch_control_builds = None
248        if os_type == task.OS_TYPE_CROS:
249            board, type, milestone, manifest = utils.ParseBuildName(build_name)
250            branch_builds = {task.PickBranchName(type, milestone): [build_name]}
251            logging.info('Testing build R%s-%s on %s', milestone, manifest,
252                         board)
253        else:
254            logging.info('Build is not a ChromeOS build, try to parse as a '
255                         'Launch Control build.')
256            _,target,_ = utils.parse_launch_control_build(build_name)
257            board = utils.parse_launch_control_target(target)[0]
258            # Translate board name in build target to the actual board name.
259            board = utils.ANDROID_TARGET_TO_BOARD_MAP.get(board, board)
260            launch_control_builds = [build_name]
261            logging.info('Testing Launch Control build %s on %s', build_name,
262                         board)
263
264        for e in self._events.itervalues():
265            if e.keyword in keywords:
266                e.Handle(self._scheduler, branch_builds, board, force=True,
267                         launch_control_builds=launch_control_builds)
268