1 2# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Scheduler helper libraries. 7""" 8import logging 9import os 10 11import common 12 13from autotest_lib.client.common_lib import global_config 14from autotest_lib.client.common_lib import logging_config 15from autotest_lib.client.common_lib import logging_manager 16from autotest_lib.client.common_lib import utils 17from autotest_lib.database import database_connection 18from autotest_lib.frontend import setup_django_environment 19from autotest_lib.frontend.afe import readonly_connection 20from autotest_lib.server import utils as server_utils 21 22 23DB_CONFIG_SECTION = 'AUTOTEST_WEB' 24 25# Translations necessary for scheduler queries to work with SQLite. 26# Though this is only used for testing it is included in this module to avoid 27# circular imports. 28_re_translator = database_connection.TranslatingDatabase.make_regexp_translator 29_DB_TRANSLATORS = ( 30 _re_translator(r'NOW\(\)', 'time("now")'), 31 _re_translator(r'LAST_INSERT_ID\(\)', 'LAST_INSERT_ROWID()'), 32 # older SQLite doesn't support group_concat, so just don't bother until 33 # it arises in an important query 34 _re_translator(r'GROUP_CONCAT\((.*?)\)', r'\1'), 35 _re_translator(r'TRUNCATE TABLE', 'DELETE FROM'), 36 _re_translator(r'ISNULL\(([a-z,_]+)\)', 37 r'ifnull(nullif(\1, NULL), \1) DESC'), 38) 39 40 41class SchedulerError(Exception): 42 """General parent class for exceptions raised by scheduler code.""" 43 44 45class MalformedRecordError(SchedulerError): 46 """Exception raised when an individual job or record is malformed. 47 48 Code that handles individual records (e.g. afe jobs, hqe entries, special 49 tasks) should treat such an exception as a signal to skip or permanently 50 discard this record.""" 51 52 53class NoHostIdError(MalformedRecordError): 54 """Raised by the scheduler when a non-hostless job's host is None.""" 55 56 57class ConnectionManager(object): 58 """Manager for the django database connections. 59 60 The connection is used through scheduler_models and monitor_db. 61 """ 62 __metaclass__ = server_utils.Singleton 63 64 def __init__(self, readonly=True, autocommit=True): 65 """Set global django database options for correct connection handling. 66 67 @param readonly: Globally disable readonly connections. 68 @param autocommit: Initialize django autocommit options. 69 """ 70 self.db_connection = None 71 # bypass the readonly connection 72 readonly_connection.set_globally_disabled(readonly) 73 if autocommit: 74 # ensure Django connection is in autocommit 75 setup_django_environment.enable_autocommit() 76 77 78 @classmethod 79 def open_connection(cls): 80 """Open a new database connection. 81 82 @return: An instance of the newly opened connection. 83 """ 84 db = database_connection.DatabaseConnection(DB_CONFIG_SECTION) 85 db.connect(db_type='django') 86 return db 87 88 89 def get_connection(self): 90 """Get a connection. 91 92 @return: A database connection. 93 """ 94 if self.db_connection is None: 95 self.db_connection = self.open_connection() 96 return self.db_connection 97 98 99 def disconnect(self): 100 """Close the database connection.""" 101 try: 102 self.db_connection.disconnect() 103 except Exception as e: 104 logging.debug('Could not close the db connection. %s', e) 105 106 107 def __del__(self): 108 self.disconnect() 109 110 111class SchedulerLoggingConfig(logging_config.LoggingConfig): 112 """Configure timestamped logging for a scheduler.""" 113 GLOBAL_LEVEL = logging.INFO 114 115 @classmethod 116 def get_log_name(cls, timestamped_logfile_prefix): 117 """Get the name of a logfile. 118 119 @param timestamped_logfile_prefix: The prefix to apply to the 120 a timestamped log. Eg: 'scheduler' will create a logfile named 121 scheduler.log.2014-05-12-17.24.02. 122 123 @return: The timestamped log name. 124 """ 125 return cls.get_timestamped_log_name(timestamped_logfile_prefix) 126 127 128 def configure_logging(self, log_dir=None, logfile_name=None, 129 timestamped_logfile_prefix='scheduler'): 130 """Configure logging to a specified logfile. 131 132 @param log_dir: The directory to log into. 133 @param logfile_name: The name of the log file. 134 @timestamped_logfile_prefix: The prefix to apply to the name of 135 the logfile, if a log file name isn't specified. 136 """ 137 super(SchedulerLoggingConfig, self).configure_logging(use_console=True) 138 139 if log_dir is None: 140 log_dir = self.get_server_log_dir() 141 if not logfile_name: 142 logfile_name = self.get_log_name(timestamped_logfile_prefix) 143 144 self.add_file_handler(logfile_name, logging.DEBUG, log_dir=log_dir) 145 symlink_path = os.path.join( 146 log_dir, '%s.latest' % timestamped_logfile_prefix) 147 try: 148 os.unlink(symlink_path) 149 except OSError: 150 pass 151 os.symlink(os.path.join(log_dir, logfile_name), symlink_path) 152 153 154def setup_logging(log_dir, log_name, timestamped_logfile_prefix='scheduler'): 155 """Setup logging to a given log directory and log file. 156 157 @param log_dir: The directory to log into. 158 @param log_name: Name of the log file. 159 @param timestamped_logfile_prefix: The prefix to apply to the logfile. 160 """ 161 logging_manager.configure_logging( 162 SchedulerLoggingConfig(), log_dir=log_dir, logfile_name=log_name, 163 timestamped_logfile_prefix=timestamped_logfile_prefix) 164 165 166def check_production_settings(scheduler_options): 167 """Check the scheduler option's production settings. 168 169 @param scheduler_options: Settings for scheduler. 170 171 @raises SchedulerError: If a loclhost scheduler is started with 172 production settings. 173 """ 174 db_server = global_config.global_config.get_config_value('AUTOTEST_WEB', 175 'host') 176 if (not scheduler_options.production and 177 not utils.is_localhost(db_server)): 178 raise SchedulerError('Scheduler is not running in production mode, you ' 179 'should not set database to hosts other than ' 180 'localhost. It\'s currently set to %s.\nAdd option' 181 ' --production if you want to skip this check.' % 182 db_server) 183