• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2#
3# Copyright 2010 Google Inc. All Rights Reserved.
4
5"""Tool to check the data consistency between master autotest db and replica.
6
7This tool will issue 'show master status;' and 'show slave status;' commands to
8two replicated databases to compare its log position.
9
10It will also take a delta command line argument to allow certain time delay
11between master and slave. If the delta of two log positions falls into the
12defined range, it will be treated as synced.
13
14It will send out an email notification upon any problem if specified an --to
15argument.
16"""
17
18import getpass
19import MySQLdb
20import optparse
21import os
22import socket
23import sys
24
25import common
26from autotest_lib.client.common_lib import global_config
27
28
29c = global_config.global_config
30_section = 'AUTOTEST_WEB'
31DATABASE_HOST = c.get_config_value(_section, "host")
32REPLICA_DATABASE_HOST = c.get_config_value(_section, "readonly_host")
33DATABASE_NAME = c.get_config_value(_section, "database")
34DATABASE_USER = c.get_config_value(_section, "user")
35DATABASE_PASSWORD = c.get_config_value(_section, "password")
36SYSTEM_USER = 'chromeos-test'
37
38
39def ParseOptions():
40  parser = optparse.OptionParser()
41  parser.add_option('-d', '--delta', help='Difference between master and '
42                    'replica db', type='int', dest='delta', default=0)
43  parser.add_option('--to', help='Comma separated Email notification TO '
44                    'recipients.', dest='to', type='string', default='')
45  parser.add_option('--cc', help='Comma separated Email notification CC '
46                    'recipients.', dest='cc', type='string', default='')
47  parser.add_option('-t', '--test-mode', help='skip common group email',
48                    dest='testmode', action='store_true', default=False)
49  options, _ = parser.parse_args()
50  return options
51
52
53def FetchMasterResult():
54  master_conn = MySQLdb.connect(host=DATABASE_HOST,
55                                user=DATABASE_USER,
56                                passwd=DATABASE_PASSWORD,
57                                db=DATABASE_NAME )
58  cursor = master_conn.cursor(MySQLdb.cursors.DictCursor)
59  cursor.execute ("show master status;")
60  master_result = cursor.fetchone()
61  master_conn.close()
62  return master_result
63
64
65def FetchSlaveResult():
66  replica_conn = MySQLdb.connect(host=REPLICA_DATABASE_HOST,
67                                 user=DATABASE_USER,
68                                 passwd=DATABASE_PASSWORD,
69                                 db=DATABASE_NAME )
70  cursor = replica_conn.cursor(MySQLdb.cursors.DictCursor)
71  cursor.execute ("show slave status;")
72  slave_result = cursor.fetchone()
73  replica_conn.close()
74  return slave_result
75
76
77def RunChecks(options, master_result, slave_result):
78  master_pos = master_result['Position']
79  slave_pos = slave_result['Read_Master_Log_Pos']
80  if (master_pos - slave_pos) > options.delta:
81    return 'DELTA EXCEEDED: master=%s, slave=%s' % (master_pos, slave_pos)
82  if slave_result['Last_SQL_Error'] != '':
83    return 'SLAVE Last_SQL_Error'
84  if slave_result['Slave_IO_State'] != 'Waiting for master to send event':
85    return 'SLAVE Slave_IO_State'
86  if slave_result['Last_IO_Error'] != '':
87    return 'SLAVE Last_IO_Error'
88  if slave_result['Slave_SQL_Running'] != 'Yes':
89    return 'SLAVE Slave_SQL_Running'
90  if slave_result['Slave_IO_Running'] != 'Yes':
91    return 'SLAVE Slave_IO_Running'
92  return None
93
94
95def ShowStatus(options, master_result, slave_result, msg):
96  summary = 'Master (%s) and slave (%s) databases are out of sync.' % (
97      DATABASE_HOST, REPLICA_DATABASE_HOST) + msg
98  if not options.to:
99    print summary
100    print 'Master status:'
101    print str(master_result)
102    print 'Slave status:'
103    print str(slave_result)
104  else:
105    email_to = ['%s@google.com' % to.strip() for to in options.to.split(',')]
106    email_cc = []
107    if options.cc:
108      email_cc.extend(
109          '%s@google.com' % cc.strip() for cc in options.cc.split(','))
110    if getpass.getuser() == SYSTEM_USER and not options.testmode:
111      email_cc.append('chromeos-build-alerts+db-replica-checker@google.com')
112    body = ('%s\n\n'
113            'Master (%s) status:\n%s\n\n'
114            'Slave (%s) status:\n%s' % (summary, DATABASE_HOST, master_result,
115                                        REPLICA_DATABASE_HOST, slave_result))
116    p = os.popen('/usr/sbin/sendmail -t', 'w')
117    p.write('To: %s\n' % ','.join(email_to))
118    if email_cc:
119      p.write('Cc: %s\n' % ','.join(email_cc))
120
121    p.write('Subject: Inconsistency detected in cautotest DB replica on %s.\n'
122            % socket.gethostname())
123    p.write('Content-Type: text/plain')
124    p.write('\n')  # blank line separating headers from body
125    p.write(body)
126    p.write('\n')
127    return_code = p.close()
128    if return_code is not None:
129      print 'Sendmail exit status %s' % return_code
130
131
132def main():
133  options = ParseOptions()
134  master_result = FetchMasterResult()
135  slave_result = FetchSlaveResult()
136  problem_msg = RunChecks(options, master_result, slave_result)
137  if problem_msg:
138    ShowStatus(options, master_result, slave_result, problem_msg)
139    sys.exit(-1)
140
141if __name__ == '__main__':
142  main()
143