1#!/usr/bin/env python 2 3""" This runs netstat on a local or remote server. It calculates some simple 4statistical information on the number of external inet connections. It groups 5by IP address. This can be used to detect if one IP address is taking up an 6excessive number of connections. It can also send an email alert if a given IP 7address exceeds a threshold between runs of the script. This script can be used 8as a drop-in Munin plugin or it can be used stand-alone from cron. I used this 9on a busy web server that would sometimes get hit with denial of service 10attacks. This made it easy to see if a script was opening many multiple 11connections. A typical browser would open fewer than 10 connections at once. A 12script might open over 100 simultaneous connections. 13 14./topip.py [-s server_hostname] [-u username] [-p password] {-a from_addr,to_addr} {-n N} {-v} {--ipv6} 15 16 -s : hostname of the remote server to login to. 17 -u : username to user for login. 18 -p : password to user for login. 19 -n : print stddev for the the number of the top 'N' ipaddresses. 20 -v : verbose - print stats and list of top ipaddresses. 21 -a : send alert if stddev goes over 20. 22 -l : to log message to /var/log/topip.log 23 --ipv6 : this parses netstat output that includes ipv6 format. 24 Note that this actually only works with ipv4 addresses, but for versions of 25 netstat that print in ipv6 format. 26 --stdev=N : Where N is an integer. This sets the trigger point for alerts and logs. 27 Default is to trigger if max value is above 5 standard deviations. 28 29Example: 30 31 This will print stats for the top IP addresses connected to the given host: 32 33 ./topip.py -s www.example.com -u mylogin -p mypassword -n 10 -v 34 35 This will send an alert email if the maxip goes over the stddev trigger value and 36 the the current top ip is the same as the last top ip (/tmp/topip.last): 37 38 ./topip.py -s www.example.com -u mylogin -p mypassword -n 10 -v -a alert@example.com,user@example.com 39 40 This will print the connection stats for the localhost in Munin format: 41 42 ./topip.py 43 44Noah Spurrier 45 46$Id: topip.py 489 2007-11-28 23:40:34Z noah $ 47""" 48 49import pexpect, pxssh # See http://pexpect.sourceforge.net/ 50import os, sys, time, re, getopt, pickle, getpass, smtplib 51import traceback 52from pprint import pprint 53 54TOPIP_LOG_FILE = '/var/log/topip.log' 55TOPIP_LAST_RUN_STATS = '/var/run/topip.last' 56 57def exit_with_usage(): 58 59 print globals()['__doc__'] 60 os._exit(1) 61 62def stats(r): 63 64 """This returns a dict of the median, average, standard deviation, min and max of the given sequence. 65 66 >>> from topip import stats 67 >>> print stats([5,6,8,9]) 68 {'med': 8, 'max': 9, 'avg': 7.0, 'stddev': 1.5811388300841898, 'min': 5} 69 >>> print stats([1000,1006,1008,1014]) 70 {'med': 1008, 'max': 1014, 'avg': 1007.0, 'stddev': 5.0, 'min': 1000} 71 >>> print stats([1,3,4,5,18,16,4,3,3,5,13]) 72 {'med': 4, 'max': 18, 'avg': 6.8181818181818183, 'stddev': 5.6216817577237475, 'min': 1} 73 >>> print stats([1,3,4,5,18,16,4,3,3,5,13,14,5,6,7,8,7,6,6,7,5,6,4,14,7]) 74 {'med': 6, 'max': 18, 'avg': 7.0800000000000001, 'stddev': 4.3259218670706474, 'min': 1} 75 """ 76 77 total = sum(r) 78 avg = float(total)/float(len(r)) 79 sdsq = sum([(i-avg)**2 for i in r]) 80 s = list(r) 81 s.sort() 82 return dict(zip(['med', 'avg', 'stddev', 'min', 'max'] , (s[len(s)//2], avg, (sdsq/len(r))**.5, min(r), max(r)))) 83 84def send_alert (message, subject, addr_from, addr_to, smtp_server='localhost'): 85 86 """This sends an email alert. 87 """ 88 89 message = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n' % (addr_from, addr_to, subject) + message 90 server = smtplib.SMTP(smtp_server) 91 server.sendmail(addr_from, addr_to, message) 92 server.quit() 93 94def main(): 95 96 ###################################################################### 97 ## Parse the options, arguments, etc. 98 ###################################################################### 99 try: 100 optlist, args = getopt.getopt(sys.argv[1:], 'h?valqs:u:p:n:', ['help','h','?','ipv6','stddev=']) 101 except Exception, e: 102 print str(e) 103 exit_with_usage() 104 options = dict(optlist) 105 106 munin_flag = False 107 if len(args) > 0: 108 if args[0] == 'config': 109 print 'graph_title Netstat Connections per IP' 110 print 'graph_vlabel Socket connections per IP' 111 print 'connections_max.label max' 112 print 'connections_max.info Maximum number of connections per IP' 113 print 'connections_avg.label avg' 114 print 'connections_avg.info Average number of connections per IP' 115 print 'connections_stddev.label stddev' 116 print 'connections_stddev.info Standard deviation' 117 return 0 118 elif args[0] != '': 119 print args, len(args) 120 return 0 121 exit_with_usage() 122 if [elem for elem in options if elem in ['-h','--h','-?','--?','--help']]: 123 print 'Help:' 124 exit_with_usage() 125 if '-s' in options: 126 hostname = options['-s'] 127 else: 128 # if host was not specified then assume localhost munin plugin. 129 munin_flag = True 130 hostname = 'localhost' 131 # If localhost then don't ask for username/password. 132 if hostname != 'localhost' and hostname != '127.0.0.1': 133 if '-u' in options: 134 username = options['-u'] 135 else: 136 username = raw_input('username: ') 137 if '-p' in options: 138 password = options['-p'] 139 else: 140 password = getpass.getpass('password: ') 141 else: 142 use_localhost = True 143 144 if '-l' in options: 145 log_flag = True 146 else: 147 log_flag = False 148 if '-n' in options: 149 average_n = int(options['-n']) 150 else: 151 average_n = None 152 if '-v' in options: 153 verbose = True 154 else: 155 verbose = False 156 if '-a' in options: 157 alert_flag = True 158 (alert_addr_from, alert_addr_to) = tuple(options['-a'].split(',')) 159 else: 160 alert_flag = False 161 if '--ipv6' in options: 162 ipv6_flag = True 163 else: 164 ipv6_flag = False 165 if '--stddev' in options: 166 stddev_trigger = float(options['--stddev']) 167 else: 168 stddev_trigger = 5 169 170 if ipv6_flag: 171 netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+::ffff:(\S+):(\S+)\s+.*?\r' 172 else: 173 netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(?:::ffff:)*(\S+):(\S+)\s+.*?\r' 174 #netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+):(\S+)\s+.*?\r' 175 176 # run netstat (either locally or via SSH). 177 if use_localhost: 178 p = pexpect.spawn('netstat -n -t') 179 PROMPT = pexpect.TIMEOUT 180 else: 181 p = pxssh.pxssh() 182 p.login(hostname, username, password) 183 p.sendline('netstat -n -t') 184 PROMPT = p.PROMPT 185 186 # loop through each matching netstat_pattern and put the ip address in the list. 187 ip_list = {} 188 try: 189 while 1: 190 i = p.expect([PROMPT, netstat_pattern]) 191 if i == 0: 192 break 193 k = p.match.groups()[4] 194 if k in ip_list: 195 ip_list[k] = ip_list[k] + 1 196 else: 197 ip_list[k] = 1 198 except: 199 pass 200 201 # remove a few common, uninteresting addresses from the dictionary. 202 ip_list = dict([ (key,value) for key,value in ip_list.items() if '192.168.' not in key]) 203 ip_list = dict([ (key,value) for key,value in ip_list.items() if '127.0.0.1' not in key]) 204 205 # sort dict by value (count) 206 #ip_list = sorted(ip_list.iteritems(),lambda x,y:cmp(x[1], y[1]),reverse=True) 207 ip_list = ip_list.items() 208 if len(ip_list) < 1: 209 if verbose: print 'Warning: no networks connections worth looking at.' 210 return 0 211 ip_list.sort(lambda x,y:cmp(y[1],x[1])) 212 213 # generate some stats for the ip addresses found. 214 if average_n <= 1: 215 average_n = None 216 s = stats(zip(*ip_list[0:average_n])[1]) # The * unary operator treats the list elements as arguments 217 s['maxip'] = ip_list[0] 218 219 # print munin-style or verbose results for the stats. 220 if munin_flag: 221 print 'connections_max.value', s['max'] 222 print 'connections_avg.value', s['avg'] 223 print 'connections_stddev.value', s['stddev'] 224 return 0 225 if verbose: 226 pprint (s) 227 print 228 pprint (ip_list[0:average_n]) 229 230 # load the stats from the last run. 231 try: 232 last_stats = pickle.load(file(TOPIP_LAST_RUN_STATS)) 233 except: 234 last_stats = {'maxip':None} 235 236 if s['maxip'][1] > (s['stddev'] * stddev_trigger) and s['maxip']==last_stats['maxip']: 237 if verbose: print 'The maxip has been above trigger for two consecutive samples.' 238 if alert_flag: 239 if verbose: print 'SENDING ALERT EMAIL' 240 send_alert(str(s), 'ALERT on %s' % hostname, alert_addr_from, alert_addr_to) 241 if log_flag: 242 if verbose: print 'LOGGING THIS EVENT' 243 fout = file(TOPIP_LOG_FILE,'a') 244 #dts = time.strftime('%Y:%m:%d:%H:%M:%S', time.localtime()) 245 dts = time.asctime() 246 fout.write ('%s - %d connections from %s\n' % (dts,s['maxip'][1],str(s['maxip'][0]))) 247 fout.close() 248 249 # save state to TOPIP_LAST_RUN_STATS 250 try: 251 pickle.dump(s, file(TOPIP_LAST_RUN_STATS,'w')) 252 os.chmod (TOPIP_LAST_RUN_STATS, 0664) 253 except: 254 pass 255 # p.logout() 256 257if __name__ == '__main__': 258 try: 259 main() 260 sys.exit(0) 261 except SystemExit, e: 262 raise e 263 except Exception, e: 264 print str(e) 265 traceback.print_exc() 266 os._exit(1) 267 268