1#!/usr/bin/env python2 2"""A crontab script to delete night test data.""" 3 4from __future__ import print_function 5 6__author__ = 'shenhan@google.com (Han Shen)' 7 8import argparse 9import datetime 10import os 11import re 12import sys 13 14from cros_utils import command_executer 15from cros_utils import constants 16from cros_utils import misc 17 18DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun') 19 20 21def CleanNumberedDir(s, dry_run=False): 22 """Deleted directories under each dated_dir.""" 23 chromeos_dirs = [ 24 os.path.join(s, x) for x in os.listdir(s) 25 if misc.IsChromeOsTree(os.path.join(s, x)) 26 ] 27 ce = command_executer.GetCommandExecuter(log_level='none') 28 all_succeeded = True 29 for cd in chromeos_dirs: 30 if misc.DeleteChromeOsTree(cd, dry_run=dry_run): 31 print('Successfully removed chromeos tree "{0}".'.format(cd)) 32 else: 33 all_succeeded = False 34 print('Failed to remove chromeos tree "{0}", please check.'.format(cd)) 35 36 if not all_succeeded: 37 print('Failed to delete at least one chromeos tree, please check.') 38 return False 39 40 ## Now delete the numbered dir Before forcibly removing the directory, just 41 ## check 's' to make sure it is sane. A valid dir to be removed must be 42 ## '/usr/local/google/crostc/(SUN|MON|TUE...|SAT)'. 43 valid_dir_pattern = ( 44 '^' + constants.CROSTC_WORKSPACE + '/(' + '|'.join(DIR_BY_WEEKDAY) + ')') 45 if not re.search(valid_dir_pattern, s): 46 print('Trying to delete an invalid dir "{0}" (must match "{1}"), ' 47 'please check.'.format(s, valid_dir_pattern)) 48 return False 49 50 cmd = 'rm -fr {0}'.format(s) 51 if dry_run: 52 print(cmd) 53 else: 54 if ce.RunCommand(cmd, print_to_console=False, terminated_timeout=480) == 0: 55 print('Successfully removed "{0}".'.format(s)) 56 else: 57 all_succeeded = False 58 print('Failed to remove "{0}", please check.'.format(s)) 59 return all_succeeded 60 61 62def CleanDatedDir(dated_dir, dry_run=False): 63 # List subdirs under dir 64 subdirs = [ 65 os.path.join(dated_dir, x) for x in os.listdir(dated_dir) 66 if os.path.isdir(os.path.join(dated_dir, x)) 67 ] 68 all_succeeded = True 69 for s in subdirs: 70 if not CleanNumberedDir(s, dry_run): 71 all_succeeded = False 72 return all_succeeded 73 74 75def ProcessArguments(argv): 76 """Process arguments.""" 77 parser = argparse.ArgumentParser( 78 description='Automatically delete nightly test data directories.', 79 usage='auto_delete_nightly_test_data.py options') 80 parser.add_argument( 81 '-d', 82 '--dry_run', 83 dest='dry_run', 84 default=False, 85 action='store_true', 86 help='Only print command line, do not execute anything.') 87 parser.add_argument( 88 '--days_to_preserve', 89 dest='days_to_preserve', 90 default=3, 91 help=('Specify the number of days (not including today),' 92 ' test data generated on these days will *NOT* be ' 93 'deleted. Defaults to 3.')) 94 options = parser.parse_args(argv) 95 return options 96 97 98def CleanChromeOsTmpFiles(chroot_tmp, days_to_preserve, dry_run): 99 rv = 0 100 ce = command_executer.GetCommandExecuter() 101 # Clean chroot/tmp/test_that_* and chroot/tmp/tmpxxxxxx, that were last 102 # accessed more than specified time. 103 minutes = 1440 * days_to_preserve 104 cmd = (r'find {0} -maxdepth 1 -type d ' 105 r'\( -name "test_that_*" -amin +{1} -o ' 106 r' -name "cros-update*" -amin +{1} -o ' 107 r' -regex "{0}/tmp......" -amin +{1} \) ' 108 r'-exec bash -c "echo rm -fr {{}}" \; ' 109 r'-exec bash -c "rm -fr {{}}" \;').format(chroot_tmp, minutes) 110 if dry_run: 111 print('Going to execute:\n%s' % cmd) 112 else: 113 rv = ce.RunCommand(cmd, print_to_console=False) 114 if rv == 0: 115 print('Successfully cleaned chromeos tree tmp directory ' 116 '"{0}".'.format(chroot_tmp)) 117 else: 118 print('Some directories were not removed under chromeos tree ' 119 'tmp directory -"{0}".'.format(chroot_tmp)) 120 121 return rv 122 123 124def CleanChromeOsImageFiles(chroot_tmp, subdir_suffix, days_to_preserve, 125 dry_run): 126 rv = 0 127 rv2 = 0 128 ce = command_executer.GetCommandExecuter() 129 minutes = 1440 * days_to_preserve 130 # Clean files that were last accessed more than the specified time. 131 rv2 = 0 132 cmd = (r'find {0}/*{1}/* -maxdepth 1 -type d ' 133 r'-amin +{2} ' 134 r'-exec bash -c "echo rm -fr {{}}" \; ' 135 r'-exec bash -c "rm -fr {{}}" \;').format(chroot_tmp, subdir_suffix, 136 minutes) 137 if dry_run: 138 print('Going to execute:\n%s' % cmd) 139 else: 140 rv2 = ce.RunCommand(cmd, print_to_console=False) 141 if rv2 == 0: 142 print('Successfully cleaned chromeos image autotest directories from ' 143 '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix)) 144 else: 145 print('Some image autotest directories were not removed from ' 146 '"{0}/*{1}".'.format(chroot_tmp, subdir_suffix)) 147 148 rv += rv2 149 return rv 150 151 152def CleanChromeOsTmpAndImages(days_to_preserve=1, dry_run=False): 153 """Delete temporaries, images under crostc/chromeos.""" 154 chromeos_chroot_tmp = os.path.join(constants.CROSTC_WORKSPACE, 'chromeos', 155 'chroot', 'tmp') 156 # Clean files in tmp directory 157 rv = CleanChromeOsTmpFiles(chromeos_chroot_tmp, days_to_preserve, dry_run) 158 # Clean image files in *-tryjob directories 159 rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-tryjob', 160 days_to_preserve, dry_run) 161 # Clean image files in *-release directories 162 rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-release', 163 days_to_preserve, dry_run) 164 # Clean image files in *-pfq directories 165 rv += CleanChromeOsImageFiles(chromeos_chroot_tmp, '-pfq', days_to_preserve, 166 dry_run) 167 168 return rv 169 170 171def Main(argv): 172 """Delete nightly test data directories, tmps and test images.""" 173 options = ProcessArguments(argv) 174 # Function 'isoweekday' returns 1(Monday) - 7 (Sunday). 175 d = datetime.datetime.today().isoweekday() 176 # We go back 1 week, delete from that day till we are 177 # options.days_to_preserve away from today. 178 s = d - 7 179 e = d - int(options.days_to_preserve) 180 rv = 0 181 for i in range(s + 1, e): 182 if i <= 0: 183 ## Wrap around if index is negative. 6 is from i + 7 - 1, because 184 ## DIR_BY_WEEKDAY starts from 0, while isoweekday is from 1-7. 185 dated_dir = DIR_BY_WEEKDAY[i + 6] 186 else: 187 dated_dir = DIR_BY_WEEKDAY[i - 1] 188 189 rv += 0 if CleanDatedDir( 190 os.path.join(constants.CROSTC_WORKSPACE, dated_dir), 191 options.dry_run) else 1 192 193## Finally clean temporaries, images under crostc/chromeos 194 rv2 = CleanChromeOsTmpAndImages( 195 int(options.days_to_preserve), options.dry_run) 196 197 return rv + rv2 198 199if __name__ == '__main__': 200 retval = Main(sys.argv[1:]) 201 sys.exit(retval) 202