1#!/usr/bin/python -u 2 3""" 4Utility to upload or remove the packages from the packages repository. 5""" 6 7import logging, optparse, os, shutil, sys, tempfile 8import common 9from autotest_lib.client.common_lib import utils as client_utils 10from autotest_lib.client.common_lib import global_config, error 11from autotest_lib.client.common_lib import packages 12from autotest_lib.server import utils as server_utils 13 14c = global_config.global_config 15logging.basicConfig(level=logging.DEBUG) 16 17ACTION_REMOVE = 'remove' 18ACTION_UPLOAD = 'upload' 19ACTION_TAR_ONLY = 'tar_only' 20 21def get_exclude_string(client_dir): 22 ''' 23 Get the exclude string for the tar command to exclude specific 24 subdirectories inside client_dir. 25 For profilers we need to exclude everything except the __init__.py 26 file so that the profilers can be imported. 27 ''' 28 exclude_string = ('--exclude="deps/*" --exclude="tests/*" ' 29 '--exclude="site_tests/*" --exclude="**.pyc"') 30 31 # Get the profilers directory 32 prof_dir = os.path.join(client_dir, 'profilers') 33 34 # Include the __init__.py file for the profilers and exclude all its 35 # subdirectories 36 for f in os.listdir(prof_dir): 37 if os.path.isdir(os.path.join(prof_dir, f)): 38 exclude_string += ' --exclude="profilers/%s"' % f 39 40 # The '.' here is needed to zip the files in the current 41 # directory. We use '-C' for tar to change to the required 42 # directory i.e. src_dir and then zip up the files in that 43 # directory(which is '.') excluding the ones in the exclude_dirs 44 exclude_string += " ." 45 46 # TODO(milleral): This is sad and ugly. http://crbug.com/258161 47 # Surprisingly, |exclude_string| actually means argument list, and 48 # we'd like to package up the current global_config.ini also, so let's 49 # just tack it on here. 50 # Also note that this only works because tar prevents us from un-tarring 51 # files into parent directories. 52 exclude_string += " ../global_config.ini" 53 54 return exclude_string 55 56 57def parse_args(): 58 parser = optparse.OptionParser() 59 parser.add_option("-d", "--dependency", help="package the dependency" 60 " from client/deps directory and upload to the repo", 61 dest="dep") 62 parser.add_option("-p", "--profiler", help="package the profiler " 63 "from client/profilers directory and upload to the repo", 64 dest="prof") 65 parser.add_option("-t", "--test", help="package the test from client/tests" 66 " or client/site_tests and upload to the repo.", 67 dest="test") 68 parser.add_option("-c", "--client", help="package the client " 69 "directory alone without the tests, deps and profilers", 70 dest="client", action="store_true", default=False) 71 parser.add_option("-f", "--file", help="simply uploads the specified" 72 "file on to the repo", dest="file") 73 parser.add_option("-r", "--repository", help="the URL of the packages" 74 "repository location to upload the packages to.", 75 dest="repo", default=None) 76 parser.add_option("-o", "--output_dir", help="the output directory" 77 "to place tarballs and md5sum files in.", 78 dest="output_dir", default=None) 79 parser.add_option("-a", "--action", help="the action to perform", 80 dest="action", choices=(ACTION_UPLOAD, ACTION_REMOVE, 81 ACTION_TAR_ONLY), default=None) 82 parser.add_option("--all", help="Upload all the files locally " 83 "to all the repos specified in global_config.ini. " 84 "(includes the client, tests, deps and profilers)", 85 dest="all", action="store_true", default=False) 86 87 options, args = parser.parse_args() 88 return options, args 89 90def get_build_dir(name, dest_dir, pkg_type): 91 """Method to generate the build directory where the tarball and checksum 92 is stored. The following package types are handled: test, dep, profiler. 93 Package type 'client' is not handled. 94 """ 95 if pkg_type == 'client': 96 # NOTE: The "tar_only" action for pkg_type "client" has no use 97 # case yet. No known invocations of packager.py with 98 # --action=tar_only send in clients in the command line. Please 99 # confirm the behaviour is expected before this type is enabled for 100 # "tar_only" actions. 101 print ('Tar action not supported for pkg_type= %s, name = %s' % 102 pkg_type, name) 103 return None 104 # For all packages, the work-dir should have 'client' appended to it. 105 base_build_dir = os.path.join(dest_dir, 'client') 106 if pkg_type == 'test': 107 build_dir = os.path.join(get_test_dir(name, base_build_dir), name) 108 else: 109 # For profiler and dep, we append 's', and then append the name. 110 # TODO(pmalani): Make this less fiddly? 111 build_dir = os.path.join(base_build_dir, pkg_type + 's', name) 112 return build_dir 113 114def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, 115 action, dest_dir=None): 116 """Method to upload or remove package depending on the flag passed to it. 117 118 If tar_only is set to True, this routine is solely used to generate a 119 tarball and compute the md5sum from that tarball. 120 If the tar_only flag is True, then the remove flag is ignored. 121 """ 122 exclude_string = ' .' 123 names = [p.strip() for p in pkg_names.split(',')] 124 for name in names: 125 print "process_packages: Processing %s ... " % name 126 if pkg_type == 'client': 127 pkg_dir = src_dir 128 exclude_string = get_exclude_string(pkg_dir) 129 elif pkg_type == 'test': 130 # if the package is a test then look whether it is in client/tests 131 # or client/site_tests 132 pkg_dir = os.path.join(get_test_dir(name, src_dir), name) 133 else: 134 # for the profilers and deps 135 pkg_dir = os.path.join(src_dir, name) 136 137 pkg_name = pkgmgr.get_tarball_name(name, pkg_type) 138 139 exclude_string_tar = (( 140 ' --exclude="**%s" --exclude="**%s.checksum" ' % 141 (pkg_name, pkg_name)) + exclude_string) 142 if action == ACTION_TAR_ONLY: 143 # We don't want any pre-existing tarballs and checksums to 144 # be repackaged, so we should purge these. 145 build_dir = get_build_dir(name, dest_dir, pkg_type) 146 try: 147 packages.check_diskspace(build_dir) 148 except error.RepoDiskFullError as e: 149 msg = ("Work_dir directory for packages %s does not have " 150 "enough space available: %s" % (build_dir, e)) 151 raise error.RepoDiskFullError(msg) 152 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, 153 build_dir, exclude_string_tar) 154 155 # Create the md5 hash too. 156 md5sum = pkgmgr.compute_checksum(tarball_path) 157 md5sum_filepath = os.path.join(build_dir, pkg_name + '.checksum') 158 with open(md5sum_filepath, "w") as f: 159 f.write(md5sum) 160 161 elif action == ACTION_UPLOAD: 162 # Tar the source and upload 163 temp_dir = tempfile.mkdtemp() 164 try: 165 try: 166 packages.check_diskspace(temp_dir) 167 except error.RepoDiskFullError, e: 168 msg = ("Temporary directory for packages %s does not have " 169 "enough space available: %s" % (temp_dir, e)) 170 raise error.RepoDiskFullError(msg) 171 172 # Check if tarball already exists. If it does, then don't 173 # create a tarball again. 174 tarball_path = os.path.join(pkg_dir, pkg_name); 175 if os.path.exists(tarball_path): 176 print("process_packages: Tarball %s already exists" % 177 tarball_path) 178 else: 179 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, 180 temp_dir, 181 exclude_string_tar) 182 # Compare the checksum with what packages.checksum has. If they 183 # match then we don't need to perform the upload. 184 if not pkgmgr.compare_checksum(tarball_path): 185 pkgmgr.upload_pkg(tarball_path, update_checksum=True) 186 else: 187 logging.warning('Checksum not changed for %s, not copied ' 188 'in packages/ directory.', tarball_path) 189 finally: 190 # remove the temporary directory 191 shutil.rmtree(temp_dir) 192 elif action == ACTION_REMOVE: 193 pkgmgr.remove_pkg(pkg_name, remove_checksum=True) 194 print "Done." 195 196 197def tar_packages(pkgmgr, pkg_type, pkg_names, src_dir, temp_dir): 198 """Tar all packages up and return a list of each tar created""" 199 tarballs = [] 200 exclude_string = ' .' 201 names = [p.strip() for p in pkg_names.split(',')] 202 for name in names: 203 print "tar_packages: Processing %s ... " % name 204 if pkg_type == 'client': 205 pkg_dir = src_dir 206 exclude_string = get_exclude_string(pkg_dir) 207 elif pkg_type == 'test': 208 # if the package is a test then look whether it is in client/tests 209 # or client/site_tests 210 pkg_dir = os.path.join(get_test_dir(name, src_dir), name) 211 else: 212 # for the profilers and deps 213 pkg_dir = os.path.join(src_dir, name) 214 215 pkg_name = pkgmgr.get_tarball_name(name, pkg_type) 216 217 # We don't want any pre-existing tarballs and checksums to 218 # be repackaged, so we should purge these. 219 exclude_string_tar = (( 220 ' --exclude="**%s" --exclude="**%s.checksum" ' % 221 (pkg_name, pkg_name)) + exclude_string) 222 # Check if tarball already exists. If it does, don't duplicate 223 # the effort. 224 tarball_path = os.path.join(pkg_dir, pkg_name); 225 if os.path.exists(tarball_path): 226 print("tar_packages: Tarball %s already exists" % tarball_path); 227 else: 228 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, 229 temp_dir, exclude_string_tar) 230 tarballs.append(tarball_path) 231 return tarballs 232 233 234def process_all_packages(pkgmgr, client_dir, action): 235 """Process a full upload of packages as a directory upload.""" 236 dep_dir = os.path.join(client_dir, "deps") 237 prof_dir = os.path.join(client_dir, "profilers") 238 # Directory where all are kept 239 temp_dir = tempfile.mkdtemp() 240 try: 241 packages.check_diskspace(temp_dir) 242 except error.RepoDiskFullError, e: 243 print ("Temp destination for packages is full %s, aborting upload: %s" 244 % (temp_dir, e)) 245 os.rmdir(temp_dir) 246 sys.exit(1) 247 248 # process tests 249 tests_list = get_subdir_list('tests', client_dir) 250 tests = ','.join(tests_list) 251 252 # process site_tests 253 site_tests_list = get_subdir_list('site_tests', client_dir) 254 site_tests = ','.join(site_tests_list) 255 256 # process deps 257 deps_list = get_subdir_list('deps', client_dir) 258 deps = ','.join(deps_list) 259 260 # process profilers 261 profilers_list = get_subdir_list('profilers', client_dir) 262 profilers = ','.join(profilers_list) 263 264 # Update md5sum 265 if action == ACTION_UPLOAD: 266 all_packages = [] 267 all_packages.extend(tar_packages(pkgmgr, 'profiler', profilers, 268 prof_dir, temp_dir)) 269 all_packages.extend(tar_packages(pkgmgr, 'dep', deps, dep_dir, 270 temp_dir)) 271 all_packages.extend(tar_packages(pkgmgr, 'test', site_tests, 272 client_dir, temp_dir)) 273 all_packages.extend(tar_packages(pkgmgr, 'test', tests, client_dir, 274 temp_dir)) 275 all_packages.extend(tar_packages(pkgmgr, 'client', 'autotest', 276 client_dir, temp_dir)) 277 for package in all_packages: 278 pkgmgr.upload_pkg(package, update_checksum=True) 279 client_utils.run('rm -rf ' + temp_dir) 280 elif action == ACTION_REMOVE: 281 process_packages(pkgmgr, 'test', tests, client_dir, action=action) 282 process_packages(pkgmgr, 'test', site_tests, client_dir, action=action) 283 process_packages(pkgmgr, 'client', 'autotest', client_dir, 284 action=action) 285 process_packages(pkgmgr, 'dep', deps, dep_dir, action=action) 286 process_packages(pkgmgr, 'profiler', profilers, prof_dir, 287 action=action) 288 289 290# Get the list of sub directories present in a directory 291def get_subdir_list(name, client_dir): 292 dir_name = os.path.join(client_dir, name) 293 return [f for f in 294 os.listdir(dir_name) 295 if os.path.isdir(os.path.join(dir_name, f)) ] 296 297 298# Look whether the test is present in client/tests and client/site_tests dirs 299def get_test_dir(name, client_dir): 300 names_test = os.listdir(os.path.join(client_dir, 'tests')) 301 names_site_test = os.listdir(os.path.join(client_dir, 'site_tests')) 302 if name in names_test: 303 src_dir = os.path.join(client_dir, 'tests') 304 elif name in names_site_test: 305 src_dir = os.path.join(client_dir, 'site_tests') 306 else: 307 print "Test %s not found" % name 308 sys.exit(0) 309 return src_dir 310 311 312def main(): 313 # get options and args 314 options, args = parse_args() 315 316 server_dir = server_utils.get_server_dir() 317 autotest_dir = os.path.abspath(os.path.join(server_dir, '..')) 318 319 # extract the pkg locations from global config 320 repo_urls = c.get_config_value('PACKAGES', 'fetch_location', 321 type=list, default=[]) 322 upload_paths = c.get_config_value('PACKAGES', 'upload_location', 323 type=list, default=[]) 324 325 if options.repo: 326 upload_paths.append(options.repo) 327 328 # Having no upload paths basically means you're not using packaging. 329 if not upload_paths: 330 print("No upload locations found. Please set upload_location under" 331 " PACKAGES in the global_config.ini or provide a location using" 332 " the --repository option.") 333 return 334 335 client_dir = os.path.join(autotest_dir, "client") 336 337 # Bail out if the client directory does not exist 338 if not os.path.exists(client_dir): 339 sys.exit(0) 340 341 dep_dir = os.path.join(client_dir, "deps") 342 prof_dir = os.path.join(client_dir, "profilers") 343 344 # Due to the delayed uprev-ing of certain ebuilds, we need to support 345 # both the legacy command line and the new one. 346 # So, if the new "action" option isn't specified, try looking for the 347 # old style remove/upload argument 348 if options.action is None: 349 if len(args) == 0 or args[0] not in ['upload', 'remove']: 350 print("Either 'upload' or 'remove' needs to be specified " 351 "for the package") 352 sys.exit(0) 353 cur_action = args[0] 354 else: 355 cur_action = options.action 356 357 if cur_action == ACTION_TAR_ONLY and options.output_dir is None: 358 print("An output dir has to be specified") 359 sys.exit(0) 360 361 pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls, 362 upload_paths=upload_paths, 363 run_function_dargs={'timeout':600}) 364 365 if options.all: 366 process_all_packages(pkgmgr, client_dir, action=cur_action) 367 368 if options.client: 369 process_packages(pkgmgr, 'client', 'autotest', client_dir, 370 action=cur_action) 371 372 if options.dep: 373 process_packages(pkgmgr, 'dep', options.dep, dep_dir, 374 action=cur_action, dest_dir=options.output_dir) 375 376 if options.test: 377 process_packages(pkgmgr, 'test', options.test, client_dir, 378 action=cur_action, dest_dir=options.output_dir) 379 380 if options.prof: 381 process_packages(pkgmgr, 'profiler', options.prof, prof_dir, 382 action=cur_action, dest_dir=options.output_dir) 383 384 if options.file: 385 if cur_action == ACTION_REMOVE: 386 pkgmgr.remove_pkg(options.file, remove_checksum=True) 387 elif cur_action == ACTION_UPLOAD: 388 pkgmgr.upload_pkg(options.file, update_checksum=True) 389 390 391if __name__ == "__main__": 392 main() 393