1#!/usr/bin/python3 -u 2 3""" 4Utility to upload or remove the packages from the packages repository. 5""" 6 7from __future__ import absolute_import 8from __future__ import division 9from __future__ import print_function 10import logging, optparse, os, shutil, sys, tempfile 11import common 12from autotest_lib.client.common_lib import utils as client_utils 13from autotest_lib.client.common_lib import global_config, error 14from autotest_lib.client.common_lib import packages 15from autotest_lib.server import utils as server_utils 16 17c = global_config.global_config 18logging.basicConfig(level=logging.DEBUG) 19 20ACTION_REMOVE = 'remove' 21ACTION_UPLOAD = 'upload' 22ACTION_TAR_ONLY = 'tar_only' 23 24def get_exclude_string(client_dir): 25 ''' 26 Get the exclude string for the tar command to exclude specific 27 subdirectories inside client_dir. 28 For profilers we need to exclude everything except the __init__.py 29 file so that the profilers can be imported. 30 ''' 31 exclude_string = ('--exclude="deps/*" --exclude="tests/*" --exclude=.git ' 32 '--exclude="site_tests/*" --exclude="**.pyc"') 33 34 # Get the profilers directory 35 prof_dir = os.path.join(client_dir, 'profilers') 36 37 # Include the __init__.py file for the profilers and exclude all its 38 # subdirectories 39 for f in os.listdir(prof_dir): 40 if os.path.isdir(os.path.join(prof_dir, f)): 41 exclude_string += ' --exclude="profilers/%s"' % f 42 43 # The '.' here is needed to zip the files in the current 44 # directory. We use '-C' for tar to change to the required 45 # directory i.e. src_dir and then zip up the files in that 46 # directory(which is '.') excluding the ones in the exclude_dirs 47 exclude_string += " ." 48 49 # TODO(milleral): This is sad and ugly. http://crbug.com/258161 50 # Surprisingly, |exclude_string| actually means argument list, and 51 # we'd like to package up the current global_config.ini also, so let's 52 # just tack it on here. 53 # Also note that this only works because tar prevents us from un-tarring 54 # files into parent directories. 55 exclude_string += " ../global_config.ini" 56 57 return exclude_string 58 59 60def parse_args(): 61 parser = optparse.OptionParser() 62 parser.add_option("-d", "--dependency", help="package the dependency" 63 " from client/deps directory and upload to the repo", 64 dest="dep") 65 parser.add_option("-p", "--profiler", help="package the profiler " 66 "from client/profilers directory and upload to the repo", 67 dest="prof") 68 parser.add_option("-t", "--test", help="package the test from client/tests" 69 " or client/site_tests and upload to the repo.", 70 dest="test") 71 parser.add_option("-c", "--client", help="package the client " 72 "directory alone without the tests, deps and profilers", 73 dest="client", action="store_true", default=False) 74 parser.add_option("-f", "--file", help="simply uploads the specified" 75 "file on to the repo", dest="file") 76 parser.add_option("-r", "--repository", help="the URL of the packages" 77 "repository location to upload the packages to.", 78 dest="repo", default=None) 79 parser.add_option("-o", "--output_dir", help="the output directory" 80 "to place tarballs and md5sum files in.", 81 dest="output_dir", default=None) 82 parser.add_option("-a", "--action", help="the action to perform", 83 dest="action", choices=(ACTION_UPLOAD, ACTION_REMOVE, 84 ACTION_TAR_ONLY), default=None) 85 parser.add_option("--all", help="Upload all the files locally " 86 "to all the repos specified in global_config.ini. " 87 "(includes the client, tests, deps and profilers)", 88 dest="all", action="store_true", default=False) 89 90 options, args = parser.parse_args() 91 return options, args 92 93def get_build_dir(name, dest_dir, pkg_type): 94 """Method to generate the build directory where the tarball and checksum 95 is stored. The following package types are handled: test, dep, profiler. 96 Package type 'client' is not handled. 97 """ 98 if pkg_type == 'client': 99 # NOTE: The "tar_only" action for pkg_type "client" has no use 100 # case yet. No known invocations of packager.py with 101 # --action=tar_only send in clients in the command line. Please 102 # confirm the behaviour is expected before this type is enabled for 103 # "tar_only" actions. 104 print('Tar action not supported for pkg_type= %s, name = %s' % 105 pkg_type, name) 106 return None 107 # For all packages, the work-dir should have 'client' appended to it. 108 base_build_dir = os.path.join(dest_dir, 'client') 109 if pkg_type == 'test': 110 build_dir = os.path.join(get_test_dir(name, base_build_dir), name) 111 else: 112 # For profiler and dep, we append 's', and then append the name. 113 # TODO(pmalani): Make this less fiddly? 114 build_dir = os.path.join(base_build_dir, pkg_type + 's', name) 115 return build_dir 116 117def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, 118 action, dest_dir=None): 119 """Method to upload or remove package depending on the flag passed to it. 120 121 If tar_only is set to True, this routine is solely used to generate a 122 tarball and compute the md5sum from that tarball. 123 If the tar_only flag is True, then the remove flag is ignored. 124 """ 125 exclude_string = ' .' 126 names = [p.strip() for p in pkg_names.split(',')] 127 for name in names: 128 print("process_packages: Processing %s ... " % name) 129 if pkg_type == 'client': 130 pkg_dir = src_dir 131 exclude_string = get_exclude_string(pkg_dir) 132 elif pkg_type == 'test': 133 # if the package is a test then look whether it is in client/tests 134 # or client/site_tests 135 pkg_dir = os.path.join(get_test_dir(name, src_dir), name) 136 else: 137 # for the profilers and deps 138 pkg_dir = os.path.join(src_dir, name) 139 140 pkg_name = pkgmgr.get_tarball_name(name, pkg_type) 141 142 exclude_string_tar = (( 143 ' --exclude="**%s" --exclude="**%s.checksum" ' % 144 (pkg_name, pkg_name)) + exclude_string) 145 if action == ACTION_TAR_ONLY: 146 # We don't want any pre-existing tarballs and checksums to 147 # be repackaged, so we should purge these. 148 build_dir = get_build_dir(name, dest_dir, pkg_type) 149 try: 150 packages.check_diskspace(build_dir) 151 except error.RepoDiskFullError as e: 152 msg = ("Work_dir directory for packages %s does not have " 153 "enough space available: %s" % (build_dir, e)) 154 raise error.RepoDiskFullError(msg) 155 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, 156 build_dir, exclude_string_tar) 157 158 # Create the md5 hash too. 159 md5sum = pkgmgr.compute_checksum(tarball_path) 160 md5sum_filepath = os.path.join(build_dir, pkg_name + '.checksum') 161 with open(md5sum_filepath, "w") as f: 162 f.write(md5sum) 163 164 elif action == ACTION_UPLOAD: 165 # Tar the source and upload 166 temp_dir = tempfile.mkdtemp() 167 try: 168 try: 169 packages.check_diskspace(temp_dir) 170 except error.RepoDiskFullError as e: 171 msg = ("Temporary directory for packages %s does not have " 172 "enough space available: %s" % (temp_dir, e)) 173 raise error.RepoDiskFullError(msg) 174 175 # Check if tarball already exists. If it does, then don't 176 # create a tarball again. 177 tarball_path = os.path.join(pkg_dir, pkg_name); 178 if os.path.exists(tarball_path): 179 print("process_packages: Tarball %s already exists" % 180 tarball_path) 181 else: 182 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, 183 temp_dir, 184 exclude_string_tar) 185 # Compare the checksum with what packages.checksum has. If they 186 # match then we don't need to perform the upload. 187 if not pkgmgr.compare_checksum(tarball_path): 188 pkgmgr.upload_pkg(tarball_path, update_checksum=True) 189 else: 190 logging.warning('Checksum not changed for %s, not copied ' 191 'in packages/ directory.', tarball_path) 192 finally: 193 # remove the temporary directory 194 shutil.rmtree(temp_dir) 195 elif action == ACTION_REMOVE: 196 pkgmgr.remove_pkg(pkg_name, remove_checksum=True) 197 print("Done.") 198 199 200def tar_packages(pkgmgr, pkg_type, pkg_names, src_dir, temp_dir): 201 """Tar all packages up and return a list of each tar created""" 202 tarballs = [] 203 exclude_string = ' .' 204 names = [p.strip() for p in pkg_names.split(',')] 205 for name in names: 206 print("tar_packages: Processing %s ... " % name) 207 if pkg_type == 'client': 208 pkg_dir = src_dir 209 exclude_string = get_exclude_string(pkg_dir) 210 elif pkg_type == 'test': 211 # if the package is a test then look whether it is in client/tests 212 # or client/site_tests 213 pkg_dir = os.path.join(get_test_dir(name, src_dir), name) 214 else: 215 # for the profilers and deps 216 pkg_dir = os.path.join(src_dir, name) 217 218 pkg_name = pkgmgr.get_tarball_name(name, pkg_type) 219 220 # We don't want any pre-existing tarballs and checksums to 221 # be repackaged, so we should purge these. 222 exclude_string_tar = (( 223 ' --exclude="**%s" --exclude="**%s.checksum" ' % 224 (pkg_name, pkg_name)) + exclude_string) 225 # Check if tarball already exists. If it does, don't duplicate 226 # the effort. 227 tarball_path = os.path.join(pkg_dir, pkg_name) 228 if os.path.exists(tarball_path): 229 print("tar_packages: Tarball %s already exists" % tarball_path) 230 else: 231 tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, 232 temp_dir, exclude_string_tar) 233 tarballs.append(tarball_path) 234 return tarballs 235 236 237def process_all_packages(pkgmgr, client_dir, action): 238 """Process a full upload of packages as a directory upload.""" 239 dep_dir = os.path.join(client_dir, "deps") 240 prof_dir = os.path.join(client_dir, "profilers") 241 # Directory where all are kept 242 temp_dir = tempfile.mkdtemp() 243 try: 244 packages.check_diskspace(temp_dir) 245 except error.RepoDiskFullError as e: 246 print("Temp destination for packages is full %s, aborting upload: %s" 247 % (temp_dir, e)) 248 os.rmdir(temp_dir) 249 sys.exit(1) 250 251 # process tests 252 tests_list = get_subdir_list('tests', client_dir) 253 tests = ','.join(tests_list) 254 255 # process site_tests 256 site_tests_list = get_subdir_list('site_tests', client_dir) 257 site_tests = ','.join(site_tests_list) 258 259 # process deps 260 deps_list = get_subdir_list('deps', client_dir) 261 deps = ','.join(deps_list) 262 263 # process profilers 264 profilers_list = get_subdir_list('profilers', client_dir) 265 profilers = ','.join(profilers_list) 266 267 # Update md5sum 268 if action == ACTION_UPLOAD: 269 all_packages = [] 270 all_packages.extend(tar_packages(pkgmgr, 'profiler', profilers, 271 prof_dir, temp_dir)) 272 all_packages.extend(tar_packages(pkgmgr, 'dep', deps, dep_dir, 273 temp_dir)) 274 all_packages.extend(tar_packages(pkgmgr, 'test', site_tests, 275 client_dir, temp_dir)) 276 all_packages.extend(tar_packages(pkgmgr, 'test', tests, client_dir, 277 temp_dir)) 278 all_packages.extend(tar_packages(pkgmgr, 'client', 'autotest', 279 client_dir, temp_dir)) 280 for package in all_packages: 281 pkgmgr.upload_pkg(package, update_checksum=True) 282 client_utils.run('rm -rf ' + temp_dir) 283 elif action == ACTION_REMOVE: 284 process_packages(pkgmgr, 'test', tests, client_dir, action=action) 285 process_packages(pkgmgr, 'test', site_tests, client_dir, action=action) 286 process_packages(pkgmgr, 'client', 'autotest', client_dir, 287 action=action) 288 process_packages(pkgmgr, 'dep', deps, dep_dir, action=action) 289 process_packages(pkgmgr, 'profiler', profilers, prof_dir, 290 action=action) 291 292 293# Get the list of sub directories present in a directory 294def get_subdir_list(name, client_dir): 295 dir_name = os.path.join(client_dir, name) 296 return [f for f in 297 os.listdir(dir_name) 298 if os.path.isdir(os.path.join(dir_name, f)) ] 299 300 301# Look whether the test is present in client/tests and client/site_tests dirs 302def get_test_dir(name, client_dir): 303 names_test = os.listdir(os.path.join(client_dir, 'tests')) 304 names_site_test = os.listdir(os.path.join(client_dir, 'site_tests')) 305 if name in names_test: 306 src_dir = os.path.join(client_dir, 'tests') 307 elif name in names_site_test: 308 src_dir = os.path.join(client_dir, 'site_tests') 309 else: 310 print("Test %s not found" % name) 311 sys.exit(0) 312 return src_dir 313 314 315def main(): 316 # get options and args 317 options, args = parse_args() 318 319 server_dir = server_utils.get_server_dir() 320 autotest_dir = os.path.abspath(os.path.join(server_dir, '..')) 321 322 # extract the pkg locations from global config 323 repo_urls = c.get_config_value('PACKAGES', 'fetch_location', 324 type=list, default=[]) 325 upload_paths = c.get_config_value('PACKAGES', 'upload_location', 326 type=list, default=[]) 327 328 if options.repo: 329 upload_paths.append(options.repo) 330 331 # Having no upload paths basically means you're not using packaging. 332 if not upload_paths: 333 print("No upload locations found. Please set upload_location under" 334 " PACKAGES in the global_config.ini or provide a location using" 335 " the --repository option.") 336 return 337 338 client_dir = os.path.join(autotest_dir, "client") 339 340 # Bail out if the client directory does not exist 341 if not os.path.exists(client_dir): 342 sys.exit(0) 343 344 dep_dir = os.path.join(client_dir, "deps") 345 prof_dir = os.path.join(client_dir, "profilers") 346 347 # Due to the delayed uprev-ing of certain ebuilds, we need to support 348 # both the legacy command line and the new one. 349 # So, if the new "action" option isn't specified, try looking for the 350 # old style remove/upload argument 351 if options.action is None: 352 if len(args) == 0 or args[0] not in ['upload', 'remove']: 353 print("Either 'upload' or 'remove' needs to be specified " 354 "for the package") 355 sys.exit(0) 356 cur_action = args[0] 357 else: 358 cur_action = options.action 359 360 if cur_action == ACTION_TAR_ONLY and options.output_dir is None: 361 print("An output dir has to be specified") 362 sys.exit(0) 363 364 pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls, 365 upload_paths=upload_paths, 366 run_function_dargs={'timeout':600}) 367 368 if options.all: 369 process_all_packages(pkgmgr, client_dir, action=cur_action) 370 371 if options.client: 372 process_packages(pkgmgr, 'client', 'autotest', client_dir, 373 action=cur_action) 374 375 if options.dep: 376 process_packages(pkgmgr, 'dep', options.dep, dep_dir, 377 action=cur_action, dest_dir=options.output_dir) 378 379 if options.test: 380 process_packages(pkgmgr, 'test', options.test, client_dir, 381 action=cur_action, dest_dir=options.output_dir) 382 383 if options.prof: 384 process_packages(pkgmgr, 'profiler', options.prof, prof_dir, 385 action=cur_action, dest_dir=options.output_dir) 386 387 if options.file: 388 if cur_action == ACTION_REMOVE: 389 pkgmgr.remove_pkg(options.file, remove_checksum=True) 390 elif cur_action == ACTION_UPLOAD: 391 pkgmgr.upload_pkg(options.file, update_checksum=True) 392 393 394if __name__ == "__main__": 395 main() 396