1# Copyright 2018, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Integration Finder class. 17""" 18 19# pylint: disable=line-too-long 20 21import copy 22import logging 23import os 24import re 25import tempfile 26import xml.etree.ElementTree as ElementTree 27 28from zipfile import ZipFile 29 30from atest import atest_error 31from atest import constants 32 33from atest.test_finders import test_info 34from atest.test_finders import test_finder_base 35from atest.test_finders import test_finder_utils 36from atest.test_runners import atest_tf_test_runner 37 38# Find integration name based on file path of integration config xml file. 39# Group matches "foo/bar" given "blah/res/config/foo/bar.xml from source code 40# res directory or "blah/config/foo/bar.xml from prebuilt jars. 41_INT_NAME_RE = re.compile(r'^.*\/config\/(?P<int_name>.*).xml$') 42_TF_TARGETS = frozenset(['tradefed', 'tradefed-contrib']) 43_GTF_TARGETS = frozenset(['google-tradefed', 'google-tradefed-contrib']) 44_CONTRIB_TARGETS = frozenset(['google-tradefed-contrib']) 45_TF_RES_DIRS = frozenset(['../res/config', 'res/config']) 46 47 48class TFIntegrationFinder(test_finder_base.TestFinderBase): 49 """Integration Finder class.""" 50 NAME = 'INTEGRATION' 51 _TEST_RUNNER = atest_tf_test_runner.AtestTradefedTestRunner.NAME 52 53 54 def __init__(self, module_info=None): 55 super().__init__() 56 self.root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 57 self.module_info = module_info 58 # TODO: Break this up into AOSP/google_tf integration finders. 59 self.tf_dirs, self.gtf_dirs = self._get_integration_dirs() 60 self.integration_dirs = self.tf_dirs + self.gtf_dirs 61 self.temp_dir = tempfile.TemporaryDirectory() 62 63 def _get_mod_paths(self, module_name): 64 """Return the paths of the given module name.""" 65 if self.module_info: 66 # Since aosp/801774 merged, the path of test configs have been 67 # changed to ../res/config. 68 if module_name in _CONTRIB_TARGETS: 69 mod_paths = self.module_info.get_paths(module_name) 70 return [os.path.join(path, res_path) for path in mod_paths 71 for res_path in _TF_RES_DIRS] 72 return self.module_info.get_paths(module_name) 73 return [] 74 75 def _get_integration_dirs(self): 76 """Get integration dirs from MODULE_INFO based on targets. 77 78 Returns: 79 A tuple of lists of strings of integration dir rel to repo root. 80 """ 81 tf_dirs = list(filter(None, [d for x in _TF_TARGETS for d in self._get_mod_paths(x)])) 82 gtf_dirs = list(filter(None, [d for x in _GTF_TARGETS for d in self._get_mod_paths(x)])) 83 return tf_dirs, gtf_dirs 84 85 def _get_build_targets(self, rel_config): 86 config_file = os.path.join(self.root_dir, rel_config) 87 xml_root = self._load_xml_file(config_file) 88 targets = test_finder_utils.get_targets_from_xml_root(xml_root, 89 self.module_info) 90 if self.gtf_dirs: 91 targets.add(constants.GTF_TARGET) 92 return frozenset(targets) 93 94 def _load_xml_file(self, path): 95 """Load an xml file with option to expand <include> tags 96 97 Args: 98 path: A string of path to xml file. 99 100 Returns: 101 An xml.etree.ElementTree.Element instance of the root of the tree. 102 """ 103 tree = ElementTree.parse(path) 104 root = tree.getroot() 105 self._load_include_tags(root) 106 return root 107 108 #pylint: disable=invalid-name 109 def _load_include_tags(self, root): 110 """Recursively expand in-place the <include> tags in a given xml tree. 111 112 Python xml libraries don't support our type of <include> tags. Logic 113 used below is modified version of the built-in ElementInclude logic 114 found here: 115 https://github.com/python/cpython/blob/2.7/Lib/xml/etree/ElementInclude.py 116 117 Args: 118 root: The root xml.etree.ElementTree.Element. 119 120 Returns: 121 An xml.etree.ElementTree.Element instance with 122 include tags expanded. 123 """ 124 i = 0 125 while i < len(root): 126 elem = root[i] 127 if elem.tag == 'include': 128 # expand included xml file 129 integration_name = elem.get('name') 130 if not integration_name: 131 logging.warning('skipping <include> tag with no "name" value') 132 continue 133 full_paths = self._search_integration_dirs(integration_name) 134 if not full_paths: 135 full_paths = self._search_prebuilt_jars(integration_name) 136 node = None 137 if full_paths: 138 node = self._load_xml_file(full_paths[0]) 139 if node is None: 140 raise atest_error.FatalIncludeError("can't load %r" % 141 integration_name) 142 node = copy.copy(node) 143 if elem.tail: 144 node.tail = (node.tail or "") + elem.tail 145 root[i] = node 146 i = i + 1 147 148 def _search_integration_dirs(self, name): 149 """Search integration dirs for name and return full path. 150 Args: 151 name: A string of integration name as seen in tf's list configs. 152 153 Returns: 154 A list of test path. 155 """ 156 test_files = [] 157 for integration_dir in self.integration_dirs: 158 abs_path = os.path.join(self.root_dir, integration_dir) 159 found_test_files = test_finder_utils.run_find_cmd( 160 test_finder_utils.TestReferenceType.INTEGRATION, 161 abs_path, name) 162 if found_test_files: 163 test_files.extend(found_test_files) 164 return test_files 165 166 def find_test_by_integration_name(self, name): 167 """Find the test info matching the given integration name. 168 169 Args: 170 name: A string of integration name as seen in tf's list configs. 171 172 Returns: 173 A populated TestInfo namedtuple if test found, else None 174 """ 175 class_name = None 176 parse_result = test_finder_utils.parse_test_reference(name) 177 if parse_result: 178 name = parse_result['module_name'] 179 class_name = parse_result['pkg_class_name'] 180 method = parse_result.get('method_name', '') 181 if method: 182 class_name = class_name + '#' + method 183 test_files = self._search_integration_dirs(name) 184 if not test_files: 185 # Check prebuilt jars if input name is in jars. 186 test_files = self._search_prebuilt_jars(name) 187 # Don't use names that simply match the path, 188 # must be the actual name used by TF to run the test. 189 t_infos = [] 190 for test_file in test_files: 191 t_info = self._get_test_info(name, test_file, class_name) 192 if t_info: 193 t_infos.append(t_info) 194 return t_infos 195 196 def _get_prebuilt_jars(self): 197 """Get prebuilt jars based on targets. 198 199 Returns: 200 A tuple of lists of strings of prebuilt jars. 201 """ 202 prebuilt_jars = [] 203 for tf_dir in self.tf_dirs: 204 for tf_target in _TF_TARGETS: 205 jar_path = os.path.join( 206 self.root_dir, tf_dir, '..', 'filegroups', 'tradefed', 207 tf_target + '.jar') 208 if os.path.exists(jar_path): 209 prebuilt_jars.append(jar_path) 210 for gtf_dir in self.gtf_dirs: 211 for gtf_target in _GTF_TARGETS: 212 jar_path = os.path.join( 213 self.root_dir, gtf_dir, '..', 'filegroups', 214 'google-tradefed', gtf_target + '.jar') 215 if os.path.exists(jar_path): 216 prebuilt_jars.append(jar_path) 217 return prebuilt_jars 218 219 def _search_prebuilt_jars(self, name): 220 """Search tradefed prebuilt jar which has matched name. 221 222 Search if input name matched prebuilt tradefed jar. If matched, extract 223 the jar file to temp directly for later on test info handling. 224 225 Args: 226 name: A string of integration name as seen in tf's list configs. 227 228 Returns: 229 A list of test path. 230 """ 231 232 xml_path = 'config/{}.xml'.format(name) 233 test_files = [] 234 prebuilt_jars = self._get_prebuilt_jars() 235 logging.debug('Found prebuilt_jars=%s', prebuilt_jars) 236 for prebuilt_jar in prebuilt_jars: 237 with ZipFile(prebuilt_jar, 'r') as jar_file: 238 jar_contents = jar_file.namelist() 239 if xml_path in jar_contents: 240 extract_path = os.path.join( 241 self.temp_dir.name, os.path.basename(prebuilt_jar)) 242 if not os.path.exists(extract_path): 243 logging.debug('Extracting %s to %s', 244 prebuilt_jar, extract_path) 245 jar_file.extractall(extract_path) 246 test_files.append(os.path.join(extract_path, xml_path)) 247 return test_files 248 249 def _get_test_info(self, name, test_file, class_name): 250 """Find the test info matching the given test_file and class_name. 251 252 Args: 253 name: A string of integration name as seen in tf's list configs. 254 test_file: A string of test_file full path. 255 class_name: A string of user's input. 256 257 Returns: 258 A populated TestInfo namedtuple if test found, else None. 259 """ 260 match = _INT_NAME_RE.match(test_file) 261 if not match: 262 logging.error('Integration test outside config dir: %s', 263 test_file) 264 return None 265 int_name = match.group('int_name') 266 if int_name != name: 267 logging.debug('Input (%s) not valid integration name, ' 268 'did you mean: %s?', name, int_name) 269 return None 270 rel_config = os.path.relpath(test_file, self.root_dir) 271 filters = frozenset() 272 if class_name: 273 class_name, methods = test_finder_utils.split_methods(class_name) 274 test_filters = [] 275 if '.' in class_name: 276 test_filters.append(test_info.TestFilter(class_name, methods)) 277 else: 278 logging.debug('Looking up fully qualified class name for: %s.' 279 'Improve speed by using fully qualified names.', 280 class_name) 281 paths = test_finder_utils.find_class_file(self.root_dir, 282 class_name) 283 if not paths: 284 return None 285 for path in paths: 286 class_name = ( 287 test_finder_utils.get_fully_qualified_class_name( 288 path)) 289 test_filters.append(test_info.TestFilter( 290 class_name, methods)) 291 filters = frozenset(test_filters) 292 return test_info.TestInfo( 293 test_name=name, 294 test_runner=self._TEST_RUNNER, 295 build_targets=self._get_build_targets(rel_config), 296 data={constants.TI_REL_CONFIG: rel_config, 297 constants.TI_FILTER: filters}) 298 299 def find_int_test_by_path(self, path): 300 """Find the first test info matching the given path. 301 302 Strategy: 303 path_to_integration_file --> Resolve to INTEGRATION 304 # If the path is a dir, we return nothing. 305 path_to_dir_with_integration_files --> Return None 306 307 Args: 308 path: A string of the test's path. 309 310 Returns: 311 A list of populated TestInfo namedtuple if test found, else None 312 """ 313 path, _ = test_finder_utils.split_methods(path) 314 315 # Make sure we're looking for a config. 316 if not path.endswith('.xml'): 317 return None 318 319 # TODO: See if this can be generalized and shared with methods above 320 # create absolute path from cwd and remove symbolic links 321 path = os.path.realpath(path) 322 if not os.path.exists(path): 323 logging.debug('"%s": file not found!', path) 324 return None 325 int_dir = test_finder_utils.get_int_dir_from_path(path, 326 self.integration_dirs) 327 if int_dir: 328 rel_config = os.path.relpath(path, self.root_dir) 329 match = _INT_NAME_RE.match(rel_config) 330 if not match: 331 logging.error('Integration test outside config dir: %s', 332 rel_config) 333 return None 334 int_name = match.group('int_name') 335 return [test_info.TestInfo( 336 test_name=int_name, 337 test_runner=self._TEST_RUNNER, 338 build_targets=self._get_build_targets(rel_config), 339 data={constants.TI_REL_CONFIG: rel_config, 340 constants.TI_FILTER: frozenset()})] 341 return None 342