1# Copyright 2018, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Integration Finder class. 17""" 18 19# pylint: disable=line-too-long 20 21import copy 22import logging 23import os 24import re 25import tempfile 26import xml.etree.ElementTree as ElementTree 27 28from zipfile import ZipFile 29 30import atest_error 31import constants 32 33from test_finders import test_info 34from test_finders import test_finder_base 35from test_finders import test_finder_utils 36from test_runners import atest_tf_test_runner 37 38# Find integration name based on file path of integration config xml file. 39# Group matches "foo/bar" given "blah/res/config/foo/bar.xml from source code 40# res directory or "blah/config/foo/bar.xml from prebuilt jars. 41_INT_NAME_RE = re.compile(r'^.*\/config\/(?P<int_name>.*).xml$') 42_TF_TARGETS = frozenset(['tradefed', 'tradefed-contrib']) 43_GTF_TARGETS = frozenset(['google-tradefed', 'google-tradefed-contrib']) 44_CONTRIB_TARGETS = frozenset(['google-tradefed-contrib']) 45_TF_RES_DIRS = frozenset(['../res/config', 'res/config']) 46 47 48class TFIntegrationFinder(test_finder_base.TestFinderBase): 49 """Integration Finder class.""" 50 NAME = 'INTEGRATION' 51 _TEST_RUNNER = atest_tf_test_runner.AtestTradefedTestRunner.NAME 52 53 54 def __init__(self, module_info=None): 55 super().__init__() 56 self.root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 57 self.module_info = module_info 58 # TODO: Break this up into AOSP/google_tf integration finders. 59 self.tf_dirs, self.gtf_dirs = self._get_integration_dirs() 60 self.integration_dirs = self.tf_dirs + self.gtf_dirs 61 self.temp_dir = tempfile.TemporaryDirectory() 62 63 def _get_mod_paths(self, module_name): 64 """Return the paths of the given module name.""" 65 if self.module_info: 66 # Since aosp/801774 merged, the path of test configs have been 67 # changed to ../res/config. 68 if module_name in _CONTRIB_TARGETS: 69 mod_paths = self.module_info.get_paths(module_name) 70 return [os.path.join(path, res_path) for path in mod_paths 71 for res_path in _TF_RES_DIRS] 72 return self.module_info.get_paths(module_name) 73 return [] 74 75 def _get_integration_dirs(self): 76 """Get integration dirs from MODULE_INFO based on targets. 77 78 Returns: 79 A tuple of lists of strings of integration dir rel to repo root. 80 """ 81 tf_dirs = list(filter(None, [d for x in _TF_TARGETS for d in self._get_mod_paths(x)])) 82 gtf_dirs = list(filter(None, [d for x in _GTF_TARGETS for d in self._get_mod_paths(x)])) 83 return tf_dirs, gtf_dirs 84 85 def _get_build_targets(self, rel_config): 86 config_file = os.path.join(self.root_dir, rel_config) 87 xml_root = self._load_xml_file(config_file) 88 targets = test_finder_utils.get_targets_from_xml_root(xml_root, 89 self.module_info) 90 if self.gtf_dirs: 91 targets.add(constants.GTF_TARGET) 92 return frozenset(targets) 93 94 def _load_xml_file(self, path): 95 """Load an xml file with option to expand <include> tags 96 97 Args: 98 path: A string of path to xml file. 99 100 Returns: 101 An xml.etree.ElementTree.Element instance of the root of the tree. 102 """ 103 tree = ElementTree.parse(path) 104 root = tree.getroot() 105 self._load_include_tags(root) 106 return root 107 108 #pylint: disable=invalid-name 109 def _load_include_tags(self, root): 110 """Recursively expand in-place the <include> tags in a given xml tree. 111 112 Python xml libraries don't support our type of <include> tags. Logic 113 used below is modified version of the built-in ElementInclude logic 114 found here: 115 https://github.com/python/cpython/blob/2.7/Lib/xml/etree/ElementInclude.py 116 117 Args: 118 root: The root xml.etree.ElementTree.Element. 119 120 Returns: 121 An xml.etree.ElementTree.Element instance with 122 include tags expanded. 123 """ 124 i = 0 125 while i < len(root): 126 elem = root[i] 127 if elem.tag == 'include': 128 # expand included xml file 129 integration_name = elem.get('name') 130 if not integration_name: 131 logging.warning('skipping <include> tag with no "name" value') 132 continue 133 full_paths = self._search_integration_dirs(integration_name) 134 if not full_paths: 135 full_paths = self._search_prebuilt_jars(integration_name) 136 node = None 137 if full_paths: 138 node = self._load_xml_file(full_paths[0]) 139 if node is None: 140 raise atest_error.FatalIncludeError("can't load %r" % 141 integration_name) 142 node = copy.copy(node) 143 if elem.tail: 144 node.tail = (node.tail or "") + elem.tail 145 root[i] = node 146 i = i + 1 147 148 def _search_integration_dirs(self, name): 149 """Search integration dirs for name and return full path. 150 Args: 151 name: A string of integration name as seen in tf's list configs. 152 153 Returns: 154 A list of test path. 155 """ 156 test_files = [] 157 for integration_dir in self.integration_dirs: 158 abs_path = os.path.join(self.root_dir, integration_dir) 159 found_test_files = test_finder_utils.run_find_cmd( 160 test_finder_utils.FIND_REFERENCE_TYPE.INTEGRATION, 161 abs_path, name) 162 if found_test_files: 163 test_files.extend(found_test_files) 164 return test_files 165 166 def find_test_by_integration_name(self, name): 167 """Find the test info matching the given integration name. 168 169 Args: 170 name: A string of integration name as seen in tf's list configs. 171 172 Returns: 173 A populated TestInfo namedtuple if test found, else None 174 """ 175 class_name = None 176 if ':' in name: 177 name, class_name = name.split(':') 178 test_files = self._search_integration_dirs(name) 179 if not test_files: 180 # Check prebuilt jars if input name is in jars. 181 test_files = self._search_prebuilt_jars(name) 182 # Don't use names that simply match the path, 183 # must be the actual name used by TF to run the test. 184 t_infos = [] 185 for test_file in test_files: 186 t_info = self._get_test_info(name, test_file, class_name) 187 if t_info: 188 t_infos.append(t_info) 189 return t_infos 190 191 def _get_prebuilt_jars(self): 192 """Get prebuilt jars based on targets. 193 194 Returns: 195 A tuple of lists of strings of prebuilt jars. 196 """ 197 prebuilt_jars = [] 198 for tf_dir in self.tf_dirs: 199 for tf_target in _TF_TARGETS: 200 jar_path = os.path.join( 201 self.root_dir, tf_dir, '..', 'filegroups', 'tradefed', 202 tf_target + '.jar') 203 if os.path.exists(jar_path): 204 prebuilt_jars.append(jar_path) 205 for gtf_dir in self.gtf_dirs: 206 for gtf_target in _GTF_TARGETS: 207 jar_path = os.path.join( 208 self.root_dir, gtf_dir, '..', 'filegroups', 209 'google-tradefed', gtf_target + '.jar') 210 if os.path.exists(jar_path): 211 prebuilt_jars.append(jar_path) 212 return prebuilt_jars 213 214 def _search_prebuilt_jars(self, name): 215 """Search tradefed prebuilt jar which has matched name. 216 217 Search if input name matched prebuilt tradefed jar. If matched, extract 218 the jar file to temp directly for later on test info handling. 219 220 Args: 221 name: A string of integration name as seen in tf's list configs. 222 223 Returns: 224 A list of test path. 225 """ 226 227 xml_path = 'config/{}.xml'.format(name) 228 test_files = [] 229 prebuilt_jars = self._get_prebuilt_jars() 230 logging.debug('Found prebuilt_jars=%s', prebuilt_jars) 231 for prebuilt_jar in prebuilt_jars: 232 with ZipFile(prebuilt_jar, 'r') as jar_file: 233 jar_contents = jar_file.namelist() 234 if xml_path in jar_contents: 235 extract_path = os.path.join( 236 self.temp_dir.name, os.path.basename(prebuilt_jar)) 237 if not os.path.exists(extract_path): 238 logging.debug('Extracting %s to %s', 239 prebuilt_jar, extract_path) 240 jar_file.extractall(extract_path) 241 test_files.append(os.path.join(extract_path, xml_path)) 242 return test_files 243 244 def _get_test_info(self, name, test_file, class_name): 245 """Find the test info matching the given test_file and class_name. 246 247 Args: 248 name: A string of integration name as seen in tf's list configs. 249 test_file: A string of test_file full path. 250 class_name: A string of user's input. 251 252 Returns: 253 A populated TestInfo namedtuple if test found, else None. 254 """ 255 match = _INT_NAME_RE.match(test_file) 256 if not match: 257 logging.error('Integration test outside config dir: %s', 258 test_file) 259 return None 260 int_name = match.group('int_name') 261 if int_name != name: 262 logging.debug('Input (%s) not valid integration name, ' 263 'did you mean: %s?', name, int_name) 264 return None 265 rel_config = os.path.relpath(test_file, self.root_dir) 266 filters = frozenset() 267 if class_name: 268 class_name, methods = test_finder_utils.split_methods(class_name) 269 test_filters = [] 270 if '.' in class_name: 271 test_filters.append(test_info.TestFilter(class_name, methods)) 272 else: 273 logging.debug('Looking up fully qualified class name for: %s.' 274 'Improve speed by using fully qualified names.', 275 class_name) 276 paths = test_finder_utils.find_class_file(self.root_dir, 277 class_name) 278 if not paths: 279 return None 280 for path in paths: 281 class_name = ( 282 test_finder_utils.get_fully_qualified_class_name( 283 path)) 284 test_filters.append(test_info.TestFilter( 285 class_name, methods)) 286 filters = frozenset(test_filters) 287 return test_info.TestInfo( 288 test_name=name, 289 test_runner=self._TEST_RUNNER, 290 build_targets=self._get_build_targets(rel_config), 291 data={constants.TI_REL_CONFIG: rel_config, 292 constants.TI_FILTER: filters}) 293 294 def find_int_test_by_path(self, path): 295 """Find the first test info matching the given path. 296 297 Strategy: 298 path_to_integration_file --> Resolve to INTEGRATION 299 # If the path is a dir, we return nothing. 300 path_to_dir_with_integration_files --> Return None 301 302 Args: 303 path: A string of the test's path. 304 305 Returns: 306 A list of populated TestInfo namedtuple if test found, else None 307 """ 308 path, _ = test_finder_utils.split_methods(path) 309 310 # Make sure we're looking for a config. 311 if not path.endswith('.xml'): 312 return None 313 314 # TODO: See if this can be generalized and shared with methods above 315 # create absolute path from cwd and remove symbolic links 316 path = os.path.realpath(path) 317 if not os.path.exists(path): 318 logging.debug('"%s": file not found!', path) 319 return None 320 int_dir = test_finder_utils.get_int_dir_from_path(path, 321 self.integration_dirs) 322 if int_dir: 323 rel_config = os.path.relpath(path, self.root_dir) 324 match = _INT_NAME_RE.match(rel_config) 325 if not match: 326 logging.error('Integration test outside config dir: %s', 327 rel_config) 328 return None 329 int_name = match.group('int_name') 330 return [test_info.TestInfo( 331 test_name=int_name, 332 test_runner=self._TEST_RUNNER, 333 build_targets=self._get_build_targets(rel_config), 334 data={constants.TI_REL_CONFIG: rel_config, 335 constants.TI_FILTER: frozenset()})] 336 return None 337