Lines Matching +full:test +full:- +full:suite
3 # Use of this source code is governed by a BSD-style license that can be
48 print(' - Be run in the chroot.')
49 print(' - (not yet supported) be run after running ')
53 'au', 'bvt', 'bvt-cq', 'bvt-inline', 'calibration', 'paygen_au_beta',
67 - state:
77 For example, assume we have a test with JOB_RETRIES=5 and
85 - retry_max:
91 @var _max_retries: Maximum retry limit at suite level.
92 Regardless how many times each individual test
94 the suite can't exceed _max_retries.
106 jobs that are originally scheduled by the suite.
110 for the suite. Default to None, no max.
116 for job_id, test in initial_jobs_to_tests.items():
117 if test.job_retries > 0:
119 retry_max=test.job_retries)
121 logging.debug("Test %s has no retries", test.name)
125 """Add a newly-created job to the retry map.
129 the test if the job fails.
143 """Return whether maximum retry limit for a suite has been reached."""
167 retry_max=old_record['retry_max'] - 1)
168 self._max_retries -= 1
184 raise ValueError('Unexpected state transition: %s -> %s' %
195 - no retry map entry -> retry not required, no following retry
196 - has retry map entry:
197 - already retried -> has following retry
198 - has not retried
200 - retry_max == 0 --> the last retry job, no more retry
201 - retry_max > 0
202 - attempted, but has failed in scheduling a
203 following retry due to rpc error --> no more retry
204 - has not attempped --> has following retry if test failed.
223 a) The test was actually executed, meaning that if
228 c) The test requires retry, i.e. the job has an entry in the retry map.
230 Note that if a test has JOB_RETRIES=5, and the second time
256 a) The test requires retry, i.e. the job has an entry in the retry map.
269 logging.debug('suite max_retries reached, not retrying.')
279 logging.debug('test-level retries exhausted, not retrying')
309 """Create test jobs for a suite."""
330 @param tag: a string with which to tag jobs run in this suite.
331 @param builds: the builds on which we're running this suite.
332 @param board: the board on which we're running this suite.
334 @param max_runtime_mins: Maximum suite runtime, in minutes.
342 to add to each test being scheduled.
346 @param test_source_build: Build that contains the server-side test code.
377 def create_job(self, test, retry_for=None): argument
381 @param test: ControlData object for a test to run.
382 @param retry_for: If the to-be-created job is a retry for an
395 test_priority = max(self._priority, test.priority)
397 reboot_before = (model_attributes.RebootBefore.NEVER if test.fast
401 control_file=test.text,
405 test.name),
406 control_type=test.test_type.capitalize(),
407 meta_hosts=[self._board]*test.sync_count,
408 dependencies=self._create_job_deps(test),
409 keyvals=self._create_keyvals_for_test_job(test, retry_for),
414 run_reset=not test.fast,
416 synch_count=test.sync_count,
417 require_ssp=test.require_ssp)
419 test_obj.test_name = test.name
423 def _create_job_deps(self, test): argument
424 """Create job deps list for a test job.
431 job_deps = list(test.dependencies)
436 def _create_keyvals_for_test_job(self, test, retry_for=None): argument
437 """Create keyvals dict for creating a test job.
439 @param test: ControlData object for a test to run.
440 @param retry_for: If the to-be-created job is a retry for an
444 @returns: A keyvals dict for creating the test job.
449 constants.JOB_EXPERIMENTAL_KEY: test.experimental,
472 # Add suite job id to keyvals so tko parser can read it from keyval
508 @param run_prod_code: If true, the retrieved tests will run the test
509 code that lives in prod aka the test code
512 @param test_args: A dict of args to be seeded in test control file under
522 """Retrieve a test's control data.
527 @param test_name: Name of test to retrieve.
542 the search space to just this suite's control files.
554 for test in six.itervalues(tests):
555 test.require_ssp = False
562 Parses all ControlData objects with a SUITE tag and extracts all
563 defined suite names.
565 @param build: the build on which we're running this suite.
577 for test in find_and_parse_tests(cf_getter, predicate):
578 suites.update(test.suite_tag_parts)
583 """Returns predicate that gets the similarity based on a test's file
588 similarity between the test file name and the given test_file_pattern.
594 test file name and the given test_file_pattern.
602 """Returns predicate that matched based on a test's name.
605 and returns a tuple of (test name, ratio), where ratio is the similarity
606 between the test name and the given test_name.
608 @param test_name: the test name to base the predicate on.
610 (test name, ratio), where ratio is the similarity between the
611 test name and the given test_name.
623 ans returns True if the test attributes satisfy the given attribute
627 test, like 'system:all and interval:daily'.
629 @return a callable that takes a ControlData and returns True if the test
637 """Returns predicate that matches based on a test's file name pattern.
640 and returns True if the test's control file name matches the given
653 """Returns predicate that matches based on a test's name pattern.
656 and returns True if the test name matches the given regular expression.
659 test names.
668 """Returns predicate that matched based on a test's name.
671 and returns True if the test name is equal to |test_name|.
673 @param test_name: the test name to base the predicate on.
685 and returns a list of tuples of (suite name, ratio), where suite name
686 is each suite listed in the control file, and ratio is the similarity
687 between each suite and the given name.
689 @param name: the suite name to base the predicate on.
691 of (suite name, ratio), where suite name is each suite listed in
692 the control file, and ratio is the similarity between each suite
695 return lambda t: [(suite,
696 difflib.SequenceMatcher(a=suite, b=name).ratio())
697 for suite in t.suite_tag_parts] or [(None, 0)]
704 and returns True if the SUITE tag is present and contains |name|.
706 @param name: the suite name to base the predicate on.
708 ControlData object's suite member.
718 # currently hard-coded places to look for tests.
727 @param build: the build on which we're running this suite.
735 """Test predicate for non-experimental tests."""
753 this Suite.
755 the search space to just this suite's control files.
762 @param run_prod_code: If true, the suite will run the test code that
763 lives in prod aka the test code currently on the
766 @param test_args: A dict of args to be seeded in test control file.
773 on the TIME setting in control file, slowest test comes first.
775 logging.debug('Getting control file list for suite: %s', suite_name)
799 should be in this Suite. `name` is the key to be compared, e.g.,
800 a suite name or test name. `ratio` is a value between [0,1]
803 the search space to just this suite's control files.
806 @return list of top names that similar to the given test, sorted by
809 logging.debug('Getting control file list for suite: %s', suite_name)
813 for test in six.itervalues(tests):
814 ratios = predicate(test)
827 """Decorator for deprecated Suite static methods.
830 static methods on Suite.
835 warnings.warn('Calling method "%s" from Suite is deprecated' %
843 A suite of tests, defined by some predicate over control file variables.
851 this Suite.
852 @var _tag: a string with which to tag jobs run in this suite.
853 @var _builds: the builds on which we're running this suite.
895 @param tag: a string with which to tag jobs run in this suite.
896 @param builds: the builds on which we're running this suite.
897 @param board: the board on which we're running this suite.
903 This must be set if you want job_id of sub-jobs
905 @param max_runtime_mins: Maximum suite runtime, in minutes.
913 to add to each test being scheduled.
915 @param wait_for_results: Set to False to run the suite job without
916 waiting for test jobs to finish. Default is
922 @param max_retries: Maximum retry limit at suite level.
923 Regardless how many times each individual test
925 happening in the suite can't exceed _max_retries.
929 @param test_source_build: Build that contains the server-side test code.
989 def _schedule_test(self, record, test, retry_for=None): argument
990 """Schedule a single test and return the job.
992 Schedule a single test by creating a job, and then update relevant
995 Emits a TEST_NA status log entry if it failed to schedule the test due
996 to NoEligibleHostException or a non-existent board label.
998 Returns a frontend.Job object if the test is successfully scheduled.
999 If scheduling failed due to NoEligibleHostException or a non-existent
1004 @param test: ControlData for a test to run.
1005 @param retry_for: If we are scheduling a test to retry an
1011 msg = 'Scheduling %s' % test.name
1017 job = self._job_creator.create_job(test, retry_for=retry_for)
1022 # Treat a dependency on a non-existent board label the same as
1026 'Emitting TEST_NA.', test.name)
1027 Status('TEST_NA', test.name,
1028 'Skipping: test not supported on this board/pool.',
1041 self._jobs_to_tests[job.id] = test
1046 retry_count = (test.job_retries -
1076 # as part of a suite. Remove this hack once provision is separated
1077 # out in its own suite.
1079 for test in self.tests:
1080 scheduled_job = self._schedule_test(record, test)
1082 scheduled_test_names.append(test.name)
1091 logging.exception('Exception while scheduling suite')
1093 'Exception while scheduling suite').record_result(record)
1096 logging.debug("Initializing RetryHandler for suite %s.", self._tag)
1103 logging.info("Will not retry jobs from suite %s.", self._tag)
1108 """Bump up individual test retries to match suite retry options."""
1112 for test in tests:
1113 # We do honor if a test insists on JOB_RETRIES = 0.
1114 if test.job_retries is None:
1116 'Test %s did not request retries, but suite requires '
1119 test.name)
1120 test.job_retries = 1
1124 """Make a keyvals dict to write for scheduled test names.
1126 @param scheduled_test_names: A list of scheduled test name strings.
1188 """Return whether the suite is finished waiting for child jobs."""
1194 Handle a test job result.
1208 # TODO (crbug.com/751428): If the suite times out before a retry could
1219 Record a test job result.
1232 Retry a test job result.
1241 test = self._jobs_to_tests[result.id]
1244 # 1) in first try, test.fast=True.
1245 # 2) in second try, test will be run in normal mode, so reset
1246 # test.fast=False.
1247 test.fast = False
1249 record=record, test=test, retry_for=result.id)
1251 logging.error('Failed to schedule test: %s, Reason: %s',
1252 test.name, e)
1279 Abort all scheduled test jobs.
1288 Record provided job as a suite job keyval, for later referencing.
1294 job_id_owner = '%s-%s' % (job.id, job.owner)
1302 class Suite(_BaseSuite): class
1304 A suite of tests, defined by some predicate over control file variables.
1312 this Suite.
1313 @var _tag: a string with which to tag jobs run in this suite.
1314 @var _builds: the builds on which we're running this suite.
1327 # TODO(ayatane): These methods are kept on the Suite class for
1360 Create a Suite using a given predicate test filters.
1362 Uses supplied predicate(s) to instantiate a Suite. Looks for tests in
1368 representations of control files. A test will be
1369 included in suite if all callables in this list
1371 @param builds: the builds on which we're running this suite. It's a
1373 @param board: the board on which we're running this suite.
1377 @param name: name of suite. Defaults to 'ad_hoc_suite'
1378 @param run_prod_code: If true, the suite will run the tests that
1379 lives in prod aka the test code currently on the
1381 @param **dargs: Any other Suite constructor parameters, as described
1382 in Suite.__init__ docstring.
1383 @return a Suite instance.
1400 Create a Suite using a predicate based on the SUITE control file var.
1402 Makes a predicate based on |name| and uses it to instantiate a Suite
1407 @param name: a value of the SUITE control file variable to search for.
1408 @param builds: the builds on which we're running this suite. It's a
1410 @param board: the board on which we're running this suite.
1414 @param **dargs: Any other Suite constructor parameters, as described
1415 in Suite.__init__ docstring.
1416 @return a Suite instance.
1460 representations of control files. A test will be
1461 included in suite if all callables in this list
1463 @param tag: a string with which to tag jobs run in this suite.
1464 @param builds: the builds on which we're running this suite.
1465 @param board: the board on which we're running this suite.
1471 @param run_prod_code: If true, the suite will run the test code that
1472 lives in prod aka the test code currently on the
1475 This must be set if you want job_id of sub-jobs
1477 @param max_runtime_mins: Maximum suite runtime, in minutes.
1485 to add to each test being scheduled.
1487 @param wait_for_results: Set to False to run the suite job without
1488 waiting for test jobs to finish. Default is
1494 @param max_retries: Maximum retry limit at suite level.
1495 Regardless how many times each individual test
1497 happening in the suite can't exceed _max_retries.
1501 @param test_source_build: Build that contains the server-side test code.
1505 test that will be actually ran.
1519 super(Suite, self).__init__(
1548 A suite for provisioning DUTs.
1570 @param tag: a string with which to tag jobs run in this suite.
1571 @param builds: the builds on which we're running this suite.
1572 @param board: the board on which we're running this suite.
1577 is no cap, a test is created for each eligible host.
1580 test that will be actually ran.
1581 @param test_source_build: Build that contains the server-side test code.
1598 logging.info('Provision suite is disabled on suites pool')
1633 Load and return the dummy pass test.
1635 @param builds: the builds on which we're running this suite.
1639 test that will be actually ran.
1640 @param test_source_build: Build that contains the server-side test code.
1660 Predicates are functions that take a test control data object and
1661 return True of that test is to be included. The returned
1710 """Abstract base class for reporting test results.
1712 Usually, this is used to report test failures.
1717 """Report test result.
1724 """Class that emails based on test failures."""
1726 def __init__(self, suite, bug_template=None): argument
1727 self._suite = suite
1733 @param result: Status instance for a test job.
1737 # Such dependency can cause issue to any module tries to import suite.py
1738 # without building site-packages first. Since the reporting modules are
1752 """Get BugTemplate for test job.
1756 filing options for failures in this suite.
1760 # Such dependency can cause issue to any module tries to import suite.py
1761 # without building site-packages first. Since the reporting modules are
1767 # Try to merge with bug template in test control file.
1774 # Test control file does not have bug template defined.
1785 # issue to any module tries to import suite.py without
1786 # building site-packages first. Since the reporting