#!/usr/bin/env vpython3 # Copyright 2014 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Unit tests for instrumentation_test_instance.""" # pylint: disable=protected-access import collections import tempfile import unittest from pylib.base import base_test_result from pylib.instrumentation import instrumentation_test_instance import mock # pylint: disable=import-error _INSTRUMENTATION_TEST_INSTANCE_PATH = ( 'pylib.instrumentation.instrumentation_test_instance.%s') class InstrumentationTestInstanceTest(unittest.TestCase): @staticmethod def createTestInstance(): c = _INSTRUMENTATION_TEST_INSTANCE_PATH % 'InstrumentationTestInstance' # yapf: disable with mock.patch('%s._initializeApkAttributes' % c), ( mock.patch('%s._initializeDataDependencyAttributes' % c)), ( mock.patch('%s._initializeTestFilterAttributes' %c)), ( mock.patch('%s._initializeFlagAttributes' % c)), ( mock.patch('%s._initializeTestControlAttributes' % c)), ( mock.patch('%s._initializeTestCoverageAttributes' % c)), ( mock.patch('%s._initializeSkiaGoldAttributes' % c)): # yapf: enable return instrumentation_test_instance.InstrumentationTestInstance( mock.MagicMock(), mock.MagicMock(), lambda s: None) _FlagAttributesArgs = collections.namedtuple('_FlagAttributesArgs', [ 'command_line_flags', 'device_flags_file', 'strict_mode', 'use_apk_under_test_flags_file', 'coverage_dir' ]) def createFlagAttributesArgs(self, command_line_flags=None, device_flags_file=None, strict_mode=None, use_apk_under_test_flags_file=False, coverage_dir=None): return self._FlagAttributesArgs(command_line_flags, device_flags_file, strict_mode, use_apk_under_test_flags_file, coverage_dir) def test_initializeFlagAttributes_commandLineFlags(self): o = self.createTestInstance() args = self.createFlagAttributesArgs(command_line_flags=['--foo', '--bar']) o._initializeFlagAttributes(args) self.assertEqual(o._flags, ['--enable-test-intents', '--foo', '--bar']) def test_initializeFlagAttributes_deviceFlagsFile(self): o = self.createTestInstance() with tempfile.NamedTemporaryFile(mode='w') as flags_file: flags_file.write('\n'.join(['--foo', '--bar'])) flags_file.flush() args = self.createFlagAttributesArgs(device_flags_file=flags_file.name) o._initializeFlagAttributes(args) self.assertEqual(o._flags, ['--enable-test-intents', '--foo', '--bar']) def test_initializeFlagAttributes_strictModeOn(self): o = self.createTestInstance() args = self.createFlagAttributesArgs(strict_mode='on') o._initializeFlagAttributes(args) self.assertEqual(o._flags, ['--enable-test-intents', '--strict-mode=on']) def test_initializeFlagAttributes_strictModeOn_coverageOn(self): o = self.createTestInstance() args = self.createFlagAttributesArgs( strict_mode='on', coverage_dir='/coverage/dir') o._initializeFlagAttributes(args) self.assertEqual(o._flags, ['--enable-test-intents']) def test_initializeFlagAttributes_strictModeOff(self): o = self.createTestInstance() args = self.createFlagAttributesArgs(strict_mode='off') o._initializeFlagAttributes(args) self.assertEqual(o._flags, ['--enable-test-intents']) def testGetTests_noFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, { 'annotations': { 'Feature': {'value': ['Foo']}, 'MediumTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod2', }, { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_simpleGtestFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, ] o._test_filters = ['org.chromium.test.SampleTest.testMethod1'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_simpleGtestPositiveAndNegativeFilter(self): o = self.createTestInstance() raw_tests = [{ 'annotations': { 'Feature': { 'value': ['Foo'] } }, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': { 'SmallTest': None }, 'method': 'testMethod1', }, { 'annotations': { 'MediumTest': None }, 'method': 'testMethod2', }, ], }, { 'annotations': { 'Feature': { 'value': ['Foo'] } }, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [{ 'annotations': { 'SmallTest': None }, 'method': 'testMethod1', }], }] expected_tests = [ { 'annotations': { 'Feature': { 'value': ['Foo'] }, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, ] o._test_filters = [ 'org.chromium.test.SampleTest.*'\ '-org.chromium.test.SampleTest.testMethod2' ] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_multipleGtestPositiveAndNegativeFilter(self): o = self.createTestInstance() raw_tests = [{ 'annotations': { 'Feature': { 'value': ['Foo'] } }, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': { 'SmallTest': None }, 'method': 'testMethod1', }, { 'annotations': { 'MediumTest': None }, 'method': 'testMethod2', }, ], }, { 'annotations': { 'Feature': { 'value': ['Foo'] } }, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [{ 'annotations': { 'SmallTest': None }, 'method': 'testMethod1', }], }] expected_tests = [ { 'annotations': { 'Feature': { 'value': ['Foo'] }, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, ] o._test_filters = [ 'org.chromium.test.SampleTest*testMethod1', 'org.chromium.test.SampleTest.*'\ '-org.chromium.test.SampleTest.testMethod2' ] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_simpleGtestUnqualifiedNameFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, ] o._test_filters = ['SampleTest.testMethod1'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_parameterizedTestGtestFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'SmallTest': None}, 'method': 'testMethod1__sandboxed_mode', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1__sandboxed_mode', }, ] o._junit4_runner_class = 'J4Runner' o._test_filters = ['org.chromium.test.SampleTest.testMethod1'] actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_wildcardGtestFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._test_filters = ['org.chromium.test.SampleTest2.*'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_negativeGtestFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'MediumTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod2', }, { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._test_filters = ['*-org.chromium.test.SampleTest.testMethod1'] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_annotationFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Foo']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._annotations = [('SmallTest', None)] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_excludedAnnotationFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': { 'value': ['Foo'] }, 'MediumTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod2', }, ] o._excluded_annotations = [('SmallTest', None)] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_excludedDoNotReviveAnnotation(self): o = self.createTestInstance() raw_tests = [{ 'annotations': { 'Feature': { 'value': ['Foo'] } }, 'class': 'org.chromium.test.SampleTest', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': { 'DisabledTest': None, 'DoNotRevive': { 'reason': 'sample reason' }, }, 'method': 'testMethod1', }, { 'annotations': { 'FlakyTest': None, }, 'method': 'testMethod2', }, ], }, { 'annotations': { 'Feature': { 'value': ['Bar'] } }, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': { 'FlakyTest': None, 'DoNotRevive': { 'reason': 'sample reason' }, }, 'method': 'testMethod1', }, ], }, { 'annotations': { 'Feature': { 'value': ['Baz'] } }, 'class': 'org.chromium.test.SampleTest3', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': { 'FlakyTest': None, 'Manual': { 'message': 'sample message' }, }, 'method': 'testMethod1', }, ], }] expected_tests = [ { 'annotations': { 'Feature': { 'value': ['Foo'] }, 'FlakyTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod2', }, ] o._excluded_annotations = [('DoNotRevive', None), ('Manual', None)] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_annotationSimpleValueFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': { 'SmallTest': None, 'TestValue': '1', }, 'method': 'testMethod1', }, { 'annotations': { 'MediumTest': None, 'TestValue': '2', }, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': { 'SmallTest': None, 'TestValue': '3', }, 'method': 'testMethod1', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': { 'value': ['Foo'] }, 'SmallTest': None, 'TestValue': '1', }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod1', }, ] o._annotations = [('TestValue', '1')] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTests_annotationDictValueFilter(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': {'MediumTest': None}, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': {'value': ['Bar']}, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._annotations = [('Feature', 'Bar')] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGetTestName(self): test = { 'annotations': { 'RunWith': {'value': 'class J4Runner'}, 'SmallTest': {}, 'Test': {'expected': 'class org.junit.Test$None', 'timeout': '0'}, 'UiThreadTest': {}}, 'class': 'org.chromium.TestA', 'method': 'testSimple'} unqualified_class_test = { 'class': test['class'].split('.')[-1], 'method': test['method'] } self.assertEqual(instrumentation_test_instance.GetTestName(test, sep='.'), 'org.chromium.TestA.testSimple') self.assertEqual( instrumentation_test_instance.GetTestName(unqualified_class_test, sep='.'), 'TestA.testSimple') def testGetUniqueTestName(self): test = { 'annotations': { 'RunWith': {'value': 'class J4Runner'}, 'SmallTest': {}, 'Test': {'expected': 'class org.junit.Test$None', 'timeout': '0'}, 'UiThreadTest': {}}, 'class': 'org.chromium.TestA', 'flags': ['enable_features=abc'], 'method': 'testSimple'} self.assertEqual( instrumentation_test_instance.GetUniqueTestName(test, sep='.'), 'org.chromium.TestA.testSimple_with_enable_features=abc') def testGetTestNameWithoutParameterSuffix(self): test = { 'annotations': { 'RunWith': {'value': 'class J4Runner'}, 'SmallTest': {}, 'Test': {'expected': 'class org.junit.Test$None', 'timeout': '0'}, 'UiThreadTest': {}}, 'class': 'org.chromium.TestA__sandbox_mode', 'flags': 'enable_features=abc', 'method': 'testSimple'} unqualified_class_test = { 'class': test['class'].split('.')[-1], 'method': test['method'] } self.assertEqual( instrumentation_test_instance.GetTestNameWithoutParameterSuffix( test, sep='.'), 'org.chromium.TestA') self.assertEqual( instrumentation_test_instance.GetTestNameWithoutParameterSuffix( unqualified_class_test, sep='.'), 'TestA') def testGetTests_multipleAnnotationValuesRequested(self): o = self.createTestInstance() raw_tests = [ { 'annotations': {'Feature': {'value': ['Foo']}}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, { 'annotations': { 'Feature': {'value': ['Baz']}, 'MediumTest': None, }, 'method': 'testMethod2', }, ], }, { 'annotations': {'Feature': {'value': ['Bar']}}, 'class': 'org.chromium.test.SampleTest2', 'superclass': 'junit.framework.TestCase', 'methods': [ { 'annotations': {'SmallTest': None}, 'method': 'testMethod1', }, ], } ] expected_tests = [ { 'annotations': { 'Feature': { 'value': ['Baz'] }, 'MediumTest': None, }, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod2', }, { 'annotations': { 'Feature': { 'value': ['Bar'] }, 'SmallTest': None, }, 'class': 'org.chromium.test.SampleTest2', 'method': 'testMethod1', }, ] o._annotations = [('Feature', 'Bar'), ('Feature', 'Baz')] o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testGenerateTestResults_noStatus(self): results = instrumentation_test_instance.GenerateTestResults( None, None, [], 1000, None, None) self.assertEqual([], results) def testGenerateTestResults_testPassed(self): statuses = [ (1, { 'class': 'test.package.TestClass', 'test': 'testMethod', }), (0, { 'class': 'test.package.TestClass', 'test': 'testMethod', }), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.PASS, results[0].GetType()) def testGenerateTestResults_testFailed(self): statuses = [ (1, { 'class': 'test.package.TestClass', 'test': 'testMethod', }), (-2, { 'class': 'test.package.TestClass', 'test': 'testMethod', }), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) def testGenerateTestResults_testUnknownException(self): stacktrace = 'long\nstacktrace' statuses = [ (1, { 'class': 'test.package.TestClass', 'test': 'testMethod', }), (-1, { 'class': 'test.package.TestClass', 'test': 'testMethod', 'stack': stacktrace, }), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) self.assertEqual(stacktrace, results[0].GetLog()) def testGenerateTestResults_testSkipped_true(self): statuses = [ (1, { 'class': 'test.package.TestClass', 'test': 'testMethod', }), (-3, { 'class': 'test.package.TestClass', 'test': 'testMethod', }), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.SKIP, results[0].GetType()) def testGenerateTestResults_beforeClassFailure(self): stacktrace = 'long\nstacktrace' statuses = [ (1, { 'class': 'test.package.TestClass', 'test': 'null', }), (-2, { 'class': 'test.package.TestClass', 'test': 'null', 'stack': stacktrace, }), (1, { 'class': 'test.package.TestClass', 'test': 'testMethod1', }), (0, { 'class': 'test.package.TestClass', 'test': 'testMethod1', }), (1, { 'class': 'test.package.TestClass', 'test': 'testMethod2', }), (0, { 'class': 'test.package.TestClass', 'test': 'testMethod2', }), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None) self.assertEqual(2, len(results)) self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) self.assertEqual(base_test_result.ResultType.FAIL, results[1].GetType()) self.assertEqual(stacktrace, results[0].GetLog()) self.assertEqual(stacktrace, results[1].GetLog()) def testGenerateTestResults_afterClassFailure(self): stacktrace = 'long\nstacktrace' statuses = [ (1, { 'class': 'test.package.TestClass', 'test': 'testMethod1', }), (0, { 'class': 'test.package.TestClass', 'test': 'testMethod1', }), (1, { 'class': 'test.package.TestClass', 'test': 'testMethod2', }), (-3, { 'class': 'test.package.TestClass', 'test': 'testMethod2', }), (1, { 'class': 'test.package.TestClass', 'test': 'null', }), (-2, { 'class': 'test.package.TestClass', 'test': 'null', 'stack': stacktrace, }), ] results = instrumentation_test_instance.GenerateTestResults( None, None, statuses, 1000, None, None) self.assertEqual(2, len(results)) self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) self.assertEqual(base_test_result.ResultType.SKIP, results[1].GetType()) self.assertEqual(stacktrace, results[0].GetLog()) def testParameterizedCommandLineFlagsSwitches(self): o = self.createTestInstance() raw_tests = [{ 'annotations': { 'ParameterizedCommandLineFlags$Switches': { 'value': ['enable-features=abc', 'enable-features=def'] } }, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': { 'SmallTest': None }, 'method': 'testMethod1', }, { 'annotations': { 'MediumTest': None, 'ParameterizedCommandLineFlags$Switches': { 'value': ['enable-features=ghi', 'enable-features=jkl'] }, }, 'method': 'testMethod2', }, { 'annotations': { 'MediumTest': None, 'ParameterizedCommandLineFlags$Switches': { 'value': [] }, }, 'method': 'testMethod3', }, { 'annotations': { 'MediumTest': None, 'SkipCommandLineParameterization': None, }, 'method': 'testMethod4', }, ], }] expected_tests = [ { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--enable-features=abc', '--enable-features=def'], 'method': 'testMethod1' }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--enable-features=ghi', '--enable-features=jkl'], 'method': 'testMethod2' }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod3' }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod4' }, ] for i in range(4): expected_tests[i]['annotations'].update(raw_tests[0]['annotations']) expected_tests[i]['annotations'].update( raw_tests[0]['methods'][i]['annotations']) o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testParameterizedCommandLineFlags(self): o = self.createTestInstance() raw_tests = [{ 'annotations': { 'ParameterizedCommandLineFlags': { 'value': [ { 'ParameterizedCommandLineFlags$Switches': { 'value': [ 'enable-features=abc', 'force-fieldtrials=trial/group' ], } }, { 'ParameterizedCommandLineFlags$Switches': { 'value': [ 'enable-features=abc2', 'force-fieldtrials=trial/group2' ], } }, ], }, }, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': { 'SmallTest': None }, 'method': 'testMethod1', }, { 'annotations': { 'MediumTest': None, 'ParameterizedCommandLineFlags': { 'value': [{ 'ParameterizedCommandLineFlags$Switches': { 'value': ['enable-features=def'] } }], }, }, 'method': 'testMethod2', }, { 'annotations': { 'MediumTest': None, 'ParameterizedCommandLineFlags': { 'value': [], }, }, 'method': 'testMethod3', }, { 'annotations': { 'MediumTest': None, 'SkipCommandLineParameterization': None, }, 'method': 'testMethod4', }, ], }] expected_tests = [ { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--enable-features=abc', '--force-fieldtrials=trial/group'], 'method': 'testMethod1' }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--enable-features=def'], 'method': 'testMethod2' }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod3' }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'method': 'testMethod4' }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': [ '--enable-features=abc2', '--force-fieldtrials=trial/group2', ], 'method': 'testMethod1' }, ] for i in range(4): expected_tests[i]['annotations'].update(raw_tests[0]['annotations']) expected_tests[i]['annotations'].update( raw_tests[0]['methods'][i]['annotations']) expected_tests[4]['annotations'].update(raw_tests[0]['annotations']) expected_tests[4]['annotations'].update( raw_tests[0]['methods'][0]['annotations']) o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testDifferentCommandLineParameterizations(self): o = self.createTestInstance() raw_tests = [{ 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': { 'SmallTest': None, 'ParameterizedCommandLineFlags': { 'value': [ { 'ParameterizedCommandLineFlags$Switches': { 'value': ['a1', 'a2'], } }, ], }, }, 'method': 'testMethod2', }, { 'annotations': { 'SmallTest': None, 'ParameterizedCommandLineFlags$Switches': { 'value': ['b1', 'b2'], }, }, 'method': 'testMethod3', }, ], }] expected_tests = [ { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--a1', '--a2'], 'method': 'testMethod2' }, { 'annotations': {}, 'class': 'org.chromium.test.SampleTest', 'flags': ['--b1', '--b2'], 'method': 'testMethod3' }, ] for i in range(2): expected_tests[i]['annotations'].update( raw_tests[0]['methods'][i]['annotations']) o._junit4_runner_class = 'J4Runner' actual_tests = o.ProcessRawTests(raw_tests) self.assertEqual(actual_tests, expected_tests) def testMultipleCommandLineParameterizations_raises(self): o = self.createTestInstance() raw_tests = [ { 'annotations': { 'ParameterizedCommandLineFlags': { 'value': [ { 'ParameterizedCommandLineFlags$Switches': { 'value': [ 'enable-features=abc', 'force-fieldtrials=trial/group', ], } }, ], }, }, 'class': 'org.chromium.test.SampleTest', 'superclass': 'java.lang.Object', 'methods': [ { 'annotations': { 'SmallTest': None, 'ParameterizedCommandLineFlags$Switches': { 'value': [ 'enable-features=abc', 'force-fieldtrials=trial/group', ], }, }, 'method': 'testMethod1', }, ], }, ] o._junit4_runner_class = 'J4Runner' self.assertRaises( instrumentation_test_instance.CommandLineParameterizationException, o.ProcessRawTests, [raw_tests[0]]) if __name__ == '__main__': unittest.main(verbosity=2)