1#!/usr/bin/env vpython3 2# Copyright 2014 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Unit tests for instrumentation_test_instance.""" 7 8# pylint: disable=protected-access 9 10 11import collections 12import tempfile 13import unittest 14 15from pylib.base import base_test_result 16from pylib.instrumentation import instrumentation_test_instance 17 18import mock # pylint: disable=import-error 19 20_INSTRUMENTATION_TEST_INSTANCE_PATH = ( 21 'pylib.instrumentation.instrumentation_test_instance.%s') 22 23class InstrumentationTestInstanceTest(unittest.TestCase): 24 25 def setUp(self): 26 options = mock.Mock() 27 options.tool = '' 28 29 @staticmethod 30 def createTestInstance(): 31 c = _INSTRUMENTATION_TEST_INSTANCE_PATH % 'InstrumentationTestInstance' 32 # yapf: disable 33 with mock.patch('%s._initializeApkAttributes' % c), ( 34 mock.patch('%s._initializeDataDependencyAttributes' % c)), ( 35 mock.patch('%s._initializeTestFilterAttributes' %c)), ( 36 mock.patch('%s._initializeFlagAttributes' % c)), ( 37 mock.patch('%s._initializeTestControlAttributes' % c)), ( 38 mock.patch('%s._initializeTestCoverageAttributes' % c)), ( 39 mock.patch('%s._initializeSkiaGoldAttributes' % c)): 40 # yapf: enable 41 return instrumentation_test_instance.InstrumentationTestInstance( 42 mock.MagicMock(), mock.MagicMock(), lambda s: None) 43 44 _FlagAttributesArgs = collections.namedtuple('_FlagAttributesArgs', [ 45 'command_line_flags', 'device_flags_file', 'strict_mode', 46 'use_apk_under_test_flags_file', 'coverage_dir' 47 ]) 48 49 def createFlagAttributesArgs(self, 50 command_line_flags=None, 51 device_flags_file=None, 52 strict_mode=None, 53 use_apk_under_test_flags_file=False, 54 coverage_dir=None): 55 return self._FlagAttributesArgs(command_line_flags, device_flags_file, 56 strict_mode, use_apk_under_test_flags_file, 57 coverage_dir) 58 59 def test_initializeFlagAttributes_commandLineFlags(self): 60 o = self.createTestInstance() 61 args = self.createFlagAttributesArgs(command_line_flags=['--foo', '--bar']) 62 o._initializeFlagAttributes(args) 63 self.assertEqual(o._flags, ['--enable-test-intents', '--foo', '--bar']) 64 65 def test_initializeFlagAttributes_deviceFlagsFile(self): 66 o = self.createTestInstance() 67 with tempfile.NamedTemporaryFile(mode='w') as flags_file: 68 flags_file.write('\n'.join(['--foo', '--bar'])) 69 flags_file.flush() 70 71 args = self.createFlagAttributesArgs(device_flags_file=flags_file.name) 72 o._initializeFlagAttributes(args) 73 self.assertEqual(o._flags, ['--enable-test-intents', '--foo', '--bar']) 74 75 def test_initializeFlagAttributes_strictModeOn(self): 76 o = self.createTestInstance() 77 args = self.createFlagAttributesArgs(strict_mode='on') 78 o._initializeFlagAttributes(args) 79 self.assertEqual(o._flags, ['--enable-test-intents', '--strict-mode=on']) 80 81 def test_initializeFlagAttributes_strictModeOn_coverageOn(self): 82 o = self.createTestInstance() 83 args = self.createFlagAttributesArgs( 84 strict_mode='on', coverage_dir='/coverage/dir') 85 o._initializeFlagAttributes(args) 86 self.assertEqual(o._flags, ['--enable-test-intents']) 87 88 def test_initializeFlagAttributes_strictModeOff(self): 89 o = self.createTestInstance() 90 args = self.createFlagAttributesArgs(strict_mode='off') 91 o._initializeFlagAttributes(args) 92 self.assertEqual(o._flags, ['--enable-test-intents']) 93 94 def testGetTests_noFilter(self): 95 o = self.createTestInstance() 96 raw_tests = [ 97 { 98 'annotations': {'Feature': {'value': ['Foo']}}, 99 'class': 'org.chromium.test.SampleTest', 100 'superclass': 'java.lang.Object', 101 'methods': [ 102 { 103 'annotations': {'SmallTest': None}, 104 'method': 'testMethod1', 105 }, 106 { 107 'annotations': {'MediumTest': None}, 108 'method': 'testMethod2', 109 }, 110 ], 111 }, 112 { 113 'annotations': {'Feature': {'value': ['Bar']}}, 114 'class': 'org.chromium.test.SampleTest2', 115 'superclass': 'java.lang.Object', 116 'methods': [ 117 { 118 'annotations': {'SmallTest': None}, 119 'method': 'testMethod1', 120 }, 121 ], 122 } 123 ] 124 125 expected_tests = [ 126 { 127 'annotations': { 128 'Feature': {'value': ['Foo']}, 129 'SmallTest': None, 130 }, 131 'class': 'org.chromium.test.SampleTest', 132 'method': 'testMethod1', 133 'is_junit4': True, 134 }, 135 { 136 'annotations': { 137 'Feature': {'value': ['Foo']}, 138 'MediumTest': None, 139 }, 140 'class': 'org.chromium.test.SampleTest', 141 'method': 'testMethod2', 142 'is_junit4': True, 143 }, 144 { 145 'annotations': { 146 'Feature': {'value': ['Bar']}, 147 'SmallTest': None, 148 }, 149 'class': 'org.chromium.test.SampleTest2', 150 'method': 'testMethod1', 151 'is_junit4': True, 152 }, 153 ] 154 155 o._junit4_runner_class = 'J4Runner' 156 actual_tests = o.ProcessRawTests(raw_tests) 157 158 self.assertEqual(actual_tests, expected_tests) 159 160 def testGetTests_simpleGtestFilter(self): 161 o = self.createTestInstance() 162 raw_tests = [ 163 { 164 'annotations': {'Feature': {'value': ['Foo']}}, 165 'class': 'org.chromium.test.SampleTest', 166 'superclass': 'java.lang.Object', 167 'methods': [ 168 { 169 'annotations': {'SmallTest': None}, 170 'method': 'testMethod1', 171 }, 172 { 173 'annotations': {'MediumTest': None}, 174 'method': 'testMethod2', 175 }, 176 ], 177 } 178 ] 179 180 expected_tests = [ 181 { 182 'annotations': { 183 'Feature': {'value': ['Foo']}, 184 'SmallTest': None, 185 }, 186 'class': 'org.chromium.test.SampleTest', 187 'is_junit4': True, 188 'method': 'testMethod1', 189 }, 190 ] 191 192 o._test_filters = ['org.chromium.test.SampleTest.testMethod1'] 193 o._junit4_runner_class = 'J4Runner' 194 actual_tests = o.ProcessRawTests(raw_tests) 195 196 self.assertEqual(actual_tests, expected_tests) 197 198 def testGetTests_simpleGtestPositiveAndNegativeFilter(self): 199 o = self.createTestInstance() 200 raw_tests = [{ 201 'annotations': { 202 'Feature': { 203 'value': ['Foo'] 204 } 205 }, 206 'class': 207 'org.chromium.test.SampleTest', 208 'superclass': 209 'java.lang.Object', 210 'methods': [ 211 { 212 'annotations': { 213 'SmallTest': None 214 }, 215 'method': 'testMethod1', 216 }, 217 { 218 'annotations': { 219 'MediumTest': None 220 }, 221 'method': 'testMethod2', 222 }, 223 ], 224 }, { 225 'annotations': { 226 'Feature': { 227 'value': ['Foo'] 228 } 229 }, 230 'class': 231 'org.chromium.test.SampleTest2', 232 'superclass': 233 'java.lang.Object', 234 'methods': [{ 235 'annotations': { 236 'SmallTest': None 237 }, 238 'method': 'testMethod1', 239 }], 240 }] 241 242 expected_tests = [ 243 { 244 'annotations': { 245 'Feature': { 246 'value': ['Foo'] 247 }, 248 'SmallTest': None, 249 }, 250 'class': 'org.chromium.test.SampleTest', 251 'is_junit4': True, 252 'method': 'testMethod1', 253 }, 254 ] 255 256 o._test_filters = [ 257 'org.chromium.test.SampleTest.*'\ 258 '-org.chromium.test.SampleTest.testMethod2' 259 ] 260 o._junit4_runner_class = 'J4Runner' 261 actual_tests = o.ProcessRawTests(raw_tests) 262 263 self.assertEqual(actual_tests, expected_tests) 264 265 def testGetTests_multipleGtestPositiveAndNegativeFilter(self): 266 o = self.createTestInstance() 267 raw_tests = [{ 268 'annotations': { 269 'Feature': { 270 'value': ['Foo'] 271 } 272 }, 273 'class': 274 'org.chromium.test.SampleTest', 275 'superclass': 276 'java.lang.Object', 277 'methods': [ 278 { 279 'annotations': { 280 'SmallTest': None 281 }, 282 'method': 'testMethod1', 283 }, 284 { 285 'annotations': { 286 'MediumTest': None 287 }, 288 'method': 'testMethod2', 289 }, 290 ], 291 }, { 292 'annotations': { 293 'Feature': { 294 'value': ['Foo'] 295 } 296 }, 297 'class': 298 'org.chromium.test.SampleTest2', 299 'superclass': 300 'java.lang.Object', 301 'methods': [{ 302 'annotations': { 303 'SmallTest': None 304 }, 305 'method': 'testMethod1', 306 }], 307 }] 308 309 expected_tests = [ 310 { 311 'annotations': { 312 'Feature': { 313 'value': ['Foo'] 314 }, 315 'SmallTest': None, 316 }, 317 'class': 'org.chromium.test.SampleTest', 318 'is_junit4': True, 319 'method': 'testMethod1', 320 }, 321 ] 322 323 o._test_filters = [ 324 'org.chromium.test.SampleTest*testMethod1', 325 'org.chromium.test.SampleTest.*'\ 326 '-org.chromium.test.SampleTest.testMethod2' 327 ] 328 o._junit4_runner_class = 'J4Runner' 329 actual_tests = o.ProcessRawTests(raw_tests) 330 331 self.assertEqual(actual_tests, expected_tests) 332 333 def testGetTests_simpleGtestUnqualifiedNameFilter(self): 334 o = self.createTestInstance() 335 raw_tests = [ 336 { 337 'annotations': {'Feature': {'value': ['Foo']}}, 338 'class': 'org.chromium.test.SampleTest', 339 'superclass': 'java.lang.Object', 340 'methods': [ 341 { 342 'annotations': {'SmallTest': None}, 343 'method': 'testMethod1', 344 }, 345 { 346 'annotations': {'MediumTest': None}, 347 'method': 'testMethod2', 348 }, 349 ], 350 } 351 ] 352 353 expected_tests = [ 354 { 355 'annotations': { 356 'Feature': {'value': ['Foo']}, 357 'SmallTest': None, 358 }, 359 'class': 'org.chromium.test.SampleTest', 360 'is_junit4': True, 361 'method': 'testMethod1', 362 }, 363 ] 364 365 o._test_filters = ['SampleTest.testMethod1'] 366 o._junit4_runner_class = 'J4Runner' 367 actual_tests = o.ProcessRawTests(raw_tests) 368 369 self.assertEqual(actual_tests, expected_tests) 370 371 def testGetTests_parameterizedTestGtestFilter(self): 372 o = self.createTestInstance() 373 raw_tests = [ 374 { 375 'annotations': {'Feature': {'value': ['Foo']}}, 376 'class': 'org.chromium.test.SampleTest', 377 'superclass': 'java.lang.Object', 378 'methods': [ 379 { 380 'annotations': {'SmallTest': None}, 381 'method': 'testMethod1', 382 }, 383 { 384 'annotations': {'SmallTest': None}, 385 'method': 'testMethod1__sandboxed_mode', 386 }, 387 ], 388 }, 389 { 390 'annotations': {'Feature': {'value': ['Bar']}}, 391 'class': 'org.chromium.test.SampleTest2', 392 'superclass': 'java.lang.Object', 393 'methods': [ 394 { 395 'annotations': {'SmallTest': None}, 396 'method': 'testMethod1', 397 }, 398 ], 399 } 400 ] 401 402 expected_tests = [ 403 { 404 'annotations': { 405 'Feature': {'value': ['Foo']}, 406 'SmallTest': None, 407 }, 408 'class': 'org.chromium.test.SampleTest', 409 'method': 'testMethod1', 410 'is_junit4': True, 411 }, 412 { 413 'annotations': { 414 'Feature': {'value': ['Foo']}, 415 'SmallTest': None, 416 }, 417 'class': 'org.chromium.test.SampleTest', 418 'method': 'testMethod1__sandboxed_mode', 419 'is_junit4': True, 420 }, 421 ] 422 423 o._junit4_runner_class = 'J4Runner' 424 o._test_filters = ['org.chromium.test.SampleTest.testMethod1'] 425 actual_tests = o.ProcessRawTests(raw_tests) 426 427 self.assertEqual(actual_tests, expected_tests) 428 429 def testGetTests_wildcardGtestFilter(self): 430 o = self.createTestInstance() 431 raw_tests = [ 432 { 433 'annotations': {'Feature': {'value': ['Foo']}}, 434 'class': 'org.chromium.test.SampleTest', 435 'superclass': 'java.lang.Object', 436 'methods': [ 437 { 438 'annotations': {'SmallTest': None}, 439 'method': 'testMethod1', 440 }, 441 { 442 'annotations': {'MediumTest': None}, 443 'method': 'testMethod2', 444 }, 445 ], 446 }, 447 { 448 'annotations': {'Feature': {'value': ['Bar']}}, 449 'class': 'org.chromium.test.SampleTest2', 450 'superclass': 'java.lang.Object', 451 'methods': [ 452 { 453 'annotations': {'SmallTest': None}, 454 'method': 'testMethod1', 455 }, 456 ], 457 } 458 ] 459 460 expected_tests = [ 461 { 462 'annotations': { 463 'Feature': {'value': ['Bar']}, 464 'SmallTest': None, 465 }, 466 'class': 'org.chromium.test.SampleTest2', 467 'is_junit4': True, 468 'method': 'testMethod1', 469 }, 470 ] 471 472 o._test_filters = ['org.chromium.test.SampleTest2.*'] 473 o._junit4_runner_class = 'J4Runner' 474 actual_tests = o.ProcessRawTests(raw_tests) 475 476 self.assertEqual(actual_tests, expected_tests) 477 478 def testGetTests_negativeGtestFilter(self): 479 o = self.createTestInstance() 480 raw_tests = [ 481 { 482 'annotations': {'Feature': {'value': ['Foo']}}, 483 'class': 'org.chromium.test.SampleTest', 484 'superclass': 'java.lang.Object', 485 'methods': [ 486 { 487 'annotations': {'SmallTest': None}, 488 'method': 'testMethod1', 489 }, 490 { 491 'annotations': {'MediumTest': None}, 492 'method': 'testMethod2', 493 }, 494 ], 495 }, 496 { 497 'annotations': {'Feature': {'value': ['Bar']}}, 498 'class': 'org.chromium.test.SampleTest2', 499 'superclass': 'java.lang.Object', 500 'methods': [ 501 { 502 'annotations': {'SmallTest': None}, 503 'method': 'testMethod1', 504 }, 505 ], 506 } 507 ] 508 509 expected_tests = [ 510 { 511 'annotations': { 512 'Feature': {'value': ['Foo']}, 513 'MediumTest': None, 514 }, 515 'class': 'org.chromium.test.SampleTest', 516 'is_junit4': True, 517 'method': 'testMethod2', 518 }, 519 { 520 'annotations': { 521 'Feature': {'value': ['Bar']}, 522 'SmallTest': None, 523 }, 524 'class': 'org.chromium.test.SampleTest2', 525 'is_junit4': True, 526 'method': 'testMethod1', 527 }, 528 ] 529 530 o._test_filters = ['*-org.chromium.test.SampleTest.testMethod1'] 531 o._junit4_runner_class = 'J4Runner' 532 actual_tests = o.ProcessRawTests(raw_tests) 533 534 self.assertEqual(actual_tests, expected_tests) 535 536 def testGetTests_annotationFilter(self): 537 o = self.createTestInstance() 538 raw_tests = [ 539 { 540 'annotations': {'Feature': {'value': ['Foo']}}, 541 'class': 'org.chromium.test.SampleTest', 542 'superclass': 'java.lang.Object', 543 'methods': [ 544 { 545 'annotations': {'SmallTest': None}, 546 'method': 'testMethod1', 547 }, 548 { 549 'annotations': {'MediumTest': None}, 550 'method': 'testMethod2', 551 }, 552 ], 553 }, 554 { 555 'annotations': {'Feature': {'value': ['Bar']}}, 556 'class': 'org.chromium.test.SampleTest2', 557 'superclass': 'java.lang.Object', 558 'methods': [ 559 { 560 'annotations': {'SmallTest': None}, 561 'method': 'testMethod1', 562 }, 563 ], 564 } 565 ] 566 567 expected_tests = [ 568 { 569 'annotations': { 570 'Feature': {'value': ['Foo']}, 571 'SmallTest': None, 572 }, 573 'class': 'org.chromium.test.SampleTest', 574 'is_junit4': True, 575 'method': 'testMethod1', 576 }, 577 { 578 'annotations': { 579 'Feature': {'value': ['Bar']}, 580 'SmallTest': None, 581 }, 582 'class': 'org.chromium.test.SampleTest2', 583 'is_junit4': True, 584 'method': 'testMethod1', 585 }, 586 ] 587 588 o._annotations = [('SmallTest', None)] 589 o._junit4_runner_class = 'J4Runner' 590 actual_tests = o.ProcessRawTests(raw_tests) 591 592 self.assertEqual(actual_tests, expected_tests) 593 594 def testGetTests_excludedAnnotationFilter(self): 595 o = self.createTestInstance() 596 raw_tests = [ 597 { 598 'annotations': {'Feature': {'value': ['Foo']}}, 599 'class': 'org.chromium.test.SampleTest', 600 'superclass': 'junit.framework.TestCase', 601 'methods': [ 602 { 603 'annotations': {'SmallTest': None}, 604 'method': 'testMethod1', 605 }, 606 { 607 'annotations': {'MediumTest': None}, 608 'method': 'testMethod2', 609 }, 610 ], 611 }, 612 { 613 'annotations': {'Feature': {'value': ['Bar']}}, 614 'class': 'org.chromium.test.SampleTest2', 615 'superclass': 'junit.framework.TestCase', 616 'methods': [ 617 { 618 'annotations': {'SmallTest': None}, 619 'method': 'testMethod1', 620 }, 621 ], 622 } 623 ] 624 625 expected_tests = [ 626 { 627 'annotations': { 628 'Feature': { 629 'value': ['Foo'] 630 }, 631 'MediumTest': None, 632 }, 633 'class': 'org.chromium.test.SampleTest', 634 'is_junit4': True, 635 'method': 'testMethod2', 636 }, 637 ] 638 639 o._excluded_annotations = [('SmallTest', None)] 640 o._junit4_runner_class = 'J4Runner' 641 actual_tests = o.ProcessRawTests(raw_tests) 642 643 self.assertEqual(actual_tests, expected_tests) 644 645 def testGetTests_excludedDoNotReviveAnnotation(self): 646 o = self.createTestInstance() 647 raw_tests = [{ 648 'annotations': { 649 'Feature': { 650 'value': ['Foo'] 651 } 652 }, 653 'class': 654 'org.chromium.test.SampleTest', 655 'superclass': 656 'junit.framework.TestCase', 657 'methods': [ 658 { 659 'annotations': { 660 'DisabledTest': None, 661 'DoNotRevive': { 662 'reason': 'sample reason' 663 }, 664 }, 665 'method': 'testMethod1', 666 }, 667 { 668 'annotations': { 669 'FlakyTest': None, 670 }, 671 'method': 'testMethod2', 672 }, 673 ], 674 }, { 675 'annotations': { 676 'Feature': { 677 'value': ['Bar'] 678 } 679 }, 680 'class': 681 'org.chromium.test.SampleTest2', 682 'superclass': 683 'junit.framework.TestCase', 684 'methods': [ 685 { 686 'annotations': { 687 'FlakyTest': None, 688 'DoNotRevive': { 689 'reason': 'sample reason' 690 }, 691 }, 692 'method': 'testMethod1', 693 }, 694 ], 695 }, { 696 'annotations': { 697 'Feature': { 698 'value': ['Baz'] 699 } 700 }, 701 'class': 702 'org.chromium.test.SampleTest3', 703 'superclass': 704 'junit.framework.TestCase', 705 'methods': [ 706 { 707 'annotations': { 708 'FlakyTest': None, 709 'Manual': { 710 'message': 'sample message' 711 }, 712 }, 713 'method': 'testMethod1', 714 }, 715 ], 716 }] 717 718 expected_tests = [ 719 { 720 'annotations': { 721 'Feature': { 722 'value': ['Foo'] 723 }, 724 'FlakyTest': None, 725 }, 726 'class': 'org.chromium.test.SampleTest', 727 'is_junit4': True, 728 'method': 'testMethod2', 729 }, 730 ] 731 732 o._excluded_annotations = [('DoNotRevive', None), ('Manual', None)] 733 o._junit4_runner_class = 'J4Runner' 734 actual_tests = o.ProcessRawTests(raw_tests) 735 736 self.assertEqual(actual_tests, expected_tests) 737 738 def testGetTests_annotationSimpleValueFilter(self): 739 o = self.createTestInstance() 740 raw_tests = [ 741 { 742 'annotations': {'Feature': {'value': ['Foo']}}, 743 'class': 'org.chromium.test.SampleTest', 744 'superclass': 'junit.framework.TestCase', 745 'methods': [ 746 { 747 'annotations': { 748 'SmallTest': None, 749 'TestValue': '1', 750 }, 751 'method': 'testMethod1', 752 }, 753 { 754 'annotations': { 755 'MediumTest': None, 756 'TestValue': '2', 757 }, 758 'method': 'testMethod2', 759 }, 760 ], 761 }, 762 { 763 'annotations': {'Feature': {'value': ['Bar']}}, 764 'class': 'org.chromium.test.SampleTest2', 765 'superclass': 'junit.framework.TestCase', 766 'methods': [ 767 { 768 'annotations': { 769 'SmallTest': None, 770 'TestValue': '3', 771 }, 772 'method': 'testMethod1', 773 }, 774 ], 775 } 776 ] 777 778 expected_tests = [ 779 { 780 'annotations': { 781 'Feature': { 782 'value': ['Foo'] 783 }, 784 'SmallTest': None, 785 'TestValue': '1', 786 }, 787 'class': 'org.chromium.test.SampleTest', 788 'is_junit4': True, 789 'method': 'testMethod1', 790 }, 791 ] 792 793 o._annotations = [('TestValue', '1')] 794 o._junit4_runner_class = 'J4Runner' 795 actual_tests = o.ProcessRawTests(raw_tests) 796 797 self.assertEqual(actual_tests, expected_tests) 798 799 def testGetTests_annotationDictValueFilter(self): 800 o = self.createTestInstance() 801 raw_tests = [ 802 { 803 'annotations': {'Feature': {'value': ['Foo']}}, 804 'class': 'org.chromium.test.SampleTest', 805 'superclass': 'java.lang.Object', 806 'methods': [ 807 { 808 'annotations': {'SmallTest': None}, 809 'method': 'testMethod1', 810 }, 811 { 812 'annotations': {'MediumTest': None}, 813 'method': 'testMethod2', 814 }, 815 ], 816 }, 817 { 818 'annotations': {'Feature': {'value': ['Bar']}}, 819 'class': 'org.chromium.test.SampleTest2', 820 'superclass': 'java.lang.Object', 821 'methods': [ 822 { 823 'annotations': {'SmallTest': None}, 824 'method': 'testMethod1', 825 }, 826 ], 827 } 828 ] 829 830 expected_tests = [ 831 { 832 'annotations': { 833 'Feature': {'value': ['Bar']}, 834 'SmallTest': None, 835 }, 836 'class': 'org.chromium.test.SampleTest2', 837 'is_junit4': True, 838 'method': 'testMethod1', 839 }, 840 ] 841 842 o._annotations = [('Feature', 'Bar')] 843 o._junit4_runner_class = 'J4Runner' 844 actual_tests = o.ProcessRawTests(raw_tests) 845 846 self.assertEqual(actual_tests, expected_tests) 847 848 def testGetTestName(self): 849 test = { 850 'annotations': { 851 'RunWith': {'value': 'class J4Runner'}, 852 'SmallTest': {}, 853 'Test': {'expected': 'class org.junit.Test$None', 854 'timeout': '0'}, 855 'UiThreadTest': {}}, 856 'class': 'org.chromium.TestA', 857 'is_junit4': True, 858 'method': 'testSimple'} 859 unqualified_class_test = { 860 'class': test['class'].split('.')[-1], 861 'method': test['method'] 862 } 863 864 self.assertEqual(instrumentation_test_instance.GetTestName(test, sep='.'), 865 'org.chromium.TestA.testSimple') 866 self.assertEqual( 867 instrumentation_test_instance.GetTestName(unqualified_class_test, 868 sep='.'), 'TestA.testSimple') 869 870 def testGetUniqueTestName(self): 871 test = { 872 'annotations': { 873 'RunWith': {'value': 'class J4Runner'}, 874 'SmallTest': {}, 875 'Test': {'expected': 'class org.junit.Test$None', 'timeout': '0'}, 876 'UiThreadTest': {}}, 877 'class': 'org.chromium.TestA', 878 'flags': ['enable_features=abc'], 879 'is_junit4': True, 880 'method': 'testSimple'} 881 self.assertEqual( 882 instrumentation_test_instance.GetUniqueTestName(test, sep='.'), 883 'org.chromium.TestA.testSimple_with_enable_features=abc') 884 885 def testGetTestNameWithoutParameterPostfix(self): 886 test = { 887 'annotations': { 888 'RunWith': {'value': 'class J4Runner'}, 889 'SmallTest': {}, 890 'Test': {'expected': 'class org.junit.Test$None', 'timeout': '0'}, 891 'UiThreadTest': {}}, 892 'class': 'org.chromium.TestA__sandbox_mode', 893 'flags': 'enable_features=abc', 894 'is_junit4': True, 895 'method': 'testSimple'} 896 unqualified_class_test = { 897 'class': test['class'].split('.')[-1], 898 'method': test['method'] 899 } 900 self.assertEqual( 901 instrumentation_test_instance.GetTestNameWithoutParameterPostfix( 902 test, sep='.'), 'org.chromium.TestA') 903 self.assertEqual( 904 instrumentation_test_instance.GetTestNameWithoutParameterPostfix( 905 unqualified_class_test, sep='.'), 'TestA') 906 907 def testGetTests_multipleAnnotationValuesRequested(self): 908 o = self.createTestInstance() 909 raw_tests = [ 910 { 911 'annotations': {'Feature': {'value': ['Foo']}}, 912 'class': 'org.chromium.test.SampleTest', 913 'superclass': 'junit.framework.TestCase', 914 'methods': [ 915 { 916 'annotations': {'SmallTest': None}, 917 'method': 'testMethod1', 918 }, 919 { 920 'annotations': { 921 'Feature': {'value': ['Baz']}, 922 'MediumTest': None, 923 }, 924 'method': 'testMethod2', 925 }, 926 ], 927 }, 928 { 929 'annotations': {'Feature': {'value': ['Bar']}}, 930 'class': 'org.chromium.test.SampleTest2', 931 'superclass': 'junit.framework.TestCase', 932 'methods': [ 933 { 934 'annotations': {'SmallTest': None}, 935 'method': 'testMethod1', 936 }, 937 ], 938 } 939 ] 940 941 expected_tests = [ 942 { 943 'annotations': { 944 'Feature': { 945 'value': ['Baz'] 946 }, 947 'MediumTest': None, 948 }, 949 'class': 'org.chromium.test.SampleTest', 950 'is_junit4': True, 951 'method': 'testMethod2', 952 }, 953 { 954 'annotations': { 955 'Feature': { 956 'value': ['Bar'] 957 }, 958 'SmallTest': None, 959 }, 960 'class': 'org.chromium.test.SampleTest2', 961 'is_junit4': True, 962 'method': 'testMethod1', 963 }, 964 ] 965 966 o._annotations = [('Feature', 'Bar'), ('Feature', 'Baz')] 967 o._junit4_runner_class = 'J4Runner' 968 actual_tests = o.ProcessRawTests(raw_tests) 969 970 self.assertEqual(actual_tests, expected_tests) 971 972 def testGenerateTestResults_noStatus(self): 973 results = instrumentation_test_instance.GenerateTestResults( 974 None, None, [], 1000, None, None) 975 self.assertEqual([], results) 976 977 def testGenerateTestResults_testPassed(self): 978 statuses = [ 979 (1, { 980 'class': 'test.package.TestClass', 981 'test': 'testMethod', 982 }), 983 (0, { 984 'class': 'test.package.TestClass', 985 'test': 'testMethod', 986 }), 987 ] 988 results = instrumentation_test_instance.GenerateTestResults( 989 None, None, statuses, 1000, None, None) 990 self.assertEqual(1, len(results)) 991 self.assertEqual(base_test_result.ResultType.PASS, results[0].GetType()) 992 993 def testGenerateTestResults_testSkipped_true(self): 994 statuses = [ 995 (1, { 996 'class': 'test.package.TestClass', 997 'test': 'testMethod', 998 }), 999 (0, { 1000 'test_skipped': 'true', 1001 'class': 'test.package.TestClass', 1002 'test': 'testMethod', 1003 }), 1004 (0, { 1005 'class': 'test.package.TestClass', 1006 'test': 'testMethod', 1007 }), 1008 ] 1009 results = instrumentation_test_instance.GenerateTestResults( 1010 None, None, statuses, 1000, None, None) 1011 self.assertEqual(1, len(results)) 1012 self.assertEqual(base_test_result.ResultType.SKIP, results[0].GetType()) 1013 1014 def testGenerateTestResults_testSkipped_false(self): 1015 statuses = [ 1016 (1, { 1017 'class': 'test.package.TestClass', 1018 'test': 'testMethod', 1019 }), 1020 (0, { 1021 'test_skipped': 'false', 1022 }), 1023 (0, { 1024 'class': 'test.package.TestClass', 1025 'test': 'testMethod', 1026 }), 1027 ] 1028 results = instrumentation_test_instance.GenerateTestResults( 1029 None, None, statuses, 1000, None, None) 1030 self.assertEqual(1, len(results)) 1031 self.assertEqual(base_test_result.ResultType.PASS, results[0].GetType()) 1032 1033 def testGenerateTestResults_testFailed(self): 1034 statuses = [ 1035 (1, { 1036 'class': 'test.package.TestClass', 1037 'test': 'testMethod', 1038 }), 1039 (-2, { 1040 'class': 'test.package.TestClass', 1041 'test': 'testMethod', 1042 }), 1043 ] 1044 results = instrumentation_test_instance.GenerateTestResults( 1045 None, None, statuses, 1000, None, None) 1046 self.assertEqual(1, len(results)) 1047 self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) 1048 1049 def testGenerateTestResults_testUnknownException(self): 1050 stacktrace = 'long\nstacktrace' 1051 statuses = [ 1052 (1, { 1053 'class': 'test.package.TestClass', 1054 'test': 'testMethod', 1055 }), 1056 (-1, { 1057 'class': 'test.package.TestClass', 1058 'test': 'testMethod', 1059 'stack': stacktrace, 1060 }), 1061 ] 1062 results = instrumentation_test_instance.GenerateTestResults( 1063 None, None, statuses, 1000, None, None) 1064 self.assertEqual(1, len(results)) 1065 self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) 1066 self.assertEqual(stacktrace, results[0].GetLog()) 1067 1068 def testGenerateJUnitTestResults_testSkipped_true(self): 1069 statuses = [ 1070 (1, { 1071 'class': 'test.package.TestClass', 1072 'test': 'testMethod', 1073 }), 1074 (-3, { 1075 'class': 'test.package.TestClass', 1076 'test': 'testMethod', 1077 }), 1078 ] 1079 results = instrumentation_test_instance.GenerateTestResults( 1080 None, None, statuses, 1000, None, None) 1081 self.assertEqual(1, len(results)) 1082 self.assertEqual(base_test_result.ResultType.SKIP, results[0].GetType()) 1083 1084 def testParameterizedCommandLineFlagsSwitches(self): 1085 o = self.createTestInstance() 1086 raw_tests = [{ 1087 'annotations': { 1088 'ParameterizedCommandLineFlags$Switches': { 1089 'value': ['enable-features=abc', 'enable-features=def'] 1090 } 1091 }, 1092 'class': 1093 'org.chromium.test.SampleTest', 1094 'superclass': 1095 'java.lang.Object', 1096 'methods': [ 1097 { 1098 'annotations': { 1099 'SmallTest': None 1100 }, 1101 'method': 'testMethod1', 1102 }, 1103 { 1104 'annotations': { 1105 'MediumTest': None, 1106 'ParameterizedCommandLineFlags$Switches': { 1107 'value': ['enable-features=ghi', 'enable-features=jkl'] 1108 }, 1109 }, 1110 'method': 'testMethod2', 1111 }, 1112 { 1113 'annotations': { 1114 'MediumTest': None, 1115 'ParameterizedCommandLineFlags$Switches': { 1116 'value': [] 1117 }, 1118 }, 1119 'method': 'testMethod3', 1120 }, 1121 { 1122 'annotations': { 1123 'MediumTest': None, 1124 'SkipCommandLineParameterization': None, 1125 }, 1126 'method': 'testMethod4', 1127 }, 1128 ], 1129 }] 1130 1131 expected_tests = [ 1132 { 1133 'annotations': {}, 1134 'class': 'org.chromium.test.SampleTest', 1135 'flags': ['--enable-features=abc', '--enable-features=def'], 1136 'is_junit4': True, 1137 'method': 'testMethod1' 1138 }, 1139 { 1140 'annotations': {}, 1141 'class': 'org.chromium.test.SampleTest', 1142 'flags': ['--enable-features=ghi', '--enable-features=jkl'], 1143 'is_junit4': True, 1144 'method': 'testMethod2' 1145 }, 1146 { 1147 'annotations': {}, 1148 'class': 'org.chromium.test.SampleTest', 1149 'is_junit4': True, 1150 'method': 'testMethod3' 1151 }, 1152 { 1153 'annotations': {}, 1154 'class': 'org.chromium.test.SampleTest', 1155 'is_junit4': True, 1156 'method': 'testMethod4' 1157 }, 1158 ] 1159 for i in range(4): 1160 expected_tests[i]['annotations'].update(raw_tests[0]['annotations']) 1161 expected_tests[i]['annotations'].update( 1162 raw_tests[0]['methods'][i]['annotations']) 1163 1164 o._junit4_runner_class = 'J4Runner' 1165 actual_tests = o.ProcessRawTests(raw_tests) 1166 self.assertEqual(actual_tests, expected_tests) 1167 1168 def testParameterizedCommandLineFlags(self): 1169 o = self.createTestInstance() 1170 raw_tests = [{ 1171 'annotations': { 1172 'ParameterizedCommandLineFlags': { 1173 'value': [ 1174 { 1175 'ParameterizedCommandLineFlags$Switches': { 1176 'value': [ 1177 'enable-features=abc', 1178 'force-fieldtrials=trial/group' 1179 ], 1180 } 1181 }, 1182 { 1183 'ParameterizedCommandLineFlags$Switches': { 1184 'value': [ 1185 'enable-features=abc2', 1186 'force-fieldtrials=trial/group2' 1187 ], 1188 } 1189 }, 1190 ], 1191 }, 1192 }, 1193 'class': 1194 'org.chromium.test.SampleTest', 1195 'superclass': 1196 'java.lang.Object', 1197 'methods': [ 1198 { 1199 'annotations': { 1200 'SmallTest': None 1201 }, 1202 'method': 'testMethod1', 1203 }, 1204 { 1205 'annotations': { 1206 'MediumTest': None, 1207 'ParameterizedCommandLineFlags': { 1208 'value': [{ 1209 'ParameterizedCommandLineFlags$Switches': { 1210 'value': ['enable-features=def'] 1211 } 1212 }], 1213 }, 1214 }, 1215 'method': 'testMethod2', 1216 }, 1217 { 1218 'annotations': { 1219 'MediumTest': None, 1220 'ParameterizedCommandLineFlags': { 1221 'value': [], 1222 }, 1223 }, 1224 'method': 'testMethod3', 1225 }, 1226 { 1227 'annotations': { 1228 'MediumTest': None, 1229 'SkipCommandLineParameterization': None, 1230 }, 1231 'method': 'testMethod4', 1232 }, 1233 ], 1234 }] 1235 1236 expected_tests = [ 1237 { 1238 'annotations': {}, 1239 'class': 'org.chromium.test.SampleTest', 1240 'flags': 1241 ['--enable-features=abc', '--force-fieldtrials=trial/group'], 1242 'is_junit4': True, 1243 'method': 'testMethod1' 1244 }, 1245 { 1246 'annotations': {}, 1247 'class': 'org.chromium.test.SampleTest', 1248 'flags': ['--enable-features=def'], 1249 'is_junit4': True, 1250 'method': 'testMethod2' 1251 }, 1252 { 1253 'annotations': {}, 1254 'class': 'org.chromium.test.SampleTest', 1255 'is_junit4': True, 1256 'method': 'testMethod3' 1257 }, 1258 { 1259 'annotations': {}, 1260 'class': 'org.chromium.test.SampleTest', 1261 'is_junit4': True, 1262 'method': 'testMethod4' 1263 }, 1264 { 1265 'annotations': {}, 1266 'class': 1267 'org.chromium.test.SampleTest', 1268 'flags': [ 1269 '--enable-features=abc2', 1270 '--force-fieldtrials=trial/group2', 1271 ], 1272 'is_junit4': 1273 True, 1274 'method': 1275 'testMethod1' 1276 }, 1277 ] 1278 for i in range(4): 1279 expected_tests[i]['annotations'].update(raw_tests[0]['annotations']) 1280 expected_tests[i]['annotations'].update( 1281 raw_tests[0]['methods'][i]['annotations']) 1282 expected_tests[4]['annotations'].update(raw_tests[0]['annotations']) 1283 expected_tests[4]['annotations'].update( 1284 raw_tests[0]['methods'][0]['annotations']) 1285 1286 o._junit4_runner_class = 'J4Runner' 1287 actual_tests = o.ProcessRawTests(raw_tests) 1288 self.assertEqual(actual_tests, expected_tests) 1289 1290 def testDifferentCommandLineParameterizations(self): 1291 o = self.createTestInstance() 1292 raw_tests = [{ 1293 'annotations': {}, 1294 'class': 1295 'org.chromium.test.SampleTest', 1296 'superclass': 1297 'java.lang.Object', 1298 'methods': [ 1299 { 1300 'annotations': { 1301 'SmallTest': None, 1302 'ParameterizedCommandLineFlags': { 1303 'value': [ 1304 { 1305 'ParameterizedCommandLineFlags$Switches': { 1306 'value': ['a1', 'a2'], 1307 } 1308 }, 1309 ], 1310 }, 1311 }, 1312 'method': 'testMethod2', 1313 }, 1314 { 1315 'annotations': { 1316 'SmallTest': None, 1317 'ParameterizedCommandLineFlags$Switches': { 1318 'value': ['b1', 'b2'], 1319 }, 1320 }, 1321 'method': 'testMethod3', 1322 }, 1323 ], 1324 }] 1325 1326 expected_tests = [ 1327 { 1328 'annotations': {}, 1329 'class': 'org.chromium.test.SampleTest', 1330 'flags': ['--a1', '--a2'], 1331 'is_junit4': True, 1332 'method': 'testMethod2' 1333 }, 1334 { 1335 'annotations': {}, 1336 'class': 'org.chromium.test.SampleTest', 1337 'flags': ['--b1', '--b2'], 1338 'is_junit4': True, 1339 'method': 'testMethod3' 1340 }, 1341 ] 1342 for i in range(2): 1343 expected_tests[i]['annotations'].update( 1344 raw_tests[0]['methods'][i]['annotations']) 1345 1346 o._junit4_runner_class = 'J4Runner' 1347 actual_tests = o.ProcessRawTests(raw_tests) 1348 self.assertEqual(actual_tests, expected_tests) 1349 1350 def testMultipleCommandLineParameterizations_raises(self): 1351 o = self.createTestInstance() 1352 raw_tests = [ 1353 { 1354 'annotations': { 1355 'ParameterizedCommandLineFlags': { 1356 'value': [ 1357 { 1358 'ParameterizedCommandLineFlags$Switches': { 1359 'value': [ 1360 'enable-features=abc', 1361 'force-fieldtrials=trial/group', 1362 ], 1363 } 1364 }, 1365 ], 1366 }, 1367 }, 1368 'class': 1369 'org.chromium.test.SampleTest', 1370 'superclass': 1371 'java.lang.Object', 1372 'methods': [ 1373 { 1374 'annotations': { 1375 'SmallTest': None, 1376 'ParameterizedCommandLineFlags$Switches': { 1377 'value': [ 1378 'enable-features=abc', 1379 'force-fieldtrials=trial/group', 1380 ], 1381 }, 1382 }, 1383 'method': 'testMethod1', 1384 }, 1385 ], 1386 }, 1387 ] 1388 1389 o._junit4_runner_class = 'J4Runner' 1390 self.assertRaises( 1391 instrumentation_test_instance.CommandLineParameterizationException, 1392 o.ProcessRawTests, [raw_tests[0]]) 1393 1394 1395if __name__ == '__main__': 1396 unittest.main(verbosity=2) 1397