• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The Abseil Authors.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Tests for test randomization."""
16
17import random
18import subprocess
19
20from absl import flags
21from absl.testing import _bazelize_command
22from absl.testing import absltest
23from absl.testing import parameterized
24from absl.testing.tests import absltest_env
25
26FLAGS = flags.FLAGS
27
28
29class TestOrderRandomizationTest(parameterized.TestCase):
30  """Integration tests: Runs a py_test binary with randomization.
31
32  This is done by setting flags and environment variables.
33  """
34
35  def setUp(self):
36    super(TestOrderRandomizationTest, self).setUp()
37    self._test_name = 'absl/testing/tests/absltest_randomization_testcase'
38
39  def _run_test(self, extra_argv, extra_env):
40    """Runs the py_test binary in a subprocess, with the given args or env.
41
42    Args:
43      extra_argv: extra args to pass to the test
44      extra_env: extra env vars to set when running the test
45
46    Returns:
47      (stdout, test_cases, exit_code) tuple of (str, list of strs, int).
48    """
49    env = absltest_env.inherited_env()
50    # If *this* test is being run with this flag, we don't want to
51    # automatically set it for all tests we run.
52    env.pop('TEST_RANDOMIZE_ORDERING_SEED', '')
53    if extra_env is not None:
54      env.update(extra_env)
55
56    command = (
57        [_bazelize_command.get_executable_path(self._test_name)] + extra_argv)
58    proc = subprocess.Popen(
59        args=command,
60        env=env,
61        stdout=subprocess.PIPE,
62        stderr=subprocess.STDOUT,
63        universal_newlines=True)
64
65    stdout, _ = proc.communicate()
66
67    test_lines = [l for l in stdout.splitlines() if l.startswith('class ')]
68    return stdout, test_lines, proc.wait()
69
70  def test_no_args(self):
71    output, tests, exit_code = self._run_test([], None)
72    self.assertEqual(0, exit_code, msg='command output: ' + output)
73    self.assertNotIn('Randomizing test order with seed:', output)
74    cases = ['class A test ' + t for t in ('A', 'B', 'C')]
75    self.assertEqual(cases, tests)
76
77  @parameterized.parameters(
78      {
79          'argv': ['--test_randomize_ordering_seed=random'],
80          'env': None,
81      },
82      {
83          'argv': [],
84          'env': {
85              'TEST_RANDOMIZE_ORDERING_SEED': 'random',
86          },
87      },)
88  def test_simple_randomization(self, argv, env):
89    output, tests, exit_code = self._run_test(argv, env)
90    self.assertEqual(0, exit_code, msg='command output: ' + output)
91    self.assertIn('Randomizing test order with seed: ', output)
92    cases = ['class A test ' + t for t in ('A', 'B', 'C')]
93    # This may come back in any order; we just know it'll be the same
94    # set of elements.
95    self.assertSameElements(cases, tests)
96
97  @parameterized.parameters(
98      {
99          'argv': ['--test_randomize_ordering_seed=1'],
100          'env': None,
101      },
102      {
103          'argv': [],
104          'env': {
105              'TEST_RANDOMIZE_ORDERING_SEED': '1'
106          },
107      },
108      {
109          'argv': [],
110          'env': {
111              'LATE_SET_TEST_RANDOMIZE_ORDERING_SEED': '1'
112          },
113      },
114  )
115  def test_fixed_seed(self, argv, env):
116    output, tests, exit_code = self._run_test(argv, env)
117    self.assertEqual(0, exit_code, msg='command output: ' + output)
118    self.assertIn('Randomizing test order with seed: 1', output)
119    # Even though we know the seed, we need to shuffle the tests here, since
120    # this behaves differently in Python2 vs Python3.
121    shuffled_cases = ['A', 'B', 'C']
122    random.Random(1).shuffle(shuffled_cases)
123    cases = ['class A test ' + t for t in shuffled_cases]
124    # We know what order this will come back for the random seed we've
125    # specified.
126    self.assertEqual(cases, tests)
127
128  @parameterized.parameters(
129      {
130          'argv': ['--test_randomize_ordering_seed=0'],
131          'env': {
132              'TEST_RANDOMIZE_ORDERING_SEED': 'random'
133          },
134      },
135      {
136          'argv': [],
137          'env': {
138              'TEST_RANDOMIZE_ORDERING_SEED': '0'
139          },
140      },)
141  def test_disabling_randomization(self, argv, env):
142    output, tests, exit_code = self._run_test(argv, env)
143    self.assertEqual(0, exit_code, msg='command output: ' + output)
144    self.assertNotIn('Randomizing test order with seed:', output)
145    cases = ['class A test ' + t for t in ('A', 'B', 'C')]
146    self.assertEqual(cases, tests)
147
148
149if __name__ == '__main__':
150  absltest.main()
151