• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2# Copyright 2014 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import json
7import logging
8import os
9import shutil
10import sys
11import tempfile
12import unittest
13from unittest import mock
14
15import six
16
17import common_merge_script_tests
18import standard_gtest_merge
19
20# gtest json output for successfully finished shard #0.
21GOOD_GTEST_JSON_0 = {
22    'all_tests': [
23        'AlignedMemoryTest.DynamicAllocation',
24        'AlignedMemoryTest.ScopedDynamicAllocation',
25        'AlignedMemoryTest.StackAlignment',
26        'AlignedMemoryTest.StaticAlignment',
27    ],
28    'disabled_tests': [
29        'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
30        'FileTest.TouchGetInfo',
31        'MessageLoopTestTypeDefault.EnsureDeletion',
32    ],
33    'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
34    'per_iteration_data': [{
35        'AlignedMemoryTest.DynamicAllocation': [{
36            'elapsed_time_ms': 0,
37            'losless_snippet': True,
38            'output_snippet': 'blah\\n',
39            'output_snippet_base64': 'YmxhaAo=',
40            'status': 'SUCCESS',
41        }],
42        'AlignedMemoryTest.ScopedDynamicAllocation': [{
43            'elapsed_time_ms': 0,
44            'losless_snippet': True,
45            'output_snippet': 'blah\\n',
46            'output_snippet_base64': 'YmxhaAo=',
47            'status': 'SUCCESS',
48        }],
49    }],
50    'test_locations': {
51        'AlignedMemoryTest.DynamicAllocation': {
52            'file': 'foo/bar/allocation_test.cc',
53            'line': 123,
54        },
55        'AlignedMemoryTest.ScopedDynamicAllocation': {
56            'file': 'foo/bar/allocation_test.cc',
57            'line': 456,
58        },
59        # This is a test from a different shard, but this happens in practice
60        # and we should not fail if information is repeated.
61        'AlignedMemoryTest.StaticAlignment': {
62            'file': 'foo/bar/allocation_test.cc',
63            'line': 12,
64        },
65    },
66}
67
68# gtest json output for successfully finished shard #1.
69GOOD_GTEST_JSON_1 = {
70    'all_tests': [
71        'AlignedMemoryTest.DynamicAllocation',
72        'AlignedMemoryTest.ScopedDynamicAllocation',
73        'AlignedMemoryTest.StackAlignment',
74        'AlignedMemoryTest.StaticAlignment',
75    ],
76    'disabled_tests': [
77        'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
78        'FileTest.TouchGetInfo',
79        'MessageLoopTestTypeDefault.EnsureDeletion',
80    ],
81    'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
82    'per_iteration_data': [{
83        'AlignedMemoryTest.StackAlignment': [{
84            'elapsed_time_ms': 0,
85            'losless_snippet': True,
86            'output_snippet': 'blah\\n',
87            'output_snippet_base64': 'YmxhaAo=',
88            'status': 'SUCCESS',
89        }],
90        'AlignedMemoryTest.StaticAlignment': [{
91            'elapsed_time_ms': 0,
92            'losless_snippet': True,
93            'output_snippet': 'blah\\n',
94            'output_snippet_base64': 'YmxhaAo=',
95            'status': 'SUCCESS',
96        }],
97    }],
98    'test_locations': {
99        'AlignedMemoryTest.StackAlignment': {
100            'file': 'foo/bar/allocation_test.cc',
101            'line': 789,
102        },
103        'AlignedMemoryTest.StaticAlignment': {
104            'file': 'foo/bar/allocation_test.cc',
105            'line': 12,
106        },
107    },
108}
109
110TIMED_OUT_GTEST_JSON_1 = {
111    'disabled_tests': [],
112    'global_tags': [],
113    'all_tests': [
114        'AlignedMemoryTest.DynamicAllocation',
115        'AlignedMemoryTest.ScopedDynamicAllocation',
116        'AlignedMemoryTest.StackAlignment',
117        'AlignedMemoryTest.StaticAlignment',
118    ],
119    'per_iteration_data': [{
120        'AlignedMemoryTest.StackAlignment': [{
121            'elapsed_time_ms': 54000,
122            'losless_snippet': True,
123            'output_snippet': 'timed out',
124            'output_snippet_base64': '',
125            'status': 'FAILURE',
126        }],
127        'AlignedMemoryTest.StaticAlignment': [{
128            'elapsed_time_ms': 0,
129            'losless_snippet': True,
130            'output_snippet': '',
131            'output_snippet_base64': '',
132            'status': 'NOTRUN',
133        }],
134    }],
135    'test_locations': {
136        'AlignedMemoryTest.StackAlignment': {
137            'file': 'foo/bar/allocation_test.cc',
138            'line': 789,
139        },
140        'AlignedMemoryTest.StaticAlignment': {
141            'file': 'foo/bar/allocation_test.cc',
142            'line': 12,
143        },
144    },
145}
146
147# GOOD_GTEST_JSON_0 and GOOD_GTEST_JSON_1 merged.
148GOOD_GTEST_JSON_MERGED = {
149    'all_tests': [
150        'AlignedMemoryTest.DynamicAllocation',
151        'AlignedMemoryTest.ScopedDynamicAllocation',
152        'AlignedMemoryTest.StackAlignment',
153        'AlignedMemoryTest.StaticAlignment',
154    ],
155    'disabled_tests': [
156        'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
157        'FileTest.TouchGetInfo',
158        'MessageLoopTestTypeDefault.EnsureDeletion',
159    ],
160    'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
161    'missing_shards': [],
162    'per_iteration_data': [{
163        'AlignedMemoryTest.DynamicAllocation': [{
164            'elapsed_time_ms': 0,
165            'losless_snippet': True,
166            'output_snippet': 'blah\\n',
167            'output_snippet_base64': 'YmxhaAo=',
168            'status': 'SUCCESS',
169        }],
170        'AlignedMemoryTest.ScopedDynamicAllocation': [{
171            'elapsed_time_ms': 0,
172            'losless_snippet': True,
173            'output_snippet': 'blah\\n',
174            'output_snippet_base64': 'YmxhaAo=',
175            'status': 'SUCCESS',
176        }],
177        'AlignedMemoryTest.StackAlignment': [{
178            'elapsed_time_ms': 0,
179            'losless_snippet': True,
180            'output_snippet': 'blah\\n',
181            'output_snippet_base64': 'YmxhaAo=',
182            'status': 'SUCCESS',
183        }],
184        'AlignedMemoryTest.StaticAlignment': [{
185            'elapsed_time_ms': 0,
186            'losless_snippet': True,
187            'output_snippet': 'blah\\n',
188            'output_snippet_base64': 'YmxhaAo=',
189            'status': 'SUCCESS',
190        }],
191    }],
192    'swarming_summary': {
193        u'shards': [{
194            u'state': u'COMPLETED',
195            u'outputs_ref': {
196                u'view_url': u'blah',
197            },
198        }],
199    },
200    'test_locations': {
201        'AlignedMemoryTest.StackAlignment': {
202            'file': 'foo/bar/allocation_test.cc',
203            'line': 789,
204        },
205        'AlignedMemoryTest.StaticAlignment': {
206            'file': 'foo/bar/allocation_test.cc',
207            'line': 12,
208        },
209        'AlignedMemoryTest.DynamicAllocation': {
210            'file': 'foo/bar/allocation_test.cc',
211            'line': 123,
212        },
213        'AlignedMemoryTest.ScopedDynamicAllocation': {
214            'file': 'foo/bar/allocation_test.cc',
215            'line': 456,
216        },
217    },
218}
219
220# Only shard #1 finished. UNRELIABLE_RESULTS is set.
221BAD_GTEST_JSON_ONLY_1_SHARD = {
222    'all_tests': [
223        'AlignedMemoryTest.DynamicAllocation',
224        'AlignedMemoryTest.ScopedDynamicAllocation',
225        'AlignedMemoryTest.StackAlignment',
226        'AlignedMemoryTest.StaticAlignment',
227    ],
228    'disabled_tests': [
229        'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
230        'FileTest.TouchGetInfo',
231        'MessageLoopTestTypeDefault.EnsureDeletion',
232    ],
233    'global_tags': [
234        'CPU_64_BITS',
235        'MODE_DEBUG',
236        'OS_LINUX',
237        'OS_POSIX',
238        'UNRELIABLE_RESULTS',
239    ],
240    'missing_shards': [0],
241    'per_iteration_data': [{
242        'AlignedMemoryTest.StackAlignment': [{
243            'elapsed_time_ms': 0,
244            'losless_snippet': True,
245            'output_snippet': 'blah\\n',
246            'output_snippet_base64': 'YmxhaAo=',
247            'status': 'SUCCESS',
248        }],
249        'AlignedMemoryTest.StaticAlignment': [{
250            'elapsed_time_ms': 0,
251            'losless_snippet': True,
252            'output_snippet': 'blah\\n',
253            'output_snippet_base64': 'YmxhaAo=',
254            'status': 'SUCCESS',
255        }],
256    }],
257    'test_locations': {
258        'AlignedMemoryTest.StackAlignment': {
259            'file': 'foo/bar/allocation_test.cc',
260            'line': 789,
261        },
262        'AlignedMemoryTest.StaticAlignment': {
263            'file': 'foo/bar/allocation_test.cc',
264            'line': 12,
265        },
266    },
267}
268
269# GOOD_GTEST_JSON_0 and TIMED_OUT_GTEST_JSON_1 merged.
270TIMED_OUT_GTEST_JSON_MERGED = {
271    'all_tests': [
272        'AlignedMemoryTest.DynamicAllocation',
273        'AlignedMemoryTest.ScopedDynamicAllocation',
274        'AlignedMemoryTest.StackAlignment',
275        'AlignedMemoryTest.StaticAlignment',
276    ],
277    'disabled_tests': [
278        'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
279        'FileTest.TouchGetInfo',
280        'MessageLoopTestTypeDefault.EnsureDeletion',
281    ],
282    'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
283    'missing_shards': [],
284    'per_iteration_data': [{
285        'AlignedMemoryTest.DynamicAllocation': [{
286            'elapsed_time_ms': 0,
287            'losless_snippet': True,
288            'output_snippet': 'blah\\n',
289            'output_snippet_base64': 'YmxhaAo=',
290            'status': 'SUCCESS',
291        }],
292        'AlignedMemoryTest.ScopedDynamicAllocation': [{
293            'elapsed_time_ms': 0,
294            'losless_snippet': True,
295            'output_snippet': 'blah\\n',
296            'output_snippet_base64': 'YmxhaAo=',
297            'status': 'SUCCESS',
298        }],
299        'AlignedMemoryTest.StackAlignment': [{
300            'elapsed_time_ms': 54000,
301            'losless_snippet': True,
302            'output_snippet': 'timed out',
303            'output_snippet_base64': '',
304            'status': 'FAILURE',
305        }],
306        'AlignedMemoryTest.StaticAlignment': [{
307            'elapsed_time_ms': 0,
308            'losless_snippet': True,
309            'output_snippet': '',
310            'output_snippet_base64': '',
311            'status': 'NOTRUN',
312        }],
313    }],
314    'swarming_summary': {
315        u'shards': [
316            {
317                u'state': u'COMPLETED',
318            },
319            {
320                u'state': u'TIMED_OUT',
321            },
322        ],
323    },
324    'test_locations': {
325        'AlignedMemoryTest.StackAlignment': {
326            'file': 'foo/bar/allocation_test.cc',
327            'line': 789,
328        },
329        'AlignedMemoryTest.StaticAlignment': {
330            'file': 'foo/bar/allocation_test.cc',
331            'line': 12,
332        },
333        'AlignedMemoryTest.DynamicAllocation': {
334            'file': 'foo/bar/allocation_test.cc',
335            'line': 123,
336        },
337        'AlignedMemoryTest.ScopedDynamicAllocation': {
338            'file': 'foo/bar/allocation_test.cc',
339            'line': 456,
340        },
341    },
342}
343
344
345class _StandardGtestMergeTest(unittest.TestCase):
346
347  def setUp(self):
348    self.temp_dir = tempfile.mkdtemp()
349
350  def tearDown(self):
351    shutil.rmtree(self.temp_dir)
352
353  def _write_temp_file(self, path, content):
354    abs_path = os.path.join(self.temp_dir, path.replace('/', os.sep))
355    if not os.path.exists(os.path.dirname(abs_path)):
356      os.makedirs(os.path.dirname(abs_path))
357    with open(abs_path, 'w') as f:
358      if isinstance(content, dict):
359        json.dump(content, f)
360      else:
361        assert isinstance(content, str)
362        f.write(content)
363    return abs_path
364
365
366class LoadShardJsonTest(_StandardGtestMergeTest):
367
368  def test_double_digit_jsons(self):
369    jsons_to_merge = []
370    for i in range(15):
371      json_dir = os.path.join(self.temp_dir, str(i))
372      json_path = os.path.join(json_dir, 'output.json')
373      if not os.path.exists(json_dir):
374        os.makedirs(json_dir)
375      with open(json_path, 'w') as f:
376        json.dump({'all_tests': ['LoadShardJsonTest.test%d' % i]}, f)
377      jsons_to_merge.append(json_path)
378
379    content, err = standard_gtest_merge.load_shard_json(0, None, jsons_to_merge)
380    self.assertEqual({'all_tests': ['LoadShardJsonTest.test0']}, content)
381    self.assertIsNone(err)
382
383    content, err = standard_gtest_merge.load_shard_json(12, None,
384                                                        jsons_to_merge)
385    self.assertEqual({'all_tests': ['LoadShardJsonTest.test12']}, content)
386    self.assertIsNone(err)
387
388  def test_double_task_id_jsons(self):
389    jsons_to_merge = []
390    for i in range(15):
391      json_dir = os.path.join(self.temp_dir, 'deadbeef%d' % i)
392      json_path = os.path.join(json_dir, 'output.json')
393      if not os.path.exists(json_dir):
394        os.makedirs(json_dir)
395      with open(json_path, 'w') as f:
396        json.dump({'all_tests': ['LoadShardJsonTest.test%d' % i]}, f)
397      jsons_to_merge.append(json_path)
398
399    content, err = standard_gtest_merge.load_shard_json(0, 'deadbeef0',
400                                                        jsons_to_merge)
401    self.assertEqual({'all_tests': ['LoadShardJsonTest.test0']}, content)
402    self.assertIsNone(err)
403
404    content, err = standard_gtest_merge.load_shard_json(12, 'deadbeef12',
405                                                        jsons_to_merge)
406    self.assertEqual({'all_tests': ['LoadShardJsonTest.test12']}, content)
407    self.assertIsNone(err)
408
409
410class MergeShardResultsTest(_StandardGtestMergeTest):
411  """Tests for merge_shard_results function."""
412
413  # pylint: disable=super-with-arguments
414  def setUp(self):
415    super(MergeShardResultsTest, self).setUp()
416    self.summary = None
417    self.test_files = []
418
419  # pylint: enable=super-with-arguments
420
421  def stage(self, summary, files):
422    self.summary = self._write_temp_file('summary.json', summary)
423    for path, content in files.items():
424      abs_path = self._write_temp_file(path, content)
425      self.test_files.append(abs_path)
426
427  def call(self):
428    stdout = six.StringIO()
429    with mock.patch('sys.stdout', stdout):
430      merged = standard_gtest_merge.merge_shard_results(self.summary,
431                                                        self.test_files)
432      return merged, stdout.getvalue().strip()
433
434  def assertUnicodeEquals(self, expectation, result):
435
436    def convert_to_unicode(key_or_value):
437      if isinstance(key_or_value, str):
438        return six.text_type(key_or_value)
439      if isinstance(key_or_value, dict):
440        return {
441            convert_to_unicode(k): convert_to_unicode(v)
442            for k, v in key_or_value.items()
443        }
444      if isinstance(key_or_value, list):
445        return [convert_to_unicode(x) for x in key_or_value]
446      return key_or_value
447
448    unicode_expectations = convert_to_unicode(expectation)
449    unicode_result = convert_to_unicode(result)
450    self.assertEqual(unicode_expectations, unicode_result)
451
452  def test_ok(self):
453    # Two shards, both successfully finished.
454    self.stage(
455        {
456            u'shards': [
457                {
458                    u'state': u'COMPLETED',
459                },
460                {
461                    u'state': u'COMPLETED',
462                },
463            ],
464        }, {
465            '0/output.json': GOOD_GTEST_JSON_0,
466            '1/output.json': GOOD_GTEST_JSON_1,
467        })
468    merged, stdout = self.call()
469    merged['swarming_summary'] = {
470        'shards': [{
471            u'state': u'COMPLETED',
472            u'outputs_ref': {
473                u'view_url': u'blah',
474            },
475        }],
476    }
477    self.assertUnicodeEquals(GOOD_GTEST_JSON_MERGED, merged)
478    self.assertEqual('', stdout)
479
480  def test_timed_out(self):
481    # Two shards, both successfully finished.
482    self.stage(
483        {
484            'shards': [
485                {
486                    'state': 'COMPLETED',
487                },
488                {
489                    'state': 'TIMED_OUT',
490                },
491            ],
492        }, {
493            '0/output.json': GOOD_GTEST_JSON_0,
494            '1/output.json': TIMED_OUT_GTEST_JSON_1,
495        })
496    merged, stdout = self.call()
497
498    self.assertUnicodeEquals(TIMED_OUT_GTEST_JSON_MERGED, merged)
499    self.assertIn('Test runtime exceeded allocated time\n', stdout)
500
501  def test_missing_summary_json(self):
502    # summary.json is missing, should return None and emit warning.
503    self.summary = os.path.join(self.temp_dir, 'summary.json')
504    merged, output = self.call()
505    self.assertEqual(None, merged)
506    self.assertIn('@@@STEP_WARNINGS@@@', output)
507    self.assertIn('summary.json is missing or can not be read', output)
508
509  def test_unfinished_shards(self):
510    # Only one shard (#1) finished. Shard #0 did not.
511    self.stage({
512        u'shards': [
513            None,
514            {
515                u'state': u'COMPLETED',
516            },
517        ],
518    }, {
519        u'1/output.json': GOOD_GTEST_JSON_1,
520    })
521    merged, stdout = self.call()
522    merged.pop('swarming_summary')
523    self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
524    self.assertIn('@@@STEP_WARNINGS@@@\nsome shards did not complete: 0\n',
525                  stdout)
526    self.assertIn(
527        '@@@STEP_LOG_LINE@some shards did not complete: 0@'
528        'Missing results from the following shard(s): 0@@@\n', stdout)
529
530  def test_missing_output_json(self):
531    # Shard #0 output json is missing.
532    self.stage(
533        {
534            u'shards': [
535                {
536                    u'state': u'COMPLETED',
537                },
538                {
539                    u'state': u'COMPLETED',
540                },
541            ],
542        }, {
543            u'1/output.json': GOOD_GTEST_JSON_1,
544        })
545    merged, stdout = self.call()
546    merged.pop('swarming_summary')
547    self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
548    self.assertIn('No result was found: '
549                  'shard 0 test output was missing', stdout)
550
551  def test_large_output_json(self):
552    # a shard is too large.
553    self.stage(
554        {
555            u'shards': [
556                {
557                    u'state': u'COMPLETED',
558                },
559                {
560                    u'state': u'COMPLETED',
561                },
562            ],
563        }, {
564            '0/output.json': GOOD_GTEST_JSON_0,
565            '1/output.json': GOOD_GTEST_JSON_1,
566        })
567    old_json_limit = standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT
568    len0 = len(json.dumps(GOOD_GTEST_JSON_0))
569    len1 = len(json.dumps(GOOD_GTEST_JSON_1))
570    large_shard = '0' if len0 > len1 else '1'
571    try:
572      # Override max output.json size just for this test.
573      standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT = min(len0, len1)
574      merged, stdout = self.call()
575      merged.pop('swarming_summary')
576      self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
577      self.assertIn(
578          'No result was found: '
579          'shard %s test output exceeded the size limit' % large_shard, stdout)
580    finally:
581      standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT = old_json_limit
582
583
584class CommandLineTest(common_merge_script_tests.CommandLineTest):
585
586  # pylint: disable=super-with-arguments
587  def __init__(self, methodName='runTest'):
588    super(CommandLineTest, self).__init__(methodName, standard_gtest_merge)
589
590  # pylint: enable=super-with-arguments
591
592
593if __name__ == '__main__':
594  logging.basicConfig(
595      level=logging.DEBUG if '-v' in sys.argv else logging.ERROR)
596  if '-v' in sys.argv:
597    unittest.TestCase.maxDiff = None
598  unittest.main()
599