1#!/usr/bin/env vpython3 2# Copyright 2021 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# pylint: disable=protected-access 7 8import datetime 9import os 10import unittest 11import unittest.mock as mock 12 13from flake_suppressor_common import common_typing as ct 14from flake_suppressor_common import data_types 15from flake_suppressor_common import tag_utils as common_tag_utils 16from flake_suppressor_common import unittest_utils as uu 17 18GENERIC_EXPECTATION_FILE_CONTENTS = """\ 19# tags: [ win ] 20# results: [ Failure ] 21crbug.com/1111 [ win ] foo_test [ Failure ] 22""" 23 24GPU_EXPECTATION_FILE_CONTENTS = """\ 25# tags: [ win ] 26# tags: [ amd nvidia ] 27# results: [ Failure ] 28crbug.com/1111 [ win nvidia ] conformance/textures/misc/video-rotation.html [ Failure ] 29""" 30 31 32class BaseResultsUnittest(unittest.TestCase): 33 def setUp(self) -> None: 34 common_tag_utils.SetTagUtilsImplementation(uu.UnitTestTagUtils) 35 expectations_processor = uu.UnitTestExpectationProcessor() 36 self._results = uu.UnitTestResultProcessor(expectations_processor) 37 self._local_patcher = mock.patch( 38 'flake_suppressor_common.results.expectations.' 39 'ExpectationProcessor.GetLocalCheckoutExpectationFileContents') 40 self._local_mock = self._local_patcher.start() 41 self._local_mock.return_value = {} 42 self.addCleanup(self._local_patcher.stop) 43 self._expectation_file_patcher = mock.patch.object( 44 uu.UnitTestExpectationProcessor, 'GetExpectationFileForSuite') 45 self._expectation_file_mock = self._expectation_file_patcher.start() 46 self.addCleanup(self._expectation_file_patcher.stop) 47 48 49class AggregateResultsUnittest(BaseResultsUnittest): 50 def testBasic(self) -> None: 51 """Basic functionality test.""" 52 query_results = [ 53 { 54 'name': ('gpu_tests.webgl_conformance_integration_test.' 55 'WebGLConformanceIntegrationTest.' 56 'conformance/textures/misc/video-rotation.html'), 57 'id': 58 'build-1111', 59 # The win-laptop tag is ignored, and thus should be removed in the 60 # output. 61 'typ_tags': ['win', 'nvidia', 'win-laptop'], 62 }, 63 { 64 'name': ('gpu_tests.webgl_conformance_integration_test.' 65 'WebGLConformanceIntegrationTest.' 66 'conformance/textures/misc/video-rotation.html'), 67 'id': 68 'build-2222', 69 'typ_tags': ['win', 'nvidia'], 70 }, 71 { 72 'name': ('gpu_tests.webgl_conformance_integration_test.' 73 'WebGLConformanceIntegrationTest.' 74 'conformance/textures/misc/video-rotation.html'), 75 'id': 76 'build-3333', 77 'typ_tags': ['win', 'amd'], 78 }, 79 { 80 'name': ('gpu_tests.webgl_conformance_integration_test.' 81 'WebGLConformanceIntegrationTest.' 82 'conformance/textures/misc/texture-npot-video.html'), 83 'id': 84 'build-4444', 85 'typ_tags': ['win', 'nvidia'], 86 }, 87 { 88 'name': ('gpu_tests.pixel_integration_test.PixelIntegrationTest.' 89 'Pixel_CSS3DBlueBox'), 90 'id': 91 'build-5555', 92 'typ_tags': ['win', 'nvidia'], 93 }, 94 ] 95 expected_output = { 96 'webgl_conformance_integration_test': { 97 'conformance/textures/misc/video-rotation.html': { 98 ('nvidia', 'win'): [ 99 'http://ci.chromium.org/b/1111', 100 'http://ci.chromium.org/b/2222', 101 ], 102 ('amd', 'win'): ['http://ci.chromium.org/b/3333'], 103 }, 104 'conformance/textures/misc/texture-npot-video.html': { 105 ('nvidia', 'win'): ['http://ci.chromium.org/b/4444'], 106 }, 107 }, 108 'pixel_integration_test': { 109 'Pixel_CSS3DBlueBox': { 110 ('nvidia', 'win'): ['http://ci.chromium.org/b/5555'], 111 }, 112 }, 113 } 114 self.assertEqual(self._results.AggregateResults(query_results), 115 expected_output) 116 117 118class AggregateTestStatusResultsUnittest(BaseResultsUnittest): 119 def testBasic(self) -> None: 120 """Basic functionality test.""" 121 query_results = [ 122 { 123 'name': ('gpu_tests.webgl_conformance_integration_test.' 124 'WebGLConformanceIntegrationTest.' 125 'conformance/textures/misc/video-rotation.html'), 126 'id': 127 'build-1111', 128 # The win-laptop tag is ignored, and thus should be removed in the 129 # output. 130 'typ_tags': ['win', 'nvidia', 'win-laptop'], 131 'status': 132 ct.ResultStatus.FAIL, 133 'date': 134 '2023-03-01', 135 'is_slow': 136 False, 137 'typ_expectations': ['Pass'], 138 }, 139 { 140 'name': ('gpu_tests.webgl_conformance_integration_test.' 141 'WebGLConformanceIntegrationTest.' 142 'conformance/textures/misc/video-rotation.html'), 143 'id': 144 'build-2222', 145 'typ_tags': ['win', 'nvidia'], 146 'status': 147 ct.ResultStatus.CRASH, 148 'date': 149 '2023-03-02', 150 'is_slow': 151 False, 152 'typ_expectations': ['Pass'], 153 }, 154 { 155 'name': ('gpu_tests.webgl_conformance_integration_test.' 156 'WebGLConformanceIntegrationTest.' 157 'conformance/textures/misc/video-rotation.html'), 158 'id': 159 'build-3333', 160 'typ_tags': ['win', 'amd'], 161 'status': 162 ct.ResultStatus.FAIL, 163 'date': 164 '2023-03-03', 165 'is_slow': 166 True, 167 'typ_expectations': ['Pass', 'Slow'], 168 }, 169 { 170 'name': ('gpu_tests.webgl_conformance_integration_test.' 171 'WebGLConformanceIntegrationTest.' 172 'conformance/textures/misc/texture-npot-video.html'), 173 'id': 174 'build-4444', 175 'typ_tags': ['win', 'nvidia'], 176 'status': 177 ct.ResultStatus.FAIL, 178 'date': 179 '2023-03-04', 180 'is_slow': 181 True, 182 'typ_expectations': ['Pass', 'Slow'], 183 }, 184 { 185 'name': ('gpu_tests.pixel_integration_test.PixelIntegrationTest.' 186 'Pixel_CSS3DBlueBox'), 187 'id': 188 'build-5555', 189 'typ_tags': ['win', 'nvidia'], 190 'status': 191 ct.ResultStatus.FAIL, 192 'date': 193 '2023-03-05', 194 'is_slow': 195 False, 196 'typ_expectations': ['Pass'], 197 }, 198 ] 199 expected_output = { 200 'webgl_conformance_integration_test': { 201 'conformance/textures/misc/video-rotation.html': { 202 ('nvidia', 'win'): [ 203 (ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/1111', 204 datetime.date.fromisoformat('2023-03-01'), False, ['Pass' 205 ]), 206 (ct.ResultStatus.CRASH, 'http://ci.chromium.org/b/2222', 207 datetime.date.fromisoformat('2023-03-02'), False, ['Pass' 208 ]), 209 ], 210 ('amd', 'win'): [ 211 (ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/3333', 212 datetime.date.fromisoformat('2023-03-03'), True, 213 ['Pass', 'Slow']), 214 ], 215 }, 216 'conformance/textures/misc/texture-npot-video.html': { 217 ('nvidia', 'win'): 218 [(ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/4444', 219 datetime.date.fromisoformat('2023-03-04'), True, 220 ['Pass', 'Slow'])], 221 }, 222 }, 223 'pixel_integration_test': { 224 'Pixel_CSS3DBlueBox': { 225 ('nvidia', 'win'): 226 [(ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/5555', 227 datetime.date.fromisoformat('2023-03-05'), False, ['Pass'])], 228 }, 229 }, 230 } 231 self.assertEqual(self._results.AggregateTestStatusResults(query_results), 232 expected_output) 233 234 235class ConvertJsonResultsToResultObjectsUnittest(BaseResultsUnittest): 236 def testBasic(self) -> None: 237 """Basic functionality test.""" 238 r = [ 239 { 240 'name': ('gpu_tests.webgl_conformance_integration_test.' 241 'WebGLConformanceIntegrationTest.' 242 'conformance/textures/misc/video-rotation.html'), 243 'id': 244 'build-1111', 245 # The win-laptop tag is ignored, and thus should be removed in the 246 # output. 247 'typ_tags': ['win', 'nvidia', 'win-laptop'], 248 }, 249 { 250 'name': ('gpu_tests.webgl_conformance_integration_test.' 251 'WebGLConformanceIntegrationTest.' 252 'conformance/textures/misc/video-rotation.html'), 253 'id': 254 'build-2222', 255 'typ_tags': ['nvidia', 'win'], 256 }, 257 ] 258 expected_results = [ 259 data_types.Result('webgl_conformance_integration_test', 260 'conformance/textures/misc/video-rotation.html', 261 ('nvidia', 'win'), '1111'), 262 data_types.Result( 263 'webgl_conformance_integration_test', 264 'conformance/textures/misc/video-rotation.html', 265 ('nvidia', 'win'), 266 '2222', 267 ), 268 ] 269 self.assertEqual(self._results._ConvertJsonResultsToResultObjects(r), 270 expected_results) 271 272 def testOnQueryResultWithOptionalAttributes(self) -> None: 273 """Functionality test on query result with optional attributes.""" 274 r = [ 275 { 276 'name': ('gpu_tests.webgl_conformance_integration_test.' 277 'WebGLConformanceIntegrationTest.' 278 'conformance/textures/misc/video-rotation.html'), 279 'id': 280 'build-1111', 281 # The win-laptop tag is ignored, and thus should be removed in the 282 # output. 283 'typ_tags': ['win', 'nvidia', 'win-laptop'], 284 'status': 285 ct.ResultStatus.FAIL, 286 'date': 287 '2023-03-01', 288 'is_slow': 289 False, 290 'typ_expectations': ['Pass'], 291 }, 292 { 293 'name': ('gpu_tests.webgl_conformance_integration_test.' 294 'WebGLConformanceIntegrationTest.' 295 'conformance/textures/misc/video-rotation.html'), 296 'id': 297 'build-2222', 298 'typ_tags': ['nvidia', 'win'], 299 'status': 300 ct.ResultStatus.CRASH, 301 'date': 302 '2023-03-02', 303 'is_slow': 304 True, 305 'typ_expectations': ['Pass', 'Slow'], 306 }, 307 ] 308 expected_results = [ 309 data_types.Result('webgl_conformance_integration_test', 310 'conformance/textures/misc/video-rotation.html', 311 ('nvidia', 'win'), '1111', ct.ResultStatus.FAIL, 312 datetime.date.fromisoformat('2023-03-01'), False, 313 ['Pass']), 314 data_types.Result('webgl_conformance_integration_test', 315 'conformance/textures/misc/video-rotation.html', 316 ('nvidia', 'win'), '2222', ct.ResultStatus.CRASH, 317 datetime.date.fromisoformat('2023-03-02'), True, 318 ['Pass', 'Slow']), 319 ] 320 self.assertEqual(self._results._ConvertJsonResultsToResultObjects(r), 321 expected_results) 322 323 324class FilterOutSuppressedResultsUnittest(BaseResultsUnittest): 325 def testNoSuppressedResults(self) -> None: 326 """Tests functionality when no expectations apply to the given results.""" 327 self._local_mock.return_value = { 328 'foo_expectations.txt': GENERIC_EXPECTATION_FILE_CONTENTS, 329 } 330 r = [ 331 data_types.Result('foo_integration_test', 'foo_test', tuple(['linux']), 332 'id'), 333 data_types.Result('foo_integration_test', 'bar_test', tuple(['win']), 334 'id'), 335 data_types.Result('bar_integration_test', 'foo_test', tuple(['win']), 336 'id') 337 ] 338 339 self.assertEqual(self._results._FilterOutSuppressedResults(r), r) 340 341 def testSuppressedResults(self) -> None: 342 """Tests functionality when expectations apply to the given results.""" 343 self._local_mock.return_value = { 344 'foo_expectations.txt': GENERIC_EXPECTATION_FILE_CONTENTS, 345 } 346 self._expectation_file_mock.return_value = os.path.join( 347 uu.ABSOLUTE_EXPECTATION_FILE_DIRECTORY, 'foo_expectations.txt') 348 349 r = [ 350 data_types.Result('foo_integration_test', 'foo_test', ('win', 'nvidia'), 351 'id'), 352 data_types.Result('foo_integration_test', 'foo_test', tuple(['win']), 353 'id'), 354 data_types.Result('foo_integration_test', 'bar_test', tuple(['win']), 355 'id'), 356 ] 357 358 expected_filtered_results = [ 359 data_types.Result('foo_integration_test', 'bar_test', tuple(['win']), 360 'id'), 361 ] 362 363 self.assertEqual(self._results._FilterOutSuppressedResults(r), 364 expected_filtered_results) 365 366 367if __name__ == '__main__': 368 unittest.main(verbosity=2) 369