• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/*
2 * Copyright (C) 2011 Google Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23 * THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26var unittest = unittest || {};
27
28(function () {
29
30module("results");
31
32var MockResultsBaseURL = 'https://storage.googleapis.com/chromium-layout-test-archives/Mock_Builder/results/layout-test-results';
33
34unittest.kExampleResultsJSON = {
35    "tests": {
36        "scrollbars": {
37            "custom-scrollbar-with-incomplete-style.html": {
38                "expected": "IMAGE",
39                "actual": "IMAGE",
40            },
41            "expected-wontfix": {
42                "expected": "WONTFIX",
43                "actual": "SKIP",
44            },
45            "flaky-scrollbar.html": {
46                "expected": "PASS",
47                "actual": "PASS TEXT",
48                "is_unexpected": true,
49            },
50            "unexpected-failing-flaky-scrollbar.html": {
51                "expected": "TEXT",
52                "actual": "TIMEOUT TEXT",
53                "is_unexpected": true,
54            },
55            "unexpected-pass.html": {
56                "expected": "FAIL",
57                "actual": "PASS",
58                "is_unexpected": true,
59            }
60        },
61        "userscripts": {
62            "user-script-video-document.html": {
63                "expected": "FAIL",
64                "actual": "TEXT",
65            },
66            "another-test.html": {
67                "expected": "PASS",
68                "actual": "TEXT",
69                "is_unexpected": true,
70            }
71        },
72    },
73    "skipped": 339,
74    "num_regressions": 14,
75    "interrupted": false,
76    "layout_tests_dir": "\/mnt\/data\/b\/build\/slave\/Webkit_Linux\/build\/src\/third_party\/WebKit\/LayoutTests",
77    "version": 3,
78    "num_passes": 15566,
79    "has_pretty_patch": false,
80    "fixable": 1233,
81    "num_flaky":1,
82    "has_wdiff": true,
83    "blink_revision": "90430"
84};
85
86test("ResultAnalyzer", 44, function() {
87    var analyzer;
88
89    analyzer = new results.ResultAnalyzer({expected: 'PASS', actual: 'TEXT', is_unexpected: true});
90    ok(analyzer.hasUnexpectedFailures());
91    deepEqual(analyzer.unexpectedResults(), ['TEXT']);
92    ok(!analyzer.succeeded());
93    ok(!analyzer.flaky());
94
95    analyzer = new results.ResultAnalyzer({expected: 'PASS TIMEOUT', actual: 'TEXT', is_unexpected: true});
96    ok(analyzer.hasUnexpectedFailures());
97    deepEqual(analyzer.unexpectedResults(), ['TEXT']);
98    ok(!analyzer.succeeded());
99    ok(!analyzer.flaky());
100
101    analyzer = new results.ResultAnalyzer({expected: 'TEXT', actual: 'TEXT TIMEOUT', is_unexpected: true});
102    ok(analyzer.hasUnexpectedFailures());
103    deepEqual(analyzer.unexpectedResults(), ['TIMEOUT']);
104    ok(!analyzer.succeeded());
105    ok(analyzer.flaky());
106
107    analyzer = new results.ResultAnalyzer({expected: 'PASS', actual: 'TEXT TIMEOUT', is_unexpected: true});
108    ok(analyzer.hasUnexpectedFailures());
109    deepEqual(analyzer.unexpectedResults(), ['TEXT', 'TIMEOUT']);
110    ok(!analyzer.succeeded());
111    ok(analyzer.flaky());
112
113    analyzer = new results.ResultAnalyzer({expected: 'PASS TIMEOUT', actual: 'PASS TIMEOUT'});
114    ok(!analyzer.hasUnexpectedFailures());
115    deepEqual(analyzer.unexpectedResults(), []);
116    ok(analyzer.succeeded());
117    ok(analyzer.flaky());
118
119    analyzer = new results.ResultAnalyzer({expected: 'PASS TIMEOUT', actual: 'TIMEOUT PASS'});
120    ok(!analyzer.hasUnexpectedFailures());
121    deepEqual(analyzer.unexpectedResults(), []);
122    ok(analyzer.succeeded());
123    ok(analyzer.flaky());
124
125    analyzer = new results.ResultAnalyzer({expected: 'FAIL', actual: 'TIMEOUT', is_unexpected: true});
126    ok(analyzer.hasUnexpectedFailures());
127    deepEqual(analyzer.unexpectedResults(), ['TIMEOUT']);
128    ok(!analyzer.succeeded());
129    ok(!analyzer.flaky());
130
131    analyzer = new results.ResultAnalyzer({expected: 'FAIL', actual: 'IMAGE', is_unexpected: true});
132    ok(analyzer.hasUnexpectedFailures());
133    deepEqual(analyzer.unexpectedResults(), ['IMAGE']);
134    ok(!analyzer.succeeded());
135    ok(!analyzer.flaky());
136
137    analyzer = new results.ResultAnalyzer({expected: 'FAIL', actual: 'AUDIO'});
138    ok(!analyzer.hasUnexpectedFailures());
139    deepEqual(analyzer.unexpectedResults(), []);
140    ok(!analyzer.succeeded());
141    ok(!analyzer.flaky());
142
143    analyzer = new results.ResultAnalyzer({expected: 'FAIL', actual: 'TEXT'});
144    ok(!analyzer.hasUnexpectedFailures());
145    deepEqual(analyzer.unexpectedResults(), []);
146    ok(!analyzer.succeeded());
147    ok(!analyzer.flaky());
148
149    analyzer = new results.ResultAnalyzer({expected: 'FAIL', actual: 'IMAGE+TEXT'});
150    ok(!analyzer.hasUnexpectedFailures());
151    deepEqual(analyzer.unexpectedResults(), []);
152    ok(!analyzer.succeeded());
153    ok(!analyzer.flaky());
154});
155
156test("expectedFailures", 1, function() {
157    var expectedFailures = results.expectedFailures(unittest.kExampleResultsJSON);
158    deepEqual(expectedFailures, {
159        "scrollbars/custom-scrollbar-with-incomplete-style.html": {
160            "expected": "IMAGE",
161            "actual": "IMAGE"
162        },
163        "userscripts/user-script-video-document.html": {
164            "expected": "FAIL",
165            "actual": "TEXT"
166        }
167    });
168});
169
170test("unexpectedFailures", 1, function() {
171    var unexpectedFailures = results.unexpectedFailures(unittest.kExampleResultsJSON);
172    deepEqual(unexpectedFailures, {
173        "userscripts/another-test.html": {
174            "expected": "PASS",
175            "actual": "TEXT",
176            "is_unexpected": true,
177        }
178    });
179});
180
181test("unexpectedFailuresByTest", 1, function() {
182    var unexpectedFailuresByTest = results.unexpectedFailuresByTest({
183        "Mock Builder": unittest.kExampleResultsJSON
184    });
185    deepEqual(unexpectedFailuresByTest, {
186        "userscripts/another-test.html": {
187            "Mock Builder": {
188                "expected": "PASS",
189                "actual": "TEXT",
190                "is_unexpected": true,
191            }
192        }
193    });
194});
195
196test("failureInfoForTestAndBuilder", 1, function() {
197    var unexpectedFailuresByTest = results.unexpectedFailuresByTest({
198        "Mock Builder": unittest.kExampleResultsJSON
199    });
200    var failureInfo = results.failureInfoForTestAndBuilder(unexpectedFailuresByTest, "userscripts/another-test.html", "Mock Builder");
201    deepEqual(failureInfo, {
202        "testName": "userscripts/another-test.html",
203        "builderName": "Mock Builder",
204        "failureTypeList": ["TEXT"],
205    });
206});
207
208test("resultKind", 12, function() {
209    equals(results.resultKind("http://example.com/foo-actual.txt"), "actual");
210    equals(results.resultKind("http://example.com/foo-expected.txt"), "expected");
211    equals(results.resultKind("http://example.com/foo-diff.txt"), "diff");
212    equals(results.resultKind("http://example.com/foo.bar-actual.txt"), "actual");
213    equals(results.resultKind("http://example.com/foo.bar-expected.txt"), "expected");
214    equals(results.resultKind("http://example.com/foo.bar-diff.txt"), "diff");
215    equals(results.resultKind("http://example.com/foo-actual.png"), "actual");
216    equals(results.resultKind("http://example.com/foo-expected.png"), "expected");
217    equals(results.resultKind("http://example.com/foo-diff.png"), "diff");
218    equals(results.resultKind("http://example.com/foo-pretty-diff.html"), "diff");
219    equals(results.resultKind("http://example.com/foo-wdiff.html"), "diff");
220    equals(results.resultKind("http://example.com/foo-xyz.html"), "unknown");
221});
222
223test("resultType", 12, function() {
224    equals(results.resultType("http://example.com/foo-actual.txt"), "text");
225    equals(results.resultType("http://example.com/foo-expected.txt"), "text");
226    equals(results.resultType("http://example.com/foo-diff.txt"), "text");
227    equals(results.resultType("http://example.com/foo.bar-actual.txt"), "text");
228    equals(results.resultType("http://example.com/foo.bar-expected.txt"), "text");
229    equals(results.resultType("http://example.com/foo.bar-diff.txt"), "text");
230    equals(results.resultType("http://example.com/foo-actual.png"), "image");
231    equals(results.resultType("http://example.com/foo-expected.png"), "image");
232    equals(results.resultType("http://example.com/foo-diff.png"), "image");
233    equals(results.resultType("http://example.com/foo-pretty-diff.html"), "text");
234    equals(results.resultType("http://example.com/foo-wdiff.html"), "text");
235    equals(results.resultType("http://example.com/foo.xyz"), "text");
236});
237
238test("resultNodeForTest", 4, function() {
239    deepEqual(results.resultNodeForTest(unittest.kExampleResultsJSON, "userscripts/another-test.html"), {
240        "expected": "PASS",
241        "actual": "TEXT",
242        "is_unexpected": true,
243    });
244    equals(results.resultNodeForTest(unittest.kExampleResultsJSON, "foo.html"), null);
245    equals(results.resultNodeForTest(unittest.kExampleResultsJSON, "userscripts/foo.html"), null);
246    equals(results.resultNodeForTest(unittest.kExampleResultsJSON, "userscripts/foo/bar.html"), null);
247});
248
249test("walkHistory", 5, function() {
250    var simulator = new NetworkSimulator();
251
252    var keyMap = {
253        "Mock_Builder": {
254            "11108": {
255                "tests": {
256                    "userscripts": {
257                        "another-test.html": {
258                            "expected": "PASS",
259                            "actual": "TEXT",
260                            "is_unexpected": true,
261                        }
262                    },
263                },
264                "blink_revision": "90430"
265            },
266            "11107":{
267                "tests": {
268                    "userscripts": {
269                        "user-script-video-document.html": {
270                            "expected": "FAIL",
271                            "actual": "TEXT",
272                            "is_unexpected": false,
273                        },
274                        "another-test.html": {
275                            "expected": "PASS",
276                            "actual": "TEXT",
277                            "is_unexpected": true,
278                        }
279                    },
280                },
281                "blink_revision": "90429"
282            },
283            "11106":{
284                "tests": {
285                    "userscripts": {
286                        "another-test.html": {
287                            "expected": "PASS",
288                            "actual": "TEXT",
289                            "is_unexpected": true,
290                        }
291                    },
292                },
293                "blink_revision": "90426"
294            },
295            "11105":{
296                "tests": {
297                    "userscripts": {
298                        "user-script-video-document.html": {
299                            "expected": "FAIL",
300                            "actual": "TEXT",
301                            "is_unexpected": false,
302                        },
303                    },
304                },
305                "blink_revision": "90424"
306            },
307        },
308        "Another_Builder": {
309            "22202":{
310                "tests": {
311                    "userscripts": {
312                        "another-test.html": {
313                            "expected": "PASS",
314                            "actual": "TEXT",
315                            "is_unexpected": true,
316                        }
317                    },
318                },
319                "blink_revision": "90426"
320            },
321            "22201":{
322                "tests": {
323                },
324                "blink_revision": "90425"
325            },
326        },
327    };
328
329    simulator.jsonp = function(url, callback) {
330        simulator.scheduleCallback(function() {
331            var result = keyMap[/[^/]+_Builder/.exec(url)][/\d+/.exec(url)];
332            callback(result ? result : {});
333        });
334    };
335
336    simulator.get = function(url, callback) {
337        simulator.scheduleCallback(function() {
338            if (/Mock_Builder/.test(url))
339                callback('<ListBucketResult>' +
340                         '<Prefix>Mock_Builder/11101/</Prefix>' +
341                         '<Prefix>Mock_Builder/11102/</Prefix>' +
342                         '<Prefix>Mock_Builder/11103/</Prefix>' +
343                         '<Prefix>Mock_Builder/11104/</Prefix>' +
344                         '<Prefix>Mock_Builder/11105/</Prefix>' +
345                         '<Prefix>Mock_Builder/11106/</Prefix>' +
346                         '<Prefix>Mock_Builder/11107/</Prefix>' +
347                         '<Prefix>Mock_Builder/11108/</Prefix>' +
348                         '</ListBucketResult>');
349            else if (/Another_Builder/.test(url))
350                callback('<ListBucketResult>' +
351                         '<Prefix>Another_Builder/22201/</Prefix>' +
352                         '<Prefix>Another_Builder/22202/</Prefix>' +
353                         '</ListBucketResult>');
354            else if (/Mock Builder/.test(url))
355                callback({cachedBuilds: [11101, 11102, 11103, 11104, 11105, 11106, 11107, 11108]});
356            else if (/Another Builder/.test(url))
357                callback({cachedBuilds: [22201, 22202]});
358            else
359                ok(false, 'Unexpected URL: ' + url);
360        });
361    };
362
363    simulator.runTest(function() {
364        results.regressionRangeForFailure("Mock Builder", "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
365            equals(oldestFailingRevision, 90426);
366            equals(newestPassingRevision, 90424);
367        });
368
369        results.unifyRegressionRanges(["Mock Builder", "Another Builder"], "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
370            equals(oldestFailingRevision, 90426);
371            equals(newestPassingRevision, 90425);
372        });
373    });
374});
375
376test("walkHistory (no revision)", 3, function() {
377    var simulator = new NetworkSimulator();
378
379    var keyMap = {
380        "Mock_Builder": {
381            "11103": {
382                "tests": {
383                    "userscripts": {
384                        "another-test.html": {
385                            "expected": "PASS",
386                            "actual": "TEXT"
387                        }
388                    },
389                },
390                "blink_revision": ""
391            },
392            "11102":{
393                "tests": {
394                },
395                "blink_revision": ""
396            },
397        },
398    };
399
400    simulator.jsonp = function(url, callback) {
401        simulator.scheduleCallback(function() {
402            var result = keyMap[/[^/]+_Builder/.exec(url)][/\d+/.exec(url)];
403            callback(result ? result : {});
404        });
405    };
406
407    simulator.get = function(url, callback) {
408        simulator.scheduleCallback(function() {
409            callback('<a href="11101/"></a><a href="11102/"></a><a href="11103/"></a>');
410        });
411    };
412
413
414    simulator.runTest(function() {
415        results.regressionRangeForFailure("Mock Builder", "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
416            equals(oldestFailingRevision, 0);
417            equals(newestPassingRevision, 0);
418        });
419    });
420});
421
422test("collectUnexpectedResults", 1, function() {
423    var dictionaryOfResultNodes = {
424        "foo": {
425            "expected": "IMAGE",
426            "actual": "IMAGE"
427        },
428        "bar": {
429            "expected": "PASS",
430            "actual": "PASS TEXT"
431        },
432        "baz": {
433            "expected": "TEXT",
434            "actual": "IMAGE"
435        },
436        "qux": {
437            "expected": "PASS",
438            "actual": "TEXT"
439        },
440        "taco": {
441            "expected": "PASS",
442            "actual": "TEXT"
443        },
444    };
445
446    var collectedResults = results.collectUnexpectedResults(dictionaryOfResultNodes);
447    deepEqual(collectedResults, ["TEXT", "IMAGE"]);
448});
449
450test("failureTypeToExtensionList", 5, function() {
451    deepEqual(results.failureTypeToExtensionList('TEXT'), ['txt']);
452    deepEqual(results.failureTypeToExtensionList('IMAGE+TEXT'), ['txt', 'png']);
453    deepEqual(results.failureTypeToExtensionList('IMAGE'), ['png']);
454    deepEqual(results.failureTypeToExtensionList('CRASH'), []);
455    deepEqual(results.failureTypeToExtensionList('TIMEOUT'), []);
456});
457
458test("fetchResultsURLs", 5, function() {
459    var simulator = new NetworkSimulator();
460
461    var probedURLs = [];
462    simulator.probe = function(url, options)
463    {
464        simulator.scheduleCallback(function() {
465            probedURLs.push(url);
466            if (base.endsWith(url, '.txt'))
467                options.success.call();
468            else if (/taco.+png$/.test(url))
469                options.success.call();
470            else
471                options.error.call();
472        });
473    };
474
475    simulator.runTest(function() {
476        results.fetchResultsURLs({
477            'builderName': "Mock Builder",
478            'testName': "userscripts/another-test.html",
479            'failureTypeList': ['IMAGE', 'CRASH'],
480        }, function(resultURLs) {
481            deepEqual(resultURLs, [
482                MockResultsBaseURL + "/userscripts/another-test-crash-log.txt"
483            ]);
484        });
485        results.fetchResultsURLs({
486            'builderName': "Mock Builder",
487            'testName': "userscripts/another-test.html",
488            'failureTypeList': ['TIMEOUT'],
489        }, function(resultURLs) {
490            deepEqual(resultURLs, []);
491        });
492        results.fetchResultsURLs({
493            'builderName': "Mock Builder",
494            'testName': "userscripts/taco.html",
495            'failureTypeList': ['IMAGE', 'IMAGE+TEXT'],
496        }, function(resultURLs) {
497            deepEqual(resultURLs, [
498                MockResultsBaseURL + "/userscripts/taco-expected.png",
499                MockResultsBaseURL + "/userscripts/taco-actual.png",
500                MockResultsBaseURL + "/userscripts/taco-diff.png",
501                MockResultsBaseURL + "/userscripts/taco-expected.txt",
502                MockResultsBaseURL + "/userscripts/taco-actual.txt",
503                MockResultsBaseURL + "/userscripts/taco-diff.txt",
504            ]);
505        });
506    });
507
508    deepEqual(probedURLs, [
509        MockResultsBaseURL + "/userscripts/another-test-expected.png",
510        MockResultsBaseURL + "/userscripts/another-test-actual.png",
511        MockResultsBaseURL + "/userscripts/another-test-diff.png",
512        MockResultsBaseURL + "/userscripts/another-test-crash-log.txt",
513        MockResultsBaseURL + "/userscripts/taco-expected.png",
514        MockResultsBaseURL + "/userscripts/taco-actual.png",
515        MockResultsBaseURL + "/userscripts/taco-diff.png",
516        MockResultsBaseURL + "/userscripts/taco-actual.txt",
517        MockResultsBaseURL + "/userscripts/taco-expected.txt",
518        MockResultsBaseURL + "/userscripts/taco-diff.txt",
519    ]);
520});
521
522test("fetchResultsByBuilder", 3, function() {
523    var simulator = new NetworkSimulator();
524
525    var probedURLs = [];
526    simulator.jsonp = function(url, callback)
527    {
528        simulator.scheduleCallback(function() {
529            probedURLs.push(url);
530            callback(base.endsWith(url, 'results/layout-test-results/failing_results.json'));
531        });
532    };
533
534    simulator.runTest(function() {
535        results.fetchResultsByBuilder(['MockBuilder1', 'MockBuilder2'], function(resultsByBuilder) {
536            deepEqual(resultsByBuilder, {
537                "MockBuilder1": true,
538                "MockBuilder2": true,
539            });
540        });
541    });
542
543    deepEqual(probedURLs, [
544        MockResultsBaseURL.replace('Mock_Builder', 'MockBuilder1') + "/failing_results.json",
545        MockResultsBaseURL.replace('Mock_Builder', 'MockBuilder2') + "/failing_results.json"
546    ]);
547
548});
549
550})();
551