1// Copyright (C) 2013 Google Inc. All rights reserved. 2// 3// Redistribution and use in source and binary forms, with or without 4// modification, are permitted provided that the following conditions are 5// met: 6// 7// * Redistributions of source code must retain the above copyright 8// notice, this list of conditions and the following disclaimer. 9// * Redistributions in binary form must reproduce the above 10// copyright notice, this list of conditions and the following disclaimer 11// in the documentation and/or other materials provided with the 12// distribution. 13// * Neither the name of Google Inc. nor the names of its 14// contributors may be used to endorse or promote products derived from 15// this software without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29var results = results || {}; 30 31(function() { 32 33// Keys in the JSON files. 34results.NUM_FAILURES_BY_TYPE = 'num_failures_by_type'; 35results.FAILURE_MAP = 'failure_map'; 36results.CHROME_REVISIONS = 'chromeRevision'; 37results.BLINK_REVISIONS = 'blinkRevision'; 38results.TIMESTAMPS = 'secondsSinceEpoch'; 39results.BUILD_NUMBERS = 'buildNumbers'; 40results.TESTS = 'tests'; 41 42// Failure types. 43results.PASS = 'PASS'; 44results.NO_DATA = 'NO DATA'; 45results.SKIP = 'SKIP'; 46results.NOTRUN = 'NOTRUN'; 47results.WONTFIX = 'WONTFIX'; 48 49// FIXME: Create a ResultsJson class or something similar that abstracts out the JSON 50// data format. Code outside this class shouldn't know about the guts of the JSON format. 51 52// Enum for indexing into the run-length encoded results in the JSON files. 53// 0 is where the count is length is stored. 1 is the value. 54results.RLE = { 55 LENGTH: 0, 56 VALUE: 1 57} 58 59var NON_FAILURE_TYPES = [results.PASS, results.NO_DATA, results.SKIP, results.NOTRUN, results.WONTFIX]; 60 61results.isFailingResult = function(failureMap, failureType) 62{ 63 return NON_FAILURE_TYPES.indexOf(failureMap[failureType]) == -1; 64} 65 66results.testCounts = function(failuresByType) 67{ 68 var countData = { 69 totalTests: [], 70 totalFailingTests: [] 71 }; 72 73 for (var failureType in failuresByType) { 74 if (failureType == results.WONTFIX) 75 continue; 76 77 var failures = failuresByType[failureType]; 78 failures.forEach(function(count, index) { 79 if (!countData.totalTests[index]) { 80 countData.totalTests[index] = 0; 81 countData.totalFailingTests[index] = 0; 82 } 83 84 countData.totalTests[index] += count; 85 if (failureType != results.PASS) 86 countData.totalFailingTests[index] += count; 87 }); 88 } 89 return countData; 90} 91 92results.determineFlakiness = function(failureMap, testResults, out) 93{ 94 // FIXME: Ideally this heuristic would be a bit smarter and not consider 95 // all passes, followed by a few consecutive failures, followed by all passes 96 // to be flakiness since that's more likely the test actually failing for a 97 // few runs due to a commit. 98 var FAILURE_TYPES_TO_IGNORE = [results.NOTRUN, results.NO_DATA, results.SKIP]; 99 var flipCount = 0; 100 var mostRecentNonIgnorableFailureType; 101 102 for (var i = 0; i < testResults.length; i++) { 103 var result = testResults[i][results.RLE.VALUE]; 104 var failureType = failureMap[result]; 105 if (failureType != mostRecentNonIgnorableFailureType && FAILURE_TYPES_TO_IGNORE.indexOf(failureType) == -1) { 106 if (mostRecentNonIgnorableFailureType) 107 flipCount++; 108 mostRecentNonIgnorableFailureType = failureType; 109 } 110 } 111 112 out.flipCount = flipCount; 113 out.isFlaky = flipCount > 1; 114} 115 116})(); 117