1#!/usr/bin/env python3
2
3"""
4Parses information about failing tests, and then generates a change to disable them.
5
6Requires that the `bugged` command-line tool is installed, see go/bugged .
7"""
8
9import argparse, csv, os, subprocess
10
11parser = argparse.ArgumentParser(
12  description=__doc__
13)
14parser.add_argument("-v", help="Verbose", action="store_true")
15
16dirOfThisScript = os.path.dirname(os.path.realpath(__file__))
17supportRoot = os.path.dirname(dirOfThisScript)
18
19logger = None
20
21class PrintLogger(object):
22  def log(self, message):
23    print(message)
24
25class DisabledLogger(object):
26  def log(self, message):
27    pass
28
29def log(message):
30  logger.log(message)
31
32class LocatedFailure(object):
33  def __init__(self, failure, location, bugId):
34    self.failure = failure
35    self.location = location
36    self.bugId = bugId
37
38class TestFailure(object):
39  def __init__(self, qualifiedClassName, methodName, testDefinitionName, branchName, testFailureUrl, bugId):
40    self.qualifiedClassName = qualifiedClassName
41    self.methodName = methodName
42    self.testDefinitionName = testDefinitionName
43    self.branchName = branchName
44    self.failureUrl = testFailureUrl
45    self.bugId = bugId
46
47  def getUrl(self):
48    return self.testFailureUrl
49
50class FailuresDatabase(object):
51  """A collection of LocatedFailure instances, organized by their locations"""
52  def __init__(self):
53    self.failuresByPath = {}
54
55  def add(self, locatedFailure):
56    path = locatedFailure.location.filePath
57    if path not in self.failuresByPath:
58      self.failuresByPath[path] = {}
59    failuresAtPath = self.failuresByPath[path]
60
61    lineNumber = locatedFailure.location.lineNumber
62    if lineNumber not in failuresAtPath:
63      failuresAtPath[lineNumber] = locatedFailure
64
65  # returns Map<String, LocatedFailure> with key being filePath
66  def getAll(self):
67    results = {}
68    for path, failuresAtPath in self.failuresByPath.items():
69      lineNumbers = sorted(failuresAtPath.keys(), reverse=True)
70      resultsAtPath = []
71      # add failures in reverse order to make it easier to modify methods without adjusting line numbers for other methods
72      for line in lineNumbers:
73        resultsAtPath.append(failuresAtPath[line])
74      results[path] = resultsAtPath
75    return results
76
77def parseBugLine(bugId, line):
78  components = line.split(" | ")
79  if len(components) < 3:
80    return None
81  testLink = components[1]
82  # Example test link: [compose-ui-uidebugAndroidTest.xml androidx.compose.ui.window.PopupAlignmentTest#popup_correctPosition_alignmentTopCenter_rtl](https://android-build.googleplex.com/builds/tests/view?testResultId=TR96929024659298098)
83  closeBracketIndex = testLink.rindex("]")
84  if closeBracketIndex <= 0:
85    raise Exception("Failed to parse b/" + bugId + " '" + line + "', testLink '" + testLink + "', closeBracketIndex = " + str(closeBracketIndex))
86  linkText = testLink[1:closeBracketIndex]
87  linkDest = testLink[closeBracketIndex + 1:]
88  # Example linkText: compose-ui-uidebugAndroidTest.xml androidx.compose.ui.window.PopupAlignmentTest#popup_correctPosition_alignmentTopCenter_rtl
89  # Example linkDest: (https://android-build.googleplex.com/builds/tests/view?testResultId=TR96929024659298098)
90  testResultUrl = linkDest.replace("(", "").replace(")", "")
91  # Example testResultUrl: https://android-build.googleplex.com/builds/tests/view?testResultId=TR96929024659298098
92  spaceIndex = linkText.index(" ")
93  if spaceIndex <= 0:
94    raise Exception("Failed to parse b/" + bugId + " '" + line + "', linkText = '" + linkText + ", spaceIndex = " + str(spaceIndex))
95  testDefinitionName = linkText[:spaceIndex]
96  testPath = linkText[spaceIndex+1:]
97  # Example test path: androidx.compose.ui.window.PopupAlignmentTest#popup_correctPosition_alignmentTopCenter_rtl
98  testPathSplit = testPath.split("#")
99  if len(testPathSplit) != 2:
100    raise Exception("Failed to parse b/" + bugId + " '" + line + "', testPath = '" + testPath + "', len(testPathSplit) = " + str(len(testPathSplit)))
101  testClass, testMethod = testPathSplit
102
103  branchName = components[2].strip()
104  print("  parsed test failure class=" + testClass + " method='" + testMethod + "' definition=" + testDefinitionName + " branch=" + branchName + " failureUrl=" + testResultUrl + " bugId=" + bugId)
105  return TestFailure(testClass, testMethod, testDefinitionName, branchName, testResultUrl, bugId)
106
107def parseBug(bugId):
108  bugText = shellRunner.runAndGetOutput(["bugged", "show", bugId])
109  log("bug text = '" + bugText + "'")
110  failures = []
111  bugLines = bugText.split("\n")
112
113  stillFailing = True
114  listingTests = False
115  for i in range(len(bugLines)):
116    line = bugLines[i]
117    #log("parsing bug line " + line)
118    if listingTests:
119      failure = parseBugLine(bugId, line)
120      if failure is not None:
121        failures.append(failure)
122    if "---|---|---|---|---" in line: # table start
123      listingTests = True
124    if " # " in line: # start of new section
125      listingTests = False
126    if "There are no more failing tests in this regression" in line or "ATN has not seen a failure for this regression recently." in line or "This regression has been identified as a duplicate of another one" in line:
127      stillFailing = False
128  if len(failures) < 1:
129    raise Exception("Failed to parse b/" + bugId + ": identified 0 failures. Rerun with -v for more information")
130  if not stillFailing:
131    print("tests no longer failing")
132    return []
133  return failures
134
135# identifies failing tests
136def getFailureData():
137  bugsQuery = ["bugged", "search", "hotlistid:5083126 status:open", "--columns", "issue"]
138  print("Searching for bugs: " + str(bugsQuery))
139  bugsOutput = shellRunner.runAndGetOutput(bugsQuery)
140  bugIds = bugsOutput.split("\n")
141  print("Checking " + str(len(bugIds)) + " bugs")
142  failures = []
143  for i in range(len(bugIds)):
144    bugId = bugIds[i].strip()
145    if bugId != "issue" and bugId != "":
146      print("")
147      print("Parsing bug " + bugId + " (" + str(i) + "/" + str(len(bugIds)) + ")")
148      failures += parseBug(bugId)
149  return failures
150
151class FileLocation(object):
152  def __init__(self, filePath, lineNumber):
153    self.filePath = filePath
154    self.lineNumber = lineNumber
155
156  def __str__(self):
157    return self.filePath + "#" + str(self.lineNumber)
158
159class ShellRunner(object):
160  def __init__(self):
161    return
162
163  def runAndGetOutput(self, args):
164    result = subprocess.run(args, capture_output=True, text=True).stdout
165    return result
166
167  def run(self, args):
168    subprocess.run(args, capture_output=False)
169
170shellRunner = ShellRunner()
171
172class FileFinder(object):
173  def __init__(self, rootPath):
174    self.rootPath = rootPath
175    self.resultsCache = {}
176
177  def findIname(self, name):
178    if name not in self.resultsCache:
179      text = shellRunner.runAndGetOutput(["find", self.rootPath , "-type", "f", "-iname", name])
180      filePaths = [path.strip() for path in text.split("\n")]
181      filePaths = [path for path in filePaths if path != ""]
182      self.resultsCache[name] = filePaths
183    return self.resultsCache[name]
184fileFinder = FileFinder(supportRoot)
185
186class ClassFinder(object):
187  """Locates the file path and line number for classes and methods"""
188  def __init__(self):
189    self.classToFile_cache = {}
190    self.methodLocations_cache = {}
191
192  def findMethod(self, qualifiedClassName, methodName):
193    bracketIndex = methodName.find("[")
194    if bracketIndex >= 0:
195      methodName = methodName[:bracketIndex]
196    fullName = qualifiedClassName + "." + methodName
197    containingFile = self.findFileContainingClass(qualifiedClassName)
198    if containingFile is None:
199      return None
200    if fullName not in self.methodLocations_cache:
201      index = -1
202      foundLineNumber = None
203      with open(containingFile) as f:
204        for line in f:
205          index += 1
206          if (" " + methodName + "(") in line:
207            if foundLineNumber is not None:
208              # found two matches, can't choose one
209              foundLineNumber = None
210              break
211            foundLineNumber = index
212      result = None
213      if foundLineNumber is not None:
214        result = FileLocation(containingFile, foundLineNumber)
215      self.methodLocations_cache[fullName] = result
216    return self.methodLocations_cache[fullName]
217
218
219  def findFileContainingClass(self, qualifiedName):
220    if qualifiedName not in self.classToFile_cache:
221      lastDotIndex = qualifiedName.rindex(".")
222      if lastDotIndex >= 0:
223        packageName = qualifiedName[:lastDotIndex]
224        className = qualifiedName[lastDotIndex + 1:]
225      else:
226        packageName = ""
227        className = qualifiedName
228      options = fileFinder.findIname(className + ".*")
229      possibleContainingFiles = sorted(options)
230      result = None
231      for f in possibleContainingFiles:
232        if self.getPackage(f) == packageName:
233          result = f
234          break
235      self.classToFile_cache[qualifiedName] = result
236    return self.classToFile_cache[qualifiedName]
237
238  def getPackage(self, filePath):
239    prefix = "package "
240    with open(filePath) as f:
241      for line in f:
242        line = line.strip()
243        if line.startswith(prefix):
244          suffix = line[len(prefix):]
245          if suffix.endswith(";"):
246            return suffix[:-1]
247          return suffix
248    return None
249
250classFinder = ClassFinder()
251
252def readFile(path):
253  f = open(path)
254  text = f.read()
255  f.close()
256  return text
257
258def writeFile(path, text):
259  f = open(path, "w")
260  f.write(text)
261  f.close()
262
263def extractIndent(text):
264  indentSize = 0
265  for c in text:
266    if c == " ":
267      indentSize += 1
268    else:
269      break
270  return " " * indentSize
271
272class SourceFile(object):
273  """An in-memory model of a source file (java, kotlin) that can be manipulated and saved"""
274  def __init__(self, path):
275    text = readFile(path)
276    self.lines = text.split("\n")
277    self.path = path
278
279  def isKotlin(self):
280    return self.path.endswith(".kt")
281
282  def maybeSemicolon(self):
283    if self.isKotlin():
284      return ""
285    return ";"
286
287  def addAnnotation(self, methodLineNumber, annotation):
288    parenIndex = annotation.find("(")
289    if parenIndex > 0:
290      baseName = annotation[:parenIndex]
291    else:
292      baseName = annotation
293    if self.findAnnotationLine(methodLineNumber, baseName) is not None:
294      # already have an annotation, don't need to add another
295      return
296    indent = extractIndent(self.lines[methodLineNumber])
297    self.insertLine(methodLineNumber, indent + annotation)
298
299  # Adds an import to this file
300  # Attempts to preserve alphabetical import ordering:
301  #  If two consecutive imports can be found such that one should precede this import and
302  #   one should follow this import, inserts between those two imports
303  #  Otherwise attempts to add this import after the last import or before the first import
304  # (Note that imports might be grouped into multiple blocks, each separated by a blank line)
305  def addImport(self, symbolText):
306    insertText = "import " + symbolText + self.maybeSemicolon()
307    if insertText in self.lines:
308      return # already added
309    # set of lines that the insertion could immediately precede
310    beforeLineNumbers = set()
311    # set of lines that the insertion could immediately follow
312    afterLineNumbers = set()
313    for i in range(len(self.lines)):
314      line = self.lines[i]
315      if line.startswith("import"):
316        # found an import. Should our import be before or after?
317        if insertText < line:
318          beforeLineNumbers.add(i)
319        else:
320          afterLineNumbers.add(i)
321    # search for two adjacent lines that the line can be inserted between
322    insertionLineNumber = None
323    for i in range(len(self.lines) - 1):
324      if i in afterLineNumbers and (i + 1) in beforeLineNumbers:
325        insertionLineNumber = i + 1
326        break
327    # search for a line we can insert after
328    if insertionLineNumber is None:
329      for i in range(len(self.lines) - 1):
330        if i in afterLineNumbers and (i + 1) not in afterLineNumbers:
331          insertionLineNumber = i + 1
332          break
333    # search for a line we can insert before
334    if insertionLineNumber is None:
335      for i in range(len(self.lines) - 1, 0, -1):
336        if i in beforeLineNumbers and (i - 1) not in beforeLineNumbers:
337          insertionLineNumber = i
338          break
339
340    if insertionLineNumber is not None:
341      self.insertLine(insertionLineNumber, insertText)
342
343  def insertLine(self, beforeLineNumber, text):
344    self.lines = self.lines[:beforeLineNumber] + [text] + self.lines[beforeLineNumber:]
345
346  def findAnnotationLine(self, methodLineNumber, annotationText):
347    lineNumber = methodLineNumber
348    while True:
349      if lineNumber < 0:
350        return None
351      if annotationText in self.lines[lineNumber]:
352        return lineNumber
353      if self.lines[lineNumber].strip() == "":
354        return None
355      lineNumber -= 1
356
357  def removeLine(self, index):
358    self.lines = self.lines[:index] + self.lines[index + 1:]
359
360  def hasAnnotation(self, methodLineNumber, annotation):
361    return self.findAnnotationLine(methodLineNumber, annotation) is not None
362
363  def save(self):
364    writeFile(self.path, "\n".join(self.lines))
365
366# converts from a List<TestFailure> to a FailuresDatabase containing LocatedFailure
367def locate(failures):
368  db = FailuresDatabase()
369  for failure in failures:
370    location = classFinder.findMethod(failure.qualifiedClassName, failure.methodName)
371    if location is not None:
372      db.add(LocatedFailure(failure, location, failure.bugId))
373    else:
374      message = "Could not locate " + str(failure.qualifiedClassName) + "#" + str(failure.methodName) + " for " + str(failure.bugId)
375      if failure.branchName != "aosp-androidx-main":
376        message += ", should be in " + failure.branchName
377      print(message)
378  return db
379
380# Given a FailureDatabase, disables all of the tests mentioned in it, by adding the appropriate
381# annotations:
382#  failures get annotated with @Ignore ,
383# Annotations link to the associated bug if possible
384def disable(failuresDatabase):
385  numUpdates = 0
386  failuresByPath = failuresDatabase.getAll()
387  for path, failuresAtPath in failuresByPath.items():
388    source = SourceFile(path)
389    addedIgnore = False
390    for failure in failuresAtPath:
391      lineNumber = failure.location.lineNumber
392      if source.hasAnnotation(lineNumber, "@Ignore"):
393        continue
394      bugId = failure.bugId
395      bugText = '"b/' + bugId + '"'
396      source.addAnnotation(lineNumber, "@Ignore(" + bugText + ")")
397      addedIgnore = True
398    if addedIgnore:
399      source.addImport("org.junit.Ignore")
400      source.save()
401      numUpdates += 1
402  print("Made " + str(numUpdates) + " updates")
403
404def commit():
405  print("Generating git commit per OWNERS file")
406  os.chdir(supportRoot)
407  commitMessage = """Autogenerated suppression of test failures
408
409This commit was created with the help of development/suppressFailingTests.py
410"""
411  shellRunner.run(["development/split_change_into_owners.sh", commitMessage])
412
413
414def main():
415  global logger
416  arguments = parser.parse_args()
417  if arguments.v:
418    logger = PrintLogger()
419  else:
420    logger = DisabledLogger()
421  failures = getFailureData()
422  if len(failures) < 1:
423    print("Found 0 failures")
424    return
425  locations = locate(failures)
426  disable(locations)
427  commit()
428
429if __name__ == "__main__":
430  main()
431
432