1#!/usr/bin/env python3 2# 3# Copyright © 2020 Google LLC 4# 5# Permission is hereby granted, free of charge, to any person obtaining a 6# copy of this software and associated documentation files (the "Software"), 7# to deal in the Software without restriction, including without limitation 8# the rights to use, copy, modify, merge, publish, distribute, sublicense, 9# and/or sell copies of the Software, and to permit persons to whom the 10# Software is furnished to do so, subject to the following conditions: 11# 12# The above copyright notice and this permission notice (including the next 13# paragraph) shall be included in all copies or substantial portions of the 14# Software. 15# 16# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22# IN THE SOFTWARE. 23 24import argparse 25import subprocess 26import re 27from serial_buffer import SerialBuffer 28import sys 29import threading 30 31 32class FastbootRun: 33 def __init__(self, args, test_timeout): 34 self.powerup = args.powerup 35 self.ser = SerialBuffer( 36 args.dev, "results/serial-output.txt", "R SERIAL> ") 37 self.fastboot = "fastboot boot -s {ser} artifacts/fastboot.img".format( 38 ser=args.fbserial) 39 self.test_timeout = test_timeout 40 41 def close(self): 42 self.ser.close() 43 44 def print_error(self, message): 45 RED = '\033[0;31m' 46 NO_COLOR = '\033[0m' 47 print(RED + message + NO_COLOR) 48 49 def logged_system(self, cmd, timeout=60): 50 print("Running '{}'".format(cmd)) 51 try: 52 return subprocess.call(cmd, shell=True, timeout=timeout) 53 except subprocess.TimeoutExpired: 54 self.print_error("timeout, restarting run...") 55 return 2 56 57 def run(self): 58 if ret := self.logged_system(self.powerup): 59 return ret 60 61 fastboot_ready = False 62 for line in self.ser.lines(timeout=2 * 60, phase="bootloader"): 63 if re.search("fastboot: processing commands", line) or \ 64 re.search("Listening for fastboot command on", line): 65 fastboot_ready = True 66 break 67 68 if re.search("data abort", line): 69 self.print_error( 70 "Detected crash during boot, restarting run...") 71 return 2 72 73 if not fastboot_ready: 74 self.print_error( 75 "Failed to get to fastboot prompt, restarting run...") 76 return 2 77 78 if ret := self.logged_system(self.fastboot): 79 return ret 80 81 print_more_lines = -1 82 for line in self.ser.lines(timeout=self.test_timeout, phase="test"): 83 if print_more_lines == 0: 84 return 2 85 if print_more_lines > 0: 86 print_more_lines -= 1 87 88 if re.search("---. end Kernel panic", line): 89 return 1 90 91 # The db820c boards intermittently reboot. Just restart the run 92 # when if we see a reboot after we got past fastboot. 93 if re.search("PON REASON", line): 94 self.print_error( 95 "Detected spontaneous reboot, restarting run...") 96 return 2 97 98 # db820c sometimes wedges around iommu fault recovery 99 if re.search("watchdog: BUG: soft lockup - CPU.* stuck", line): 100 self.print_error( 101 "Detected kernel soft lockup, restarting run...") 102 return 2 103 104 # If the network device dies, it's probably not graphics's fault, just try again. 105 if re.search("NETDEV WATCHDOG", line): 106 self.print_error( 107 "Detected network device failure, restarting run...") 108 return 2 109 110 # A3xx recovery doesn't quite work. Sometimes the GPU will get 111 # wedged and recovery will fail (because power can't be reset?) 112 # This assumes that the jobs are sufficiently well-tested that GPU 113 # hangs aren't always triggered, so just try again. But print some 114 # more lines first so that we get better information on the cause 115 # of the hang. Once a hang happens, it's pretty chatty. 116 if "[drm:adreno_recover] *ERROR* gpu hw init failed: -22" in line: 117 self.print_error( 118 "Detected GPU hang, restarting run...") 119 if print_more_lines == -1: 120 print_more_lines = 30 121 122 result = re.search("hwci: mesa: (\S*)", line) 123 if result: 124 if result.group(1) == "pass": 125 return 0 126 else: 127 return 1 128 129 self.print_error( 130 "Reached the end of the CPU serial log without finding a result, restarting run...") 131 return 2 132 133 134def main(): 135 parser = argparse.ArgumentParser() 136 parser.add_argument( 137 '--dev', type=str, help='Serial device (otherwise reading from serial-output.txt)') 138 parser.add_argument('--powerup', type=str, 139 help='shell command for rebooting', required=True) 140 parser.add_argument('--powerdown', type=str, 141 help='shell command for powering off', required=True) 142 parser.add_argument('--fbserial', type=str, 143 help='fastboot serial number of the board', required=True) 144 parser.add_argument('--test-timeout', type=int, 145 help='Test phase timeout (minutes)', required=True) 146 args = parser.parse_args() 147 148 fastboot = FastbootRun(args, args.test_timeout * 60) 149 150 while True: 151 retval = fastboot.run() 152 fastboot.close() 153 if retval != 2: 154 break 155 156 fastboot = FastbootRun(args, args.test_timeout * 60) 157 158 fastboot.logged_system(args.powerdown) 159 160 sys.exit(retval) 161 162 163if __name__ == '__main__': 164 main() 165