1#!/usr/bin/env python3 2# 3# Copyright © 2020 Google LLC 4# SPDX-License-Identifier: MIT 5 6import argparse 7import re 8import sys 9 10from custom_logger import CustomLogger 11from serial_buffer import SerialBuffer 12 13 14class CrosServoRun: 15 def __init__(self, cpu, ec, test_timeout, logger): 16 self.cpu_ser = SerialBuffer( 17 cpu, "results/serial.txt", "R SERIAL-CPU> ") 18 # Merge the EC serial into the cpu_ser's line stream so that we can 19 # effectively poll on both at the same time and not have to worry about 20 self.ec_ser = SerialBuffer( 21 ec, "results/serial-ec.txt", "R SERIAL-EC> ", line_queue=self.cpu_ser.line_queue) 22 self.test_timeout = test_timeout 23 self.logger = logger 24 25 def close(self): 26 self.ec_ser.close() 27 self.cpu_ser.close() 28 29 def ec_write(self, s): 30 print("W SERIAL-EC> %s" % s) 31 self.ec_ser.serial.write(s.encode()) 32 33 def cpu_write(self, s): 34 print("W SERIAL-CPU> %s" % s) 35 self.cpu_ser.serial.write(s.encode()) 36 37 def print_error(self, message): 38 RED = '\033[0;31m' 39 NO_COLOR = '\033[0m' 40 print(RED + message + NO_COLOR) 41 self.logger.update_status_fail(message) 42 43 def run(self): 44 # Flush any partial commands in the EC's prompt, then ask for a reboot. 45 self.ec_write("\n") 46 self.ec_write("reboot\n") 47 48 bootloader_done = False 49 self.logger.create_job_phase("boot") 50 tftp_failures = 0 51 # This is emitted right when the bootloader pauses to check for input. 52 # Emit a ^N character to request network boot, because we don't have a 53 # direct-to-netboot firmware on cheza. 54 for line in self.cpu_ser.lines(timeout=120, phase="bootloader"): 55 if re.search("load_archive: loading locale_en.bin", line): 56 self.cpu_write("\016") 57 bootloader_done = True 58 break 59 60 # The Cheza firmware seems to occasionally get stuck looping in 61 # this error state during TFTP booting, possibly based on amount of 62 # network traffic around it, but it'll usually recover after a 63 # reboot. Currently mostly visible on google-freedreno-cheza-14. 64 if re.search("R8152: Bulk read error 0xffffffbf", line): 65 tftp_failures += 1 66 if tftp_failures >= 10: 67 self.print_error( 68 "Detected intermittent tftp failure, restarting run.") 69 return 1 70 71 # If the board has a netboot firmware and we made it to booting the 72 # kernel, proceed to processing of the test run. 73 if re.search("Booting Linux", line): 74 bootloader_done = True 75 break 76 77 # The Cheza boards have issues with failing to bring up power to 78 # the system sometimes, possibly dependent on ambient temperature 79 # in the farm. 80 if re.search("POWER_GOOD not seen in time", line): 81 self.print_error( 82 "Detected intermittent poweron failure, abandoning run.") 83 return 1 84 85 if not bootloader_done: 86 self.print_error("Failed to make it through bootloader, abandoning run.") 87 return 1 88 89 self.logger.create_job_phase("test") 90 for line in self.cpu_ser.lines(timeout=self.test_timeout, phase="test"): 91 if re.search("---. end Kernel panic", line): 92 return 1 93 94 # There are very infrequent bus errors during power management transitions 95 # on cheza, which we don't expect to be the case on future boards. 96 if re.search("Kernel panic - not syncing: Asynchronous SError Interrupt", line): 97 self.print_error( 98 "Detected cheza power management bus error, abandoning run.") 99 return 1 100 101 # If the network device dies, it's probably not graphics's fault, just try again. 102 if re.search("NETDEV WATCHDOG", line): 103 self.print_error( 104 "Detected network device failure, abandoning run.") 105 return 1 106 107 # These HFI response errors started appearing with the introduction 108 # of piglit runs. CosmicPenguin says: 109 # 110 # "message ID 106 isn't a thing, so likely what happened is that we 111 # got confused when parsing the HFI queue. If it happened on only 112 # one run, then memory corruption could be a possible clue" 113 # 114 # Given that it seems to trigger randomly near a GPU fault and then 115 # break many tests after that, just restart the whole run. 116 if re.search("a6xx_hfi_send_msg.*Unexpected message id .* on the response queue", line): 117 self.print_error( 118 "Detected cheza power management bus error, abandoning run.") 119 return 1 120 121 if re.search("coreboot.*bootblock starting", line): 122 self.print_error( 123 "Detected spontaneous reboot, abandoning run.") 124 return 1 125 126 if re.search("arm-smmu 5040000.iommu: TLB sync timed out -- SMMU may be deadlocked", line): 127 self.print_error("Detected cheza MMU fail, abandoning run.") 128 return 1 129 130 result = re.search("hwci: mesa: (\S*)", line) 131 if result: 132 if result.group(1) == "pass": 133 self.logger.update_dut_job("status", "pass") 134 return 0 135 else: 136 self.logger.update_status_fail("test fail") 137 return 1 138 139 self.print_error( 140 "Reached the end of the CPU serial log without finding a result") 141 return 1 142 143 144def main(): 145 parser = argparse.ArgumentParser() 146 parser.add_argument('--cpu', type=str, 147 help='CPU Serial device', required=True) 148 parser.add_argument( 149 '--ec', type=str, help='EC Serial device', required=True) 150 parser.add_argument( 151 '--test-timeout', type=int, help='Test phase timeout (minutes)', required=True) 152 args = parser.parse_args() 153 154 logger = CustomLogger("job_detail.json") 155 logger.update_dut_time("start", None) 156 servo = CrosServoRun(args.cpu, args.ec, args.test_timeout * 60, logger) 157 retval = servo.run() 158 159 # power down the CPU on the device 160 servo.ec_write("power off\n") 161 logger.update_dut_time("end", None) 162 servo.close() 163 164 sys.exit(retval) 165 166 167if __name__ == '__main__': 168 main() 169