• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1
2#!/usr/bin/env python3
3#
4# Copyright © 2020 Google LLC
5#
6# Permission is hereby granted, free of charge, to any person obtaining a
7# copy of this software and associated documentation files (the "Software"),
8# to deal in the Software without restriction, including without limitation
9# the rights to use, copy, modify, merge, publish, distribute, sublicense,
10# and/or sell copies of the Software, and to permit persons to whom the
11# Software is furnished to do so, subject to the following conditions:
12#
13# The above copyright notice and this permission notice (including the next
14# paragraph) shall be included in all copies or substantial portions of the
15# Software.
16#
17# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
20# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23# IN THE SOFTWARE.
24
25import argparse
26import queue
27import re
28from serial_buffer import SerialBuffer
29import sys
30import threading
31
32
33class CrosServoRun:
34    def __init__(self, cpu, ec):
35        # Merged FIFO for the two serial buffers, fed by threads.
36        self.serial_queue = queue.Queue()
37        self.sentinel = object()
38        self.threads_done = 0
39
40        self.ec_ser = SerialBuffer(
41            ec, "results/serial-ec.txt", "R SERIAL-EC> ")
42        self.cpu_ser = SerialBuffer(
43            cpu, "results/serial.txt", "R SERIAL-CPU> ")
44
45        self.iter_feed_ec = threading.Thread(
46            target=self.iter_feed_queue, daemon=True, args=(self.ec_ser.lines(),))
47        self.iter_feed_ec.start()
48
49        self.iter_feed_cpu = threading.Thread(
50            target=self.iter_feed_queue, daemon=True, args=(self.cpu_ser.lines(),))
51        self.iter_feed_cpu.start()
52
53    # Feed lines from our serial queues into the merged queue, marking when our
54    # input is done.
55    def iter_feed_queue(self, it):
56        for i in it:
57            self.serial_queue.put(i)
58        self.serial_queue.put(sentinel)
59
60    # Return the next line from the queue, counting how many threads have
61    # terminated and joining when done
62    def get_serial_queue_line(self):
63        line = self.serial_queue.get()
64        if line == self.sentinel:
65            self.threads_done = self.threads_done + 1
66            if self.threads_done == 2:
67                self.iter_feed_cpu.join()
68                self.iter_feed_ec.join()
69        return line
70
71    # Returns an iterator for getting the next line.
72    def serial_queue_lines(self):
73        return iter(self.get_serial_queue_line, self.sentinel)
74
75    def ec_write(self, s):
76        print("W SERIAL-EC> %s" % s)
77        self.ec_ser.serial.write(s.encode())
78
79    def cpu_write(self, s):
80        print("W SERIAL-CPU> %s" % s)
81        self.cpu_ser.serial.write(s.encode())
82
83    def print_error(self, message):
84        RED = '\033[0;31m'
85        NO_COLOR = '\033[0m'
86        print(RED + message + NO_COLOR)
87
88    def run(self):
89        # Flush any partial commands in the EC's prompt, then ask for a reboot.
90        self.ec_write("\n")
91        self.ec_write("reboot\n")
92
93        # This is emitted right when the bootloader pauses to check for input.
94        # Emit a ^N character to request network boot, because we don't have a
95        # direct-to-netboot firmware on cheza.
96        for line in self.serial_queue_lines():
97            if re.search("load_archive: loading locale_en.bin", line):
98                self.cpu_write("\016")
99                break
100
101            # The Cheza boards have issues with failing to bring up power to
102            # the system sometimes, possibly dependent on ambient temperature
103            # in the farm.
104            if re.search("POWER_GOOD not seen in time", line):
105                self.print_error("Detected intermittent poweron failure, restarting run...")
106                return 2
107
108        tftp_failures = 0
109        for line in self.serial_queue_lines():
110            if re.search("---. end Kernel panic", line):
111                return 1
112
113            # The Cheza firmware seems to occasionally get stuck looping in
114            # this error state during TFTP booting, possibly based on amount of
115            # network traffic around it, but it'll usually recover after a
116            # reboot.
117            if re.search("R8152: Bulk read error 0xffffffbf", line):
118                tftp_failures += 1
119                if tftp_failures >= 100:
120                    self.print_error("Detected intermittent tftp failure, restarting run...")
121                    return 2
122
123            # There are very infrequent bus errors during power management transitions
124            # on cheza, which we don't expect to be the case on future boards.
125            if re.search("Kernel panic - not syncing: Asynchronous SError Interrupt", line):
126                self.print_error("Detected cheza power management bus error, restarting run...")
127                return 2
128
129            # If the network device dies, it's probably not graphics's fault, just try again.
130            if re.search("NETDEV WATCHDOG", line):
131                self.print_error(
132                    "Detected network device failure, restarting run...")
133                return 2
134
135            # These HFI response errors started appearing with the introduction
136            # of piglit runs.  CosmicPenguin says:
137            #
138            # "message ID 106 isn't a thing, so likely what happened is that we
139            # got confused when parsing the HFI queue.  If it happened on only
140            # one run, then memory corruption could be a possible clue"
141            #
142            # Given that it seems to trigger randomly near a GPU fault and then
143            # break many tests after that, just restart the whole run.
144            if re.search("a6xx_hfi_send_msg.*Unexpected message id .* on the response queue", line):
145                self.print_error("Detected cheza power management bus error, restarting run...")
146                return 2
147
148            if re.search("coreboot.*bootblock starting", line):
149                self.print_error(
150                    "Detected spontaneous reboot, restarting run...")
151                return 2
152
153            result = re.search("hwci: mesa: (\S*)", line)
154            if result:
155                if result.group(1) == "pass":
156                    return 0
157                else:
158                    return 1
159
160        self.print_error("Reached the end of the CPU serial log without finding a result")
161        return 1
162
163
164def main():
165    parser = argparse.ArgumentParser()
166    parser.add_argument('--cpu', type=str,
167                        help='CPU Serial device', required=True)
168    parser.add_argument(
169        '--ec', type=str, help='EC Serial device', required=True)
170    args = parser.parse_args()
171
172    servo = CrosServoRun(args.cpu, args.ec)
173
174    while True:
175        retval = servo.run()
176        if retval != 2:
177            break
178
179    # power down the CPU on the device
180    servo.ec_write("power off\n")
181
182    sys.exit(retval)
183
184
185if __name__ == '__main__':
186    main()
187