1import re 2from dataclasses import dataclass 3from datetime import datetime, timedelta 4from enum import Enum, auto 5from os import getenv 6from typing import Optional, Pattern, Union 7 8from lava.utils.gitlab_section import GitlabSection 9 10 11class LogSectionType(Enum): 12 UNKNOWN = auto() 13 LAVA_SUBMIT = auto() 14 LAVA_QUEUE = auto() 15 LAVA_BOOT = auto() 16 TEST_DUT_SUITE = auto() 17 TEST_SUITE = auto() 18 TEST_CASE = auto() 19 LAVA_POST_PROCESSING = auto() 20 21# How long to wait whilst we try to submit a job; make it fairly short, 22# since the job will be retried. 23LAVA_SUBMIT_TIMEOUT = int(getenv("LAVA_SUBMIT_TIMEOUT", 5)) 24 25# How long should we wait for a device to become available? 26# For post-merge jobs, this should be ~infinite, but we can fail more 27# aggressively for pre-merge. 28LAVA_QUEUE_TIMEOUT = int(getenv("LAVA_QUEUE_TIMEOUT", 60)) 29 30# Empirically, successful device boot in LAVA time takes less than 3 31# minutes. 32# LAVA itself is configured to attempt thrice to boot the device, 33# summing up to 9 minutes. 34# It is better to retry the boot than cancel the job and re-submit to avoid 35# the enqueue delay. 36LAVA_BOOT_TIMEOUT = int(getenv("LAVA_BOOT_TIMEOUT", 9)) 37 38# Estimated overhead in minutes for a job from GitLab to reach the test phase, 39# including LAVA scheduling and boot duration 40LAVA_TEST_OVERHEAD_MIN = 5 41 42# Test DUT suite phase is where the initialization happens in DUT, not on docker. 43# The device will be listening to SSH session until the end of the job. 44LAVA_TEST_DUT_SUITE_TIMEOUT = int(getenv("CI_JOB_TIMEOUT")) // 60 - LAVA_TEST_OVERHEAD_MIN 45 46# Test suite phase is where the initialization happens on docker. 47LAVA_TEST_SUITE_TIMEOUT = int(getenv("LAVA_TEST_SUITE_TIMEOUT", 5)) 48 49# Test cases may take a long time, this script has no right to interrupt 50# them. But if the test case takes almost 1h, it will never succeed due to 51# Gitlab job timeout. 52LAVA_TEST_CASE_TIMEOUT = int(getenv("CI_JOB_TIMEOUT")) // 60 - LAVA_TEST_OVERHEAD_MIN 53 54# LAVA post processing may refer to a test suite teardown, or the 55# adjustments to start the next test_case 56LAVA_POST_PROCESSING_TIMEOUT = int(getenv("LAVA_POST_PROCESSING_TIMEOUT", 5)) 57 58FALLBACK_GITLAB_SECTION_TIMEOUT = timedelta(minutes=10) 59DEFAULT_GITLAB_SECTION_TIMEOUTS = { 60 LogSectionType.LAVA_SUBMIT: timedelta(minutes=LAVA_SUBMIT_TIMEOUT), 61 LogSectionType.LAVA_QUEUE: timedelta(minutes=LAVA_QUEUE_TIMEOUT), 62 LogSectionType.LAVA_BOOT: timedelta(minutes=LAVA_BOOT_TIMEOUT), 63 LogSectionType.TEST_DUT_SUITE: timedelta(minutes=LAVA_TEST_DUT_SUITE_TIMEOUT), 64 LogSectionType.TEST_SUITE: timedelta(minutes=LAVA_TEST_SUITE_TIMEOUT), 65 LogSectionType.TEST_CASE: timedelta(minutes=LAVA_TEST_CASE_TIMEOUT), 66 LogSectionType.LAVA_POST_PROCESSING: timedelta( 67 minutes=LAVA_POST_PROCESSING_TIMEOUT 68 ), 69} 70 71 72@dataclass(frozen=True) 73class LogSection: 74 regex: Union[Pattern, str] 75 levels: tuple[str] 76 section_id: str 77 section_header: str 78 section_type: LogSectionType 79 collapsed: bool = False 80 81 def from_log_line_to_section( 82 self, lava_log_line: dict[str, str], main_test_case: Optional[str], 83 timestamp_relative_to: Optional[datetime] 84 ) -> Optional[GitlabSection]: 85 if lava_log_line["lvl"] not in self.levels: 86 return 87 88 if match := re.search(self.regex, lava_log_line["msg"]): 89 section_id = self.section_id.format(*match.groups()) 90 section_header = self.section_header.format(*match.groups()) 91 is_main_test_case = section_id == main_test_case 92 timeout = DEFAULT_GITLAB_SECTION_TIMEOUTS[self.section_type] 93 return GitlabSection( 94 id=section_id, 95 header=f"{section_header} - Timeout: {timeout}", 96 type=self.section_type, 97 start_collapsed=self.collapsed, 98 suppress_start=is_main_test_case, 99 suppress_end=is_main_test_case, 100 timestamp_relative_to=timestamp_relative_to, 101 ) 102 103 104LOG_SECTIONS = ( 105 LogSection( 106 regex=re.compile(r"<?STARTTC>? ([^>]*)"), 107 levels=("target", "debug"), 108 section_id="{}", 109 section_header="test_case {}", 110 section_type=LogSectionType.TEST_CASE, 111 ), 112 LogSection( 113 regex=re.compile(r"<?STARTRUN>? ([^>]*ssh.*server.*)"), 114 levels=("debug"), 115 section_id="{}", 116 section_header="[dut] test_suite {}", 117 section_type=LogSectionType.TEST_DUT_SUITE, 118 ), 119 LogSection( 120 regex=re.compile(r"<?STARTRUN>? ([^>]*)"), 121 levels=("debug"), 122 section_id="{}", 123 section_header="[docker] test_suite {}", 124 section_type=LogSectionType.TEST_SUITE, 125 ), 126 LogSection( 127 regex=re.compile(r"ENDTC>? ([^>]+)"), 128 levels=("target", "debug"), 129 section_id="post-{}", 130 section_header="Post test_case {}", 131 collapsed=True, 132 section_type=LogSectionType.LAVA_POST_PROCESSING, 133 ), 134) 135