• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import re
2from dataclasses import dataclass
3from datetime import datetime, timedelta
4from enum import Enum, auto
5from os import getenv
6from typing import Optional, Pattern, Union
7
8from lava.utils.gitlab_section import GitlabSection
9
10
11class LogSectionType(Enum):
12    UNKNOWN = auto()
13    LAVA_SUBMIT = auto()
14    LAVA_QUEUE = auto()
15    LAVA_BOOT = auto()
16    TEST_DUT_SUITE = auto()
17    TEST_SUITE = auto()
18    TEST_CASE = auto()
19    LAVA_POST_PROCESSING = auto()
20
21# How long to wait whilst we try to submit a job; make it fairly short,
22# since the job will be retried.
23LAVA_SUBMIT_TIMEOUT = int(getenv("LAVA_SUBMIT_TIMEOUT", 5))
24
25# How long should we wait for a device to become available?
26# For post-merge jobs, this should be ~infinite, but we can fail more
27# aggressively for pre-merge.
28LAVA_QUEUE_TIMEOUT = int(getenv("LAVA_QUEUE_TIMEOUT", 60))
29
30# Empirically, successful device boot in LAVA time takes less than 3
31# minutes.
32# LAVA itself is configured to attempt thrice to boot the device,
33# summing up to 9 minutes.
34# It is better to retry the boot than cancel the job and re-submit to avoid
35# the enqueue delay.
36LAVA_BOOT_TIMEOUT = int(getenv("LAVA_BOOT_TIMEOUT", 9))
37
38# Estimated overhead in minutes for a job from GitLab to reach the test phase,
39# including LAVA scheduling and boot duration
40LAVA_TEST_OVERHEAD_MIN = 5
41
42# Test DUT suite phase is where the initialization happens in DUT, not on docker.
43# The device will be listening to SSH session until the end of the job.
44LAVA_TEST_DUT_SUITE_TIMEOUT = int(getenv("CI_JOB_TIMEOUT")) // 60 - LAVA_TEST_OVERHEAD_MIN
45
46# Test suite phase is where the initialization happens on docker.
47LAVA_TEST_SUITE_TIMEOUT = int(getenv("LAVA_TEST_SUITE_TIMEOUT", 5))
48
49# Test cases may take a long time, this script has no right to interrupt
50# them. But if the test case takes almost 1h, it will never succeed due to
51# Gitlab job timeout.
52LAVA_TEST_CASE_TIMEOUT = int(getenv("CI_JOB_TIMEOUT")) // 60 - LAVA_TEST_OVERHEAD_MIN
53
54# LAVA post processing may refer to a test suite teardown, or the
55# adjustments to start the next test_case
56LAVA_POST_PROCESSING_TIMEOUT = int(getenv("LAVA_POST_PROCESSING_TIMEOUT", 5))
57
58FALLBACK_GITLAB_SECTION_TIMEOUT = timedelta(minutes=10)
59DEFAULT_GITLAB_SECTION_TIMEOUTS = {
60    LogSectionType.LAVA_SUBMIT: timedelta(minutes=LAVA_SUBMIT_TIMEOUT),
61    LogSectionType.LAVA_QUEUE: timedelta(minutes=LAVA_QUEUE_TIMEOUT),
62    LogSectionType.LAVA_BOOT: timedelta(minutes=LAVA_BOOT_TIMEOUT),
63    LogSectionType.TEST_DUT_SUITE: timedelta(minutes=LAVA_TEST_DUT_SUITE_TIMEOUT),
64    LogSectionType.TEST_SUITE: timedelta(minutes=LAVA_TEST_SUITE_TIMEOUT),
65    LogSectionType.TEST_CASE: timedelta(minutes=LAVA_TEST_CASE_TIMEOUT),
66    LogSectionType.LAVA_POST_PROCESSING: timedelta(
67        minutes=LAVA_POST_PROCESSING_TIMEOUT
68    ),
69}
70
71
72@dataclass(frozen=True)
73class LogSection:
74    regex: Union[Pattern, str]
75    levels: tuple[str]
76    section_id: str
77    section_header: str
78    section_type: LogSectionType
79    collapsed: bool = False
80
81    def from_log_line_to_section(
82        self, lava_log_line: dict[str, str], main_test_case: Optional[str],
83        timestamp_relative_to: Optional[datetime]
84    ) -> Optional[GitlabSection]:
85        if lava_log_line["lvl"] not in self.levels:
86            return
87
88        if match := re.search(self.regex, lava_log_line["msg"]):
89            section_id = self.section_id.format(*match.groups())
90            section_header = self.section_header.format(*match.groups())
91            is_main_test_case = section_id == main_test_case
92            timeout = DEFAULT_GITLAB_SECTION_TIMEOUTS[self.section_type]
93            return GitlabSection(
94                id=section_id,
95                header=f"{section_header} - Timeout: {timeout}",
96                type=self.section_type,
97                start_collapsed=self.collapsed,
98                suppress_start=is_main_test_case,
99                suppress_end=is_main_test_case,
100                timestamp_relative_to=timestamp_relative_to,
101            )
102
103
104LOG_SECTIONS = (
105    LogSection(
106        regex=re.compile(r"<?STARTTC>? ([^>]*)"),
107        levels=("target", "debug"),
108        section_id="{}",
109        section_header="test_case {}",
110        section_type=LogSectionType.TEST_CASE,
111    ),
112    LogSection(
113        regex=re.compile(r"<?STARTRUN>? ([^>]*ssh.*server.*)"),
114        levels=("debug"),
115        section_id="{}",
116        section_header="[dut] test_suite {}",
117        section_type=LogSectionType.TEST_DUT_SUITE,
118    ),
119    LogSection(
120        regex=re.compile(r"<?STARTRUN>? ([^>]*)"),
121        levels=("debug"),
122        section_id="{}",
123        section_header="[docker] test_suite {}",
124        section_type=LogSectionType.TEST_SUITE,
125    ),
126    LogSection(
127        regex=re.compile(r"ENDTC>? ([^>]+)"),
128        levels=("target", "debug"),
129        section_id="post-{}",
130        section_header="Post test_case {}",
131        collapsed=True,
132        section_type=LogSectionType.LAVA_POST_PROCESSING,
133    ),
134)
135