• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4# Copyright (c) 2024 Huawei Device Co., Ltd.
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18from __future__ import annotations
19import logging
20import statistics
21import inspect
22import csv
23import math
24from collections import namedtuple, defaultdict
25from dataclasses import dataclass, field
26from typing import List, Dict, Any, Optional, Union
27from operator import add
28from pathlib import Path
29
30from vmb.helpers import Jsonable, StringEnum
31
32Stat = namedtuple("Stat",
33                  "runs ir_mem local_mem time")
34AotPasses = Dict[str, Stat]
35log = logging.getLogger('vmb')
36
37
38class BUStatus(StringEnum):
39    """Enum for distinguish status of Unit."""
40
41    NOT_RUN = 'Not Run'
42    SKIPPED = 'Skipped'
43    COMPILATION_FAILED = 'CompErr'
44    EXECUTION_FAILED = 'Failed'
45    ERROR = 'Error'
46    TIMEOUT = 'Timeout'
47    PASS = 'Passed'
48
49
50@dataclass
51class BuildResult(Jsonable):
52    """Metrics for compilation step."""
53
54    compiler: str
55    size: int = 0
56    time: float = 0.0
57    rss: int = 0
58    error: Optional[str] = None  # not used yet
59
60
61@dataclass
62class GCStats(Jsonable):
63    gc_avg_bytes_reclaimed: int = 0
64    gc_avg_interval_time: float = 0.0
65    gc_avg_time: float = 0.0
66    gc_max_time: int = 0
67    gc_median_time: int = 0
68    gc_memory_total_heap: int = 0
69    gc_min_time: int = 0
70    gc_name: str = ''
71    gc_pause_count: int = 0
72    gc_pauses: List[Any] = field(default_factory=list)
73    gc_pct50_time: int = 0
74    gc_pct95_time: int = 0
75    gc_pct99_time: int = 0
76    gc_std_time: float = 0.0
77    gc_total_bytes_reclaimed: int = 0
78    gc_total_time: float = 0.0
79    gc_total_time_sum: float = 0.0
80    gc_vm_time: float = 0.0
81    time_unit: str = 'ns'
82
83
84@dataclass
85class AOTStats(Jsonable):
86    number_of_methods: int = 0
87    passes: AotPasses = field(default_factory=dict)
88
89    @classmethod
90    def from_obj(cls, **kwargs):
91        kwargs = {
92            k: v for k, v in kwargs.items()
93            if k in inspect.signature(cls).parameters
94        }
95        for k, v in kwargs.items():
96            if 'passes' == k:
97                kwargs[k] = {n: Stat(*i) for n, i in v.items()}
98        return cls(**kwargs)
99
100    @classmethod
101    def from_csv(cls, csv_file: Union[str, Path]):
102        data: Dict[Any, Any] = {}
103        with open(csv_file, mode='r', encoding='utf-8', newline='\n') as f:
104            for method, pass_name, *stat, pbc_inst_num in csv.reader(
105                    f, delimiter=','):
106                if data.get(method) is None:
107                    data[method] = {
108                        "passes": defaultdict(list),
109                        "pbc_inst_num": pbc_inst_num
110                    }
111                data[method]["passes"][pass_name].append(
112                    [int(s) for s in stat])
113        # number of runs, ir mem, local mem, time
114        aot_stats = AOTStats(number_of_methods=len(data),
115                             passes=defaultdict(lambda: Stat(0, 0, 0, 0)))
116        for info in data.values():
117            for pass_name, stats in info["passes"].items():
118                sums = [0, 0, 0]
119                for values in stats:  # [[1,2,3],[1,2,3]] -> [2,4,6]
120                    sums = [sum(i) for i in zip(values, sums)]
121                stats_sum = [len(stats)] + sums
122                vals = map(add, stats_sum, list(aot_stats.passes[pass_name]))
123                aot_stats.passes[pass_name] = Stat(*vals)
124        return aot_stats
125
126
127@dataclass
128class JITStat(Jsonable):
129    method: str
130    is_osr: bool
131    bc_size: int
132    code_size: int
133    time: int
134
135    @staticmethod
136    def from_csv(csv_file: Union[str, Path]) -> List[JITStat]:
137        data: List[JITStat] = []
138        with open(csv_file, mode='r', encoding='utf-8', newline='\n') as f:
139            for method, is_osr, bc_size, code_size, time in csv.reader(
140                    f, delimiter=','):
141                data.append(JITStat(method,
142                                    bool(int(is_osr)),
143                                    int(bc_size),
144                                    int(code_size),
145                                    int(time)))
146        return data
147
148
149@dataclass
150class AOTStatsLib(Jsonable):
151    aot_stats: AOTStats = field(default_factory=AOTStats)
152    size: int = 0
153    time: float = 0.0
154
155    @classmethod
156    def from_obj(cls, **kwargs):
157        kwargs = {
158            k: v for k, v in kwargs.items()
159            if k in inspect.signature(cls).parameters
160        }
161        for k, v in kwargs.items():
162            if 'aot_stats' == k:
163                kwargs[k] = AOTStats.from_obj(**v) if v else AOTStats()
164        return cls(**kwargs)
165
166
167AotStatEntry = Dict[str, AOTStatsLib]
168ExtInfo = Dict[str, AotStatEntry]
169
170
171@dataclass
172class RunResult(Jsonable):
173    """This is data structure to hold result of parsing output of sort.
174
175    INFO - Startup execution started: 1703945979537
176    INFO - Tuning: 16777216 ops, 66.51878356933594 ns/op => 15033347 reps
177    INFO - Iter 1:15033347 ops, 95.58749625083489 ns/op
178    INFO - Benchmark result: DemoBench_Demo 95.58749625083489
179    """
180
181    avg_time: Optional[float] = 0.0
182    iterations: List[float] = field(default_factory=list)
183    warmup: List[float] = field(default_factory=list)
184    unit: str = 'ns/op'
185
186
187@dataclass
188class TestResult(Jsonable):
189    # meta info:
190    name: str
191    component: str = 'Doclet'
192    tags: List[str] = field(default_factory=list)
193    bugs: List[str] = field(default_factory=list)
194    # build stats:
195    compile_status: int = 0
196    build: List[BuildResult] = field(default_factory=list)
197    # exec stats:
198    execution_status: Optional[int] = None
199    execution_forks: List[RunResult] = field(default_factory=list)
200    mem_bytes: int = -1
201    gc_stats: Optional[GCStats] = None
202    aot_stats: Optional[AOTStats] = None
203    jit_stats: Optional[List[JITStat]] = None
204    # no-exportable:
205    _status: BUStatus = BUStatus.NOT_RUN
206    _ext_time: float = 0.0
207    __test__ = None
208
209    def __str__(self) -> str:
210        # Note: using avg, as it reported by test itself
211        # not `mean_time` which re-calculate by iteration results
212        if self._status == BUStatus.PASS:
213            time = self.get_avg_time()
214            return f'{time:.2e} | ' \
215                   f'{self.code_size:.2e} | ' \
216                   f'{self.mem_bytes:.2e} | {self._status.value:<7}'
217        return ' | '.join(['.' * 8] * 3 + [f'{self._status.value:<7}'])
218
219    @property
220    def code_size(self) -> int:
221        return sum(b.size for b in self.build)
222
223    @property
224    def compile_time(self) -> float:
225        return sum(b.time for b in self.build)
226
227    @property
228    def iterations_count(self) -> int:
229        if not self.execution_forks:
230            return 0
231        return sum(len(f.iterations) for f in self.execution_forks)
232
233    @property
234    def mean_time(self) -> Optional[float]:
235        if not self.execution_forks:
236            return None
237        if not self.execution_forks[0].iterations:
238            return None
239        return statistics.mean(self.execution_forks[0].iterations)
240
241    @property
242    def stdev_time(self) -> Optional[float]:
243        if not self.execution_forks:
244            return None
245        if not self.execution_forks[0].iterations:
246            return None
247        if len(self.execution_forks[0].iterations) < 2:
248            return None
249        stdev = statistics.stdev(self.execution_forks[0].iterations)
250        return stdev if stdev > 0 else 0.000001
251
252    @property
253    def mean_time_error_99(self) -> Optional[float]:
254        count = self.iterations_count
255        if count <= 2:
256            return None
257        stddev = self.stdev_time
258        if stddev == 0 or stddev is None:
259            return None
260        t_dist = [  # pre-calculated T-distribution coeff
261            0.0, 0.0, 63.66, 9.92, 5.84, 4.60, 4.03, 3.71, 3.50, 3.36,
262            3.25, 3.17, 3.11, 3.05, 3.01, 2.98, 2.95, 2.92, 2.90, 2.88, 2.86]
263        c = t_dist[count] if count <= len(t_dist) else 2.7
264        return c * stddev / math.sqrt(count)
265
266    @classmethod
267    def from_obj(cls, **kwargs):
268        kwargs = {  # skipping 'unknown' props
269            k: v for k, v in kwargs.items()
270            if k in inspect.signature(cls).parameters
271        }
272        for k, v in kwargs.items():
273            if 'build' == k:
274                kwargs[k] = [BuildResult(**i) for i in v]
275            if 'execution_forks' == k:
276                kwargs[k] = [RunResult(**i) for i in v]
277            if 'gc_stats' == k:
278                kwargs[k] = GCStats(**v) if v else None
279            if 'aot_stats' == k:
280                kwargs[k] = AOTStats(**v) if v else None
281            if 'jit_stats' == k:
282                kwargs[k] = [JITStat(**i) for i in v if i] if v else None
283        return cls(**kwargs)
284
285    def get_avg_time(self) -> float:
286        """Get mean of avg_time by executions."""
287        if not self.execution_forks:
288            return 0.0
289        return statistics.mean(f.avg_time for f in self.execution_forks
290                               if f.avg_time is not None)
291
292
293@dataclass
294class RunMeta(Jsonable):
295    start_time: str = ''
296    end_time: str = ''
297    framework_version: str = ''
298    job_url: str = ''
299    mr_change_id: str = ''
300    panda_commit_hash: str = ''
301    panda_commit_msg: str = ''
302
303
304@dataclass
305class MachineMeta(Jsonable):
306    name: str = ''
307    devid: str = ''
308    hardware: str = ''
309    os: str = ''
310
311
312@dataclass
313class RunReport(Jsonable):
314    ext_info: ExtInfo = field(default_factory=dict)
315    format_version: str = '2'
316    machine: MachineMeta = field(default_factory=MachineMeta)
317    run: RunMeta = field(default_factory=RunMeta)
318    tests: List[TestResult] = field(default_factory=list)
319
320    @classmethod
321    def from_obj(cls, **kwargs):
322        kwargs = {
323            k: v for k, v in kwargs.items()
324            if k in inspect.signature(cls).parameters
325        }
326        for k, v in kwargs.items():
327            if 'machine' == k:
328                kwargs[k] = MachineMeta(**v)
329            if 'run' == k:
330                kwargs[k] = RunMeta(**v)
331            if 'tests' == k:
332                kwargs[k] = [TestResult.from_obj(**i) for i in v]
333            if 'ext_info' == k:
334                ext = {}
335                for lib, info in v.items():
336                    ext[lib] = {n: AOTStatsLib.from_obj(**s)
337                                for n, s in info.items()}
338                kwargs[k] = ext
339        return cls(**kwargs)
340