1# Copyright 2023 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import json 6import pathlib 7import unittest 8 9from crossbench.probes.metric import CSVFormatter, Metric, MetricsMerger 10from tests import test_helper 11from tests.crossbench.base import CrossbenchFakeFsTestCase 12 13 14class FormatMetricTestCase(unittest.TestCase): 15 16 def test_no_stdev(self): 17 self.assertEqual(Metric.format(100), "100") 18 self.assertEqual(Metric.format(0), "0") 19 self.assertEqual(Metric.format(1.5), "1.5") 20 self.assertEqual(Metric.format(100, 0), "100") 21 self.assertEqual(Metric.format(0, 0), "0") 22 self.assertEqual(Metric.format(1.5, 0), "1.5") 23 24 def test_stdev(self): 25 self.assertEqual(Metric.format(100, 10), "100 ± 10%") 26 self.assertEqual(Metric.format(100, 1), "100.0 ± 1.0%") 27 self.assertEqual(Metric.format(100, 1.5), "100.0 ± 1.5%") 28 self.assertEqual(Metric.format(100, 0.1), "100.00 ± 0.10%") 29 self.assertEqual(Metric.format(100, 0.12), "100.00 ± 0.12%") 30 self.assertEqual(Metric.format(100, 0.125), "100.00 ± 0.12%") 31 32 def test_round_stdev(self): 33 value = 100.123456789 34 percent = value / 100 35 self.assertEqual(Metric.format(value, percent * 10.1234), "100 ± 10%") 36 self.assertEqual(Metric.format(value, percent * 1.2345), "100.1 ± 1.2%") 37 self.assertEqual(Metric.format(value, percent * 0.12345), "100.12 ± 0.12%") 38 self.assertEqual( 39 Metric.format(value, percent * 0.012345), "100.123 ± 0.012%") 40 self.assertEqual( 41 Metric.format(value, percent * 0.0012345), "100.1235 ± 0.0012%") 42 self.assertEqual( 43 Metric.format(value, percent * 0.00012345), "100.12346 ± 0.00012%") 44 45 46class MetricTestCase(unittest.TestCase): 47 48 def test_empty(self): 49 values = Metric() 50 self.assertTrue(values.is_numeric) 51 self.assertEqual(len(values), 0) 52 53 def test_is_numeric(self): 54 values = Metric([1, 2, 3, 4]) 55 self.assertTrue(values.is_numeric) 56 values.append(5) 57 self.assertTrue(values.is_numeric) 58 values.append("6") 59 self.assertFalse(values.is_numeric) 60 61 values = Metric([1, 2, 3, "4"]) 62 self.assertFalse(values.is_numeric) 63 64 def test_to_json_empty(self): 65 json_data = Metric().to_json() 66 self.assertDictEqual(json_data, {"values": []}) 67 68 def test_to_json_any(self): 69 json_data = Metric(["a", "b", "c"]).to_json() 70 self.assertDictEqual(json_data, {"values": ["a", "b", "c"]}) 71 72 def test_to_json_repeated(self): 73 json_data = Metric(["a", "a", "a"]).to_json() 74 self.assertEqual(json_data, "a") 75 76 def test_to_json_numeric_repeated(self): 77 json_data = Metric([1, 1, 1]).to_json() 78 self.assertListEqual(json_data["values"], [1, 1, 1]) 79 self.assertEqual(json_data["min"], 1) 80 self.assertEqual(json_data["max"], 1) 81 self.assertEqual(json_data["geomean"], 1) 82 self.assertEqual(json_data["average"], 1) 83 self.assertEqual(json_data["stddevPercent"], 0) 84 85 def test_to_json_numeric_average_0(self): 86 json_data = Metric([-1, 0, 1]).to_json() 87 self.assertListEqual(json_data["values"], [-1, 0, 1]) 88 self.assertEqual(json_data["min"], -1) 89 self.assertEqual(json_data["max"], 1) 90 self.assertEqual(json_data["geomean"], 0) 91 self.assertEqual(json_data["average"], 0) 92 self.assertEqual(json_data["stddevPercent"], 0) 93 94 95class MetricsMergerTestCase(CrossbenchFakeFsTestCase): 96 97 def test_empty(self): 98 merger = MetricsMerger() 99 self.assertDictEqual(merger.to_json(), {}) 100 self.assertListEqual(CSVFormatter(merger).table, []) 101 102 def test_add_flat(self): 103 input_data = {"a": 1, "b": 2} 104 merger = MetricsMerger() 105 merger.add(input_data) 106 data = merger.data 107 self.assertEqual(len(data), 2) 108 self.assertIsInstance(data["a"], Metric) 109 self.assertIsInstance(data["b"], Metric) 110 self.assertListEqual(data["a"].values, [1]) 111 self.assertListEqual(data["b"].values, [2]) 112 113 merger.add(input_data) 114 data = merger.data 115 self.assertEqual(len(data), 2) 116 self.assertListEqual(data["a"].values, [1, 1]) 117 self.assertListEqual(data["b"].values, [2, 2]) 118 119 def test_add_hierarchical(self): 120 input_data = { 121 "a": { 122 "a": { 123 "a": 1, 124 "b": 2 125 } 126 }, 127 "b": 2, 128 } 129 merger = MetricsMerger() 130 merger.add(input_data) 131 data = merger.data 132 self.assertListEqual(list(data.keys()), ["a/a/a", "a/a/b", "b"]) 133 self.assertIsInstance(data["a/a/a"], Metric) 134 self.assertIsInstance(data["a/a/b"], Metric) 135 self.assertIsInstance(data["b"], Metric) 136 137 def test_repeated_numeric(self): 138 merger = MetricsMerger() 139 input_data = { 140 "a": { 141 "aa": 1, 142 "ab": 2 143 }, 144 "b": 3, 145 "c": { 146 "cc": { 147 "ccc": 4 148 } 149 }, 150 } 151 merger.add(input_data) 152 merger.add(input_data) 153 data = merger.data 154 self.assertEqual(len(data), 4) 155 self.assertListEqual(data["a/aa"].values, [1, 1]) 156 self.assertListEqual(data["a/ab"].values, [2, 2]) 157 self.assertListEqual(data["b"].values, [3, 3]) 158 self.assertListEqual(data["c/cc/ccc"].values, [4, 4]) 159 160 BASIC_NESTED_DATA = { 161 "a": { 162 "a": { 163 "a": 1, 164 "b": 2 165 } 166 }, 167 "b": 3, 168 } 169 170 def test_custom_key_fn(self): 171 172 def under_join(segments): 173 return "_".join(segments) 174 175 merger = MetricsMerger(key_fn=under_join) 176 merger.add(self.BASIC_NESTED_DATA) 177 data = merger.data 178 self.assertListEqual(list(data.keys()), ["a_a_a", "a_a_b", "b"]) 179 180 def test_merge_serialized_same(self): 181 merger = MetricsMerger() 182 merger.add(self.BASIC_NESTED_DATA) 183 self.assertListEqual(list(merger.data.keys()), ["a/a/a", "a/a/b", "b"]) 184 path_a = pathlib.Path("merged_a.json") 185 path_b = pathlib.Path("merged_b.json") 186 with path_a.open("w", encoding="utf-8") as f: 187 json.dump(merger.to_json(), f) 188 with path_b.open("w", encoding="utf-8") as f: 189 json.dump(merger.to_json(), f) 190 191 merger = MetricsMerger.merge_json_list([path_a, path_b], 192 merge_duplicate_paths=True) 193 data = merger.data 194 self.assertListEqual(list(data.keys()), ["a/a/a", "a/a/b", "b"]) 195 self.assertListEqual(data["a/a/a"].values, [1, 1]) 196 self.assertListEqual(data["a/a/b"].values, [2, 2]) 197 self.assertListEqual(data["b"].values, [3, 3]) 198 199 # All duplicate entries are ignored 200 merger = MetricsMerger.merge_json_list([path_a, path_b], 201 merge_duplicate_paths=False) 202 self.assertListEqual(list(merger.data.keys()), []) 203 204 def test_merge_serialized_different_data(self): 205 merger_a = MetricsMerger({"a": {"a": 1}}) 206 merger_b = MetricsMerger({"a": {"b": 2}}) 207 path_a = pathlib.Path("merged_a.json") 208 path_b = pathlib.Path("merged_b.json") 209 with path_a.open("w", encoding="utf-8") as f: 210 json.dump(merger_a.to_json(), f) 211 with path_b.open("w", encoding="utf-8") as f: 212 json.dump(merger_b.to_json(), f) 213 214 merger = MetricsMerger.merge_json_list([path_a, path_b], 215 merge_duplicate_paths=True) 216 data = merger.data 217 self.assertListEqual(list(data.keys()), ["a/a", "a/b"]) 218 self.assertListEqual(data["a/a"].values, [1]) 219 self.assertListEqual(data["a/b"].values, [2]) 220 221 merger = MetricsMerger.merge_json_list([path_a, path_b], 222 merge_duplicate_paths=False) 223 data = merger.data 224 self.assertListEqual(list(data.keys()), ["a/a", "a/b"]) 225 226 def test_to_csv_no_path(self) -> None: 227 merger = MetricsMerger() 228 merger.add(self.BASIC_NESTED_DATA) 229 csv = CSVFormatter( 230 merger, lambda metric: metric.geomean, include_parts=False).table 231 self.assertListEqual(csv, [ 232 ("a/a/a", 1.0), 233 ("a/a/b", 2.0), 234 ("b", 3.0), 235 ]) 236 237 def test_to_csv_path(self) -> None: 238 merger = MetricsMerger() 239 merger.add(self.BASIC_NESTED_DATA) 240 csv = CSVFormatter( 241 merger, lambda metric: metric.geomean, include_parts=True).table 242 self.assertListEqual(csv, [ 243 ("a/a/a", "a", "a", "a", 1.0), 244 ("a/a/b", "a", "a", "b", 2.0), 245 ("b", "b", "", "", 3.0), 246 ]) 247 248 def test_to_csv_header(self) -> None: 249 merger = MetricsMerger() 250 merger.add({"a/b/c": 1, "d": 2}) 251 headers = [ 252 ("a", "custom", "header", "line"), 253 (1, 2, 3, 4, 5), 254 ] 255 csv = CSVFormatter( 256 merger, 257 lambda metric: metric.geomean, 258 headers=headers, 259 include_parts=True).table 260 self.assertListEqual(csv, [ 261 ("a", "", "", "", "custom", "header", "line"), 262 (1, "", "", "", 2, 3, 4, 5), 263 ("a/b/c", "a", "b", "c", 1.0), 264 ("d", "d", "", "", 2.0), 265 ]) 266 267 268class CSVFormatterTestCase(unittest.TestCase): 269 270 def test_format(self): 271 metrics = MetricsMerger({ 272 "Total/average": 10, 273 "Total/score": 20, 274 "cdjs/average": 30, 275 "cdjs/score": 40, 276 }) 277 table = CSVFormatter(metrics, lambda metric: metric.geomean).table 278 self.assertSequenceEqual(table, [ 279 ("Total/average", "Total", "average", 10.0), 280 ("Total/score", "Total", "score", 20.0), 281 ("cdjs/average", "cdjs", "average", 30.0), 282 ("cdjs/score", "cdjs", "score", 40.0), 283 ]) 284 285 286if __name__ == "__main__": 287 test_helper.run_pytest(__file__) 288