1 /*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "api/test/metrics/print_result_proxy_metrics_exporter.h"
11
12 #include <map>
13 #include <string>
14 #include <vector>
15
16 #include "api/test/metrics/metric.h"
17 #include "api/units/timestamp.h"
18 #include "test/gmock.h"
19 #include "test/gtest.h"
20
21 namespace webrtc {
22 namespace test {
23 namespace {
24
25 using ::testing::TestWithParam;
26
DefaultMetadata()27 std::map<std::string, std::string> DefaultMetadata() {
28 return std::map<std::string, std::string>{{"key", "value"}};
29 }
30
Sample(double value)31 Metric::TimeSeries::Sample Sample(double value) {
32 return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1),
33 .value = value,
34 .sample_metadata = DefaultMetadata()};
35 }
36
TEST(PrintResultProxyMetricsExporterTest,ExportMetricsWithTimeSeriesFormatCorrect)37 TEST(PrintResultProxyMetricsExporterTest,
38 ExportMetricsWithTimeSeriesFormatCorrect) {
39 Metric metric1{
40 .name = "test_metric1",
41 .unit = Unit::kMilliseconds,
42 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
43 .test_case = "test_case_name1",
44 .metric_metadata = DefaultMetadata(),
45 .time_series =
46 Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}},
47 .stats =
48 Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
49 Metric metric2{
50 .name = "test_metric2",
51 .unit = Unit::kKilobitsPerSecond,
52 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
53 .test_case = "test_case_name2",
54 .metric_metadata = DefaultMetadata(),
55 .time_series =
56 Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}},
57 .stats = Metric::Stats{
58 .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}};
59
60 testing::internal::CaptureStdout();
61 PrintResultProxyMetricsExporter exporter;
62
63 std::string expected =
64 "RESULT test_metric1: test_case_name1= {15,5} "
65 "msBestFitFormat_biggerIsBetter\n"
66 "RESULT test_metric2: test_case_name2= {3750,1250} "
67 "bytesPerSecond_smallerIsBetter\n";
68
69 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2}));
70 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
71 }
72
TEST(PrintResultProxyMetricsExporterTest,ExportMetricsTimeSeriesOfSingleValueBackwardCompatibleFormat)73 TEST(PrintResultProxyMetricsExporterTest,
74 ExportMetricsTimeSeriesOfSingleValueBackwardCompatibleFormat) {
75 // This should be printed as {mean, stddev} despite only being a single data
76 // point.
77 Metric metric1{
78 .name = "available_send_bandwidth",
79 .unit = Unit::kKilobitsPerSecond,
80 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
81 .test_case = "test_case/alice",
82 .metric_metadata = DefaultMetadata(),
83 .time_series = Metric::TimeSeries{.samples = std::vector{Sample(1000)}},
84 .stats = Metric::Stats{
85 .mean = 1000.0, .stddev = 0.0, .min = 1000.0, .max = 1000.0}};
86 // This is a per-call metric that shouldn't have a stddev estimate.
87 Metric metric2{
88 .name = "min_psnr_dB",
89 .unit = Unit::kUnitless,
90 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
91 .test_case = "test_case/alice-video",
92 .metric_metadata = DefaultMetadata(),
93 .time_series = Metric::TimeSeries{.samples = std::vector{Sample(10)}},
94 .stats =
95 Metric::Stats{.mean = 10.0, .stddev = 0.0, .min = 10.0, .max = 10.0}};
96 // This is a per-call metric that shouldn't have a stddev estimate.
97 Metric metric3{
98 .name = "alice_connected",
99 .unit = Unit::kUnitless,
100 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
101 .test_case = "test_case",
102 .metric_metadata = DefaultMetadata(),
103 .time_series = Metric::TimeSeries{.samples = std::vector{Sample(1)}},
104 .stats =
105 Metric::Stats{.mean = 1.0, .stddev = 0.0, .min = 1.0, .max = 1.0}};
106
107 testing::internal::CaptureStdout();
108 PrintResultProxyMetricsExporter exporter;
109
110 std::string expected =
111 "RESULT available_send_bandwidth: test_case/alice= {125000,0} "
112 "bytesPerSecond_biggerIsBetter\n"
113 "RESULT min_psnr_dB: test_case/alice-video= 10 "
114 "unitless_biggerIsBetter\n"
115 "RESULT alice_connected: test_case= 1 "
116 "unitless_biggerIsBetter\n";
117
118 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2, metric3}));
119 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
120 }
121
TEST(PrintResultProxyMetricsExporterTest,ExportMetricsWithStatsOnlyFormatCorrect)122 TEST(PrintResultProxyMetricsExporterTest,
123 ExportMetricsWithStatsOnlyFormatCorrect) {
124 Metric metric1{.name = "test_metric1",
125 .unit = Unit::kMilliseconds,
126 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
127 .test_case = "test_case_name1",
128 .metric_metadata = DefaultMetadata(),
129 .time_series = Metric::TimeSeries{.samples = {}},
130 .stats = Metric::Stats{
131 .mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
132 Metric metric2{
133 .name = "test_metric2",
134 .unit = Unit::kKilobitsPerSecond,
135 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
136 .test_case = "test_case_name2",
137 .metric_metadata = DefaultMetadata(),
138 .time_series = Metric::TimeSeries{.samples = {}},
139 .stats = Metric::Stats{
140 .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}};
141
142 testing::internal::CaptureStdout();
143 PrintResultProxyMetricsExporter exporter;
144
145 std::string expected =
146 "RESULT test_metric1: test_case_name1= {15,5} "
147 "msBestFitFormat_biggerIsBetter\n"
148 "RESULT test_metric2: test_case_name2= {3750,1250} "
149 "bytesPerSecond_smallerIsBetter\n";
150
151 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2}));
152 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
153 }
154
TEST(PrintResultProxyMetricsExporterTest,ExportEmptyMetricOnlyFormatCorrect)155 TEST(PrintResultProxyMetricsExporterTest, ExportEmptyMetricOnlyFormatCorrect) {
156 Metric metric{.name = "test_metric",
157 .unit = Unit::kMilliseconds,
158 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
159 .test_case = "test_case_name",
160 .metric_metadata = DefaultMetadata(),
161 .time_series = Metric::TimeSeries{.samples = {}},
162 .stats = Metric::Stats{}};
163
164 testing::internal::CaptureStdout();
165 PrintResultProxyMetricsExporter exporter;
166
167 std::string expected =
168 "RESULT test_metric: test_case_name= 0 "
169 "msBestFitFormat_biggerIsBetter\n";
170
171 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
172 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
173 }
174
175 } // namespace
176 } // namespace test
177 } // namespace webrtc
178