1 //
2 // Copyright 2019 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // glmark2Benchmark:
7 // Runs the glmark2 benchmark.
8 //
9
10 #include <gtest/gtest.h>
11
12 #include <stdio.h>
13 #include <sstream>
14
15 #include "../perf_tests/third_party/perf/perf_result_reporter.h"
16 #include "ANGLEPerfTestArgs.h"
17 #include "common/platform.h"
18 #include "common/string_utils.h"
19 #include "common/system_utils.h"
20 #include "test_utils/angle_test_configs.h"
21 #include "test_utils/angle_test_instantiate.h"
22 #include "util/test_utils.h"
23
24 using namespace angle;
25
26 namespace
27 {
28
29 struct BenchmarkInfo
30 {
31 const char *glmark2Config;
32 const char *name;
33 };
34
35 // Each glmark2 scene is individually benchmarked. If glmark2 is run without a specific benchmark,
36 // it can produce an aggregate score, which is not interesting at the moment. Adding an empty
37 // string ("") to this list will enable a test where glmark2 is run with the default scenes and the
38 // score for each test as well as the overall score is output.
39 constexpr BenchmarkInfo kBenchmarks[] = {
40 {"build:use-vbo=false", "build"},
41 {"build:use-vbo=true", "build_vbo"},
42 {"texture:texture-filter=nearest", "texture_nearest"},
43 {"texture:texture-filter=linear", "texture_linear"},
44 {"texture:texture-filter=mipmap", "texture_mipmap"},
45 {"shading:shading=gouraud", "shading_gouraud"},
46 {"shading:shading=blinn-phong-inf", "shading_blinn_phong"},
47 {"shading:shading=phong", "shading_phong"},
48 {"shading:shading=cel", "shading_cel"},
49 {"bump:bump-render=high-poly", "bump_high_poly"},
50 {"bump:bump-render=normals", "bump_normals"},
51 {"bump:bump-render=height", "bump_height"},
52 {"effect2d:kernel=0,1,0;1,-4,1;0,1,0;", "effect2d_edge"},
53 {"effect2d:kernel=1,1,1,1,1;1,1,1,1,1;1,1,1,1,1;", "effect2d_blur"},
54 {"pulsar:light=false:quads=5:texture=false", "pulsar"},
55 {"desktop:blur-radius=5:effect=blur:passes=1:separable=true:windows=4", "desktop_blur"},
56 {"desktop:effect=shadow:windows=4", "desktop_shadow"},
57 {"buffer:columns=200:interleave=false:update-dispersion=0.9:update-fraction=0.5:update-method="
58 "map",
59 "buffer_map"},
60 {"buffer:columns=200:interleave=false:update-dispersion=0.9:update-fraction=0.5:update-method="
61 "subdata",
62 "buffer_subdata"},
63 {"buffer:columns=200:interleave=true:update-dispersion=0.9:update-fraction=0.5:update-method="
64 "map",
65 "buffer_map_interleave"},
66 {"ideas:speed=duration", "ideas"},
67 {"jellyfish", "jellyfish"},
68 {"terrain", "terrain"},
69 {"shadow", "shadow"},
70 {"refract", "refract"},
71 {"conditionals:fragment-steps=0:vertex-steps=0", "conditionals"},
72 {"conditionals:fragment-steps=5:vertex-steps=0", "conditionals_fragment"},
73 {"conditionals:fragment-steps=0:vertex-steps=5", "conditionals_vertex"},
74 {"function:fragment-complexity=low:fragment-steps=5", "function"},
75 {"function:fragment-complexity=medium:fragment-steps=5", "function_complex"},
76 {"loop:fragment-loop=false:fragment-steps=5:vertex-steps=5", "loop_no_fsloop"},
77 {"loop:fragment-steps=5:fragment-uniform=false:vertex-steps=5", "loop_no_uniform"},
78 {"loop:fragment-steps=5:fragment-uniform=true:vertex-steps=5", "loop"},
79 };
80
81 struct GLMark2TestParams : public PlatformParameters
82 {
83 BenchmarkInfo info;
84 };
85
operator <<(std::ostream & os,const GLMark2TestParams & params)86 std::ostream &operator<<(std::ostream &os, const GLMark2TestParams ¶ms)
87 {
88 os << static_cast<const PlatformParameters &>(params) << "_" << params.info.name;
89 return os;
90 }
91
92 class GLMark2Benchmark : public testing::TestWithParam<GLMark2TestParams>
93 {
94 public:
GLMark2Benchmark()95 GLMark2Benchmark()
96 {
97 switch (GetParam().getRenderer())
98 {
99 case EGL_PLATFORM_ANGLE_TYPE_D3D11_ANGLE:
100 mBackend = "d3d11";
101 break;
102 case EGL_PLATFORM_ANGLE_TYPE_OPENGLES_ANGLE:
103 mBackend = "gl";
104 break;
105 case EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE:
106 mBackend = "vulkan";
107 break;
108 case EGL_PLATFORM_ANGLE_TYPE_METAL_ANGLE:
109 mBackend = "metal";
110 break;
111 default:
112 break;
113 }
114 std::string story = GetParam().info.name;
115 mReporter = std::make_unique<perf_test::PerfResultReporter>("glmark2_" + mBackend, story);
116 mReporter->RegisterImportantMetric(".fps", "fps");
117 mReporter->RegisterImportantMetric(".score", "score");
118 }
119
run()120 void run()
121 {
122 // Only supported on Linux and Windows at the moment.
123 if (!IsLinux() && !IsWindows())
124 {
125 return;
126 }
127
128 const BenchmarkInfo benchmarkInfo = GetParam().info;
129 const char *benchmark = benchmarkInfo.glmark2Config;
130 const char *benchmarkName = benchmarkInfo.name;
131 bool completeRun = benchmark == nullptr || benchmark[0] == '\0';
132
133 Optional<std::string> cwd = GetCWD();
134
135 // Set the current working directory to the executable's, as the data path of glmark2 is
136 // set relative to that path.
137 std::string executableDir = GetExecutableDirectory();
138 SetCWD(executableDir.c_str());
139 SetEnvironmentVar("ANGLE_DEFAULT_PLATFORM", mBackend.c_str());
140
141 std::vector<const char *> args = {
142 "glmark2_angle",
143 };
144 if (OneFrame())
145 {
146 args.push_back("--validate");
147 }
148 if (!completeRun)
149 {
150 args.push_back("--benchmark");
151 args.push_back(benchmark);
152 fprintf(stderr, "Running benchmark: %s\n", benchmark);
153 }
154 args.push_back(nullptr);
155
156 ProcessHandle process(args, ProcessOutputCapture::StdoutOnly);
157 ASSERT_TRUE(process && process->started());
158 ASSERT_TRUE(process->finish());
159
160 // Restore the current working directory for the next tests.
161 if (cwd.valid())
162 {
163 SetCWD(cwd.value().c_str());
164 }
165
166 ASSERT_EQ(EXIT_SUCCESS, process->getExitCode());
167
168 if (!OneFrame())
169 {
170 std::string output = process->getStdout();
171 parseOutput(output, benchmarkName, completeRun);
172 }
173 }
174
175 private:
parseOutput(const std::string & output,const char * benchmarkName,bool completeRun)176 void parseOutput(const std::string &output, const char *benchmarkName, bool completeRun)
177 {
178 // Output is in the following format:
179 //
180 // =======================================================
181 // glmark2 2017.07
182 // =======================================================
183 // OpenGL Information
184 // GL_VENDOR: ...
185 // GL_RENDERER: ...
186 // GL_VERSION: ...
187 // Surface Config: ...
188 // Surface Size: ...
189 // =======================================================
190 // [test] config: FPS: uint FrameTime: float ms
191 // [test] config: Not Supported
192 // ...
193 // =======================================================
194 // glmark2 Score: uint
195 // =======================================================
196 //
197 // This function skips the header, prints one line for each test/config line where there's
198 // an FPS value, and finally prints the overall score.
199 std::istringstream glmark2Output(output);
200 std::string line;
201
202 // Forward any INFO: lines that may have been generated.
203 while (std::getline(glmark2Output, line) && BeginsWith(line, "INFO:"))
204 {
205 fprintf(stderr, "%s\n", line.c_str());
206 }
207
208 // Expect ==== at the top of the header
209 ASSERT_EQ('=', line[0]);
210
211 // Skip one line
212 std::getline(glmark2Output, line);
213
214 // Expect ==== in the middle of the header
215 std::getline(glmark2Output, line);
216 ASSERT_EQ('=', line[0]);
217
218 // Skip four lines
219 std::getline(glmark2Output, line);
220 std::getline(glmark2Output, line);
221 std::getline(glmark2Output, line);
222 std::getline(glmark2Output, line);
223
224 // The fourth line is the GL_VERSION. Expect it to include ANGLE, otherwise we are not
225 // running against ANGLE.
226 ASSERT_NE(line.find("ANGLE"), std::string::npos);
227
228 // Skip two Surface lines
229 std::getline(glmark2Output, line);
230 std::getline(glmark2Output, line);
231
232 // Expect ==== at the bottom of the header
233 std::getline(glmark2Output, line);
234 ASSERT_EQ('=', line[0]);
235
236 // Read configs until the top of the footer is reached
237 while (std::getline(glmark2Output, line) && line[0] != '=')
238 {
239 // Parse the line
240 std::istringstream lin(line);
241
242 std::string testName, testConfig;
243 lin >> testName >> testConfig;
244 EXPECT_TRUE(lin);
245
246 std::string fpsTag, frametimeTag;
247 size_t fps;
248 float frametime;
249
250 lin >> fpsTag >> fps >> frametimeTag >> frametime;
251
252 // If the line is not in `FPS: uint FrameTime: Float ms` format, the test is not
253 // supported. It will be skipped.
254 if (!lin)
255 {
256 continue;
257 }
258
259 EXPECT_EQ("FPS:", fpsTag);
260 EXPECT_EQ("FrameTime:", frametimeTag);
261
262 if (!completeRun)
263 {
264 mReporter->AddResult(".fps", fps);
265 }
266 }
267
268 // Get the score line: `glmark2 Score: uint`
269 std::string glmark2Tag, scoreTag;
270 size_t score;
271 glmark2Output >> glmark2Tag >> scoreTag >> score;
272 EXPECT_TRUE(glmark2Output);
273 EXPECT_EQ("glmark2", glmark2Tag);
274 EXPECT_EQ("Score:", scoreTag);
275
276 if (completeRun)
277 {
278 mReporter->AddResult(".score", score);
279 }
280 }
281
282 std::string mBackend = "invalid";
283 std::unique_ptr<perf_test::PerfResultReporter> mReporter;
284 };
285
TEST_P(GLMark2Benchmark,Run)286 TEST_P(GLMark2Benchmark, Run)
287 {
288 run();
289 }
290
CombineEGLPlatform(const GLMark2TestParams & in,EGLPlatformParameters eglParams)291 GLMark2TestParams CombineEGLPlatform(const GLMark2TestParams &in, EGLPlatformParameters eglParams)
292 {
293 GLMark2TestParams out = in;
294 out.eglParameters = eglParams;
295 return out;
296 }
297
CombineInfo(const GLMark2TestParams & in,BenchmarkInfo info)298 GLMark2TestParams CombineInfo(const GLMark2TestParams &in, BenchmarkInfo info)
299 {
300 GLMark2TestParams out = in;
301 out.info = info;
302 return out;
303 }
304
305 using namespace egl_platform;
306
307 std::vector<GLMark2TestParams> gTestsWithInfo =
308 CombineWithValues({GLMark2TestParams()}, kBenchmarks, CombineInfo);
309 std::vector<EGLPlatformParameters> gEGLPlatforms = {D3D11(), METAL(), OPENGLES(), VULKAN()};
310 std::vector<GLMark2TestParams> gTestsWithPlatform =
311 CombineWithValues(gTestsWithInfo, gEGLPlatforms, CombineEGLPlatform);
312
313 ANGLE_INSTANTIATE_TEST_ARRAY(GLMark2Benchmark, gTestsWithPlatform);
314
315 } // namespace
316