1 //
2 // Copyright 2019 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // glmark2Benchmark:
7 // Runs the glmark2 benchmark.
8 //
9
10 #include <gtest/gtest.h>
11
12 #include <stdio.h>
13 #include <sstream>
14
15 #include "../perf_tests/third_party/perf/perf_result_reporter.h"
16 #include "ANGLEPerfTestArgs.h"
17 #include "common/platform.h"
18 #include "common/system_utils.h"
19 #include "test_utils/angle_test_configs.h"
20 #include "test_utils/angle_test_instantiate.h"
21 #include "util/test_utils.h"
22
23 using namespace angle;
24
25 namespace
26 {
27
28 struct BenchmarkInfo
29 {
30 const char *glmark2Config;
31 const char *name;
32 };
33
34 // Each glmark2 scene is individually benchmarked. If glmark2 is run without a specific benchmark,
35 // it can produce an aggregate score, which is not interesting at the moment. Adding an empty
36 // string ("") to this list will enable a test where glmark2 is run with the default scenes and the
37 // score for each test as well as the overall score is output.
38 constexpr BenchmarkInfo kBenchmarks[] = {
39 {"build:use-vbo=false", "build"},
40 {"build:use-vbo=true", "build_vbo"},
41 {"texture:texture-filter=nearest", "texture_nearest"},
42 {"texture:texture-filter=linear", "texture_linear"},
43 {"texture:texture-filter=mipmap", "texture_mipmap"},
44 {"shading:shading=gouraud", "shading_gouraud"},
45 {"shading:shading=blinn-phong-inf", "shading_blinn_phong"},
46 {"shading:shading=phong", "shading_phong"},
47 {"shading:shading=cel", "shading_cel"},
48 {"bump:bump-render=high-poly", "bump_high_poly"},
49 {"bump:bump-render=normals", "bump_normals"},
50 {"bump:bump-render=height", "bump_height"},
51 {"effect2d:kernel=0,1,0;1,-4,1;0,1,0;", "effect2d_edge"},
52 {"effect2d:kernel=1,1,1,1,1;1,1,1,1,1;1,1,1,1,1;", "effect2d_blur"},
53 {"pulsar:light=false:quads=5:texture=false", "pulsar"},
54 {"desktop:blur-radius=5:effect=blur:passes=1:separable=true:windows=4", "desktop_blur"},
55 {"desktop:effect=shadow:windows=4", "desktop_shadow"},
56 {"buffer:columns=200:interleave=false:update-dispersion=0.9:update-fraction=0.5:update-method="
57 "map",
58 "buffer_map"},
59 {"buffer:columns=200:interleave=false:update-dispersion=0.9:update-fraction=0.5:update-method="
60 "subdata",
61 "buffer_subdata"},
62 {"buffer:columns=200:interleave=true:update-dispersion=0.9:update-fraction=0.5:update-method="
63 "map",
64 "buffer_map_interleave"},
65 {"ideas:speed=duration", "ideas"},
66 {"jellyfish", "jellyfish"},
67 {"terrain", "terrain"},
68 {"shadow", "shadow"},
69 {"refract", "refract"},
70 {"conditionals:fragment-steps=0:vertex-steps=0", "conditionals"},
71 {"conditionals:fragment-steps=5:vertex-steps=0", "conditionals_fragment"},
72 {"conditionals:fragment-steps=0:vertex-steps=5", "conditionals_vertex"},
73 {"function:fragment-complexity=low:fragment-steps=5", "function"},
74 {"function:fragment-complexity=medium:fragment-steps=5", "function_complex"},
75 {"loop:fragment-loop=false:fragment-steps=5:vertex-steps=5", "loop_no_fsloop"},
76 {"loop:fragment-steps=5:fragment-uniform=false:vertex-steps=5", "loop_no_uniform"},
77 {"loop:fragment-steps=5:fragment-uniform=true:vertex-steps=5", "loop"},
78 };
79
80 struct GLMark2TestParams : public PlatformParameters
81 {
82 BenchmarkInfo info;
83 };
84
operator <<(std::ostream & os,const GLMark2TestParams & params)85 std::ostream &operator<<(std::ostream &os, const GLMark2TestParams ¶ms)
86 {
87 os << static_cast<const PlatformParameters &>(params) << "_" << params.info.name;
88 return os;
89 }
90
91 class GLMark2Benchmark : public testing::TestWithParam<GLMark2TestParams>
92 {
93 public:
GLMark2Benchmark()94 GLMark2Benchmark()
95 {
96 switch (GetParam().getRenderer())
97 {
98 case EGL_PLATFORM_ANGLE_TYPE_D3D11_ANGLE:
99 mBackend = "d3d11";
100 break;
101 case EGL_PLATFORM_ANGLE_TYPE_OPENGLES_ANGLE:
102 mBackend = "gl";
103 break;
104 case EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE:
105 mBackend = "vulkan";
106 break;
107 default:
108 break;
109 }
110 std::string story = GetParam().info.name;
111 mReporter = std::make_unique<perf_test::PerfResultReporter>("glmark2_" + mBackend, story);
112 mReporter->RegisterImportantMetric(".fps", "fps");
113 mReporter->RegisterImportantMetric(".score", "score");
114 }
115
run()116 void run()
117 {
118 // Only supported on Linux and Windows at the moment.
119 if (!IsLinux() && !IsWindows())
120 {
121 return;
122 }
123
124 const BenchmarkInfo benchmarkInfo = GetParam().info;
125 const char *benchmark = benchmarkInfo.glmark2Config;
126 const char *benchmarkName = benchmarkInfo.name;
127 bool completeRun = benchmark == nullptr || benchmark[0] == '\0';
128
129 Optional<std::string> cwd = GetCWD();
130
131 // Set the current working directory to the executable's, as the data path of glmark2 is
132 // set relative to that path.
133 std::string executableDir = GetExecutableDirectory();
134 SetCWD(executableDir.c_str());
135 SetEnvironmentVar("ANGLE_DEFAULT_PLATFORM", mBackend.c_str());
136
137 std::vector<const char *> args = {
138 "glmark2_angle",
139 };
140 if (OneFrame())
141 {
142 args.push_back("--validate");
143 }
144 if (!completeRun)
145 {
146 args.push_back("--benchmark");
147 args.push_back(benchmark);
148 fprintf(stderr, "Running benchmark: %s\n", benchmark);
149 }
150 args.push_back(nullptr);
151
152 ProcessHandle process(args, true, false);
153 ASSERT_TRUE(process && process->started());
154 ASSERT_TRUE(process->finish());
155
156 // Restore the current working directory for the next tests.
157 if (cwd.valid())
158 {
159 SetCWD(cwd.value().c_str());
160 }
161
162 ASSERT_EQ(EXIT_SUCCESS, process->getExitCode());
163
164 if (!OneFrame())
165 {
166 std::string output = process->getStdout();
167 parseOutput(output, benchmarkName, completeRun);
168 }
169 }
170
171 private:
parseOutput(const std::string & output,const char * benchmarkName,bool completeRun)172 void parseOutput(const std::string &output, const char *benchmarkName, bool completeRun)
173 {
174 // Output is in the following format:
175 //
176 // =======================================================
177 // glmark2 2017.07
178 // =======================================================
179 // OpenGL Information
180 // GL_VENDOR: ...
181 // GL_RENDERER: ...
182 // GL_VERSION: ...
183 // =======================================================
184 // [test] config: FPS: uint FrameTime: float ms
185 // [test] config: Not Supported
186 // ...
187 // =======================================================
188 // glmark2 Score: uint
189 // =======================================================
190 //
191 // This function skips the header, prints one line for each test/config line where there's
192 // an FPS value, and finally prints the overall score.
193 std::istringstream glmark2Output(output);
194 std::string line;
195
196 // Expect ==== at the top of the header
197 std::getline(glmark2Output, line);
198 ASSERT_EQ('=', line[0]);
199
200 // Skip one line
201 std::getline(glmark2Output, line);
202
203 // Expect ==== in the middle of the header
204 std::getline(glmark2Output, line);
205 ASSERT_EQ('=', line[0]);
206
207 // Skip four lines
208 std::getline(glmark2Output, line);
209 std::getline(glmark2Output, line);
210 std::getline(glmark2Output, line);
211 std::getline(glmark2Output, line);
212
213 // The fourth line is the GL_VERSION. Expect it to include ANGLE, otherwise we are not
214 // running against ANGLE.
215 ASSERT_NE(line.find("ANGLE"), std::string::npos);
216
217 // Expect ==== at the bottom of the header
218 std::getline(glmark2Output, line);
219 ASSERT_EQ('=', line[0]);
220
221 // Read configs until the top of the footer is reached
222 while (std::getline(glmark2Output, line) && line[0] != '=')
223 {
224 // Parse the line
225 std::istringstream lin(line);
226
227 std::string testName, testConfig;
228 lin >> testName >> testConfig;
229 EXPECT_TRUE(lin);
230
231 std::string fpsTag, frametimeTag;
232 size_t fps;
233 float frametime;
234
235 lin >> fpsTag >> fps >> frametimeTag >> frametime;
236
237 // If the line is not in `FPS: uint FrameTime: Float ms` format, the test is not
238 // supported. It will be skipped.
239 if (!lin)
240 {
241 continue;
242 }
243
244 EXPECT_EQ("FPS:", fpsTag);
245 EXPECT_EQ("FrameTime:", frametimeTag);
246
247 if (!completeRun)
248 {
249 mReporter->AddResult(".fps", fps);
250 }
251 }
252
253 // Get the score line: `glmark2 Score: uint`
254 std::string glmark2Tag, scoreTag;
255 size_t score;
256 glmark2Output >> glmark2Tag >> scoreTag >> score;
257 EXPECT_TRUE(glmark2Output);
258 EXPECT_EQ("glmark2", glmark2Tag);
259 EXPECT_EQ("Score:", scoreTag);
260
261 if (completeRun)
262 {
263 mReporter->AddResult(".score", score);
264 }
265 }
266
267 std::string mBackend = "invalid";
268 std::unique_ptr<perf_test::PerfResultReporter> mReporter;
269 };
270
TEST_P(GLMark2Benchmark,Run)271 TEST_P(GLMark2Benchmark, Run)
272 {
273 run();
274 }
275
CombineEGLPlatform(const GLMark2TestParams & in,EGLPlatformParameters eglParams)276 GLMark2TestParams CombineEGLPlatform(const GLMark2TestParams &in, EGLPlatformParameters eglParams)
277 {
278 GLMark2TestParams out = in;
279 out.eglParameters = eglParams;
280 return out;
281 }
282
CombineInfo(const GLMark2TestParams & in,BenchmarkInfo info)283 GLMark2TestParams CombineInfo(const GLMark2TestParams &in, BenchmarkInfo info)
284 {
285 GLMark2TestParams out = in;
286 out.info = info;
287 return out;
288 }
289
290 using namespace egl_platform;
291
292 std::vector<GLMark2TestParams> gTestsWithInfo =
293 CombineWithValues({GLMark2TestParams()}, kBenchmarks, CombineInfo);
294 std::vector<EGLPlatformParameters> gEGLPlatforms = {D3D11(), OPENGLES(), VULKAN()};
295 std::vector<GLMark2TestParams> gTestsWithPlatform =
296 CombineWithValues(gTestsWithInfo, gEGLPlatforms, CombineEGLPlatform);
297
298 ANGLE_INSTANTIATE_TEST_ARRAY(GLMark2Benchmark, gTestsWithPlatform);
299
300 } // namespace
301