1 /* See COPYING.txt for the full license governing this code. */
2 /**
3 * \file testharness.c
4 *
5 * Source file for the test harness.
6 */
7
8 #include <stdlib.h>
9 #include <SDL_test.h>
10 #include <SDL.h>
11 #include <SDL_assert.h>
12 #include "SDL_visualtest_harness_argparser.h"
13 #include "SDL_visualtest_process.h"
14 #include "SDL_visualtest_variators.h"
15 #include "SDL_visualtest_screenshot.h"
16 #include "SDL_visualtest_mischelper.h"
17
18 #if defined(__WIN32__) && !defined(__CYGWIN__)
19 #include <direct.h>
20 #elif defined(__WIN32__) && defined(__CYGWIN__)
21 #include <signal.h>
22 #elif defined(__LINUX__)
23 #include <sys/stat.h>
24 #include <sys/types.h>
25 #include <signal.h>
26 #else
27 #error "Unsupported platform"
28 #endif
29
30 /** Code for the user event triggered when a new action is to be executed */
31 #define ACTION_TIMER_EVENT 0
32 /** Code for the user event triggered when the maximum timeout is reached */
33 #define KILL_TIMER_EVENT 1
34 /** FPS value used for delays in the action loop */
35 #define ACTION_LOOP_FPS 10
36
37 /** Value returned by RunSUTAndTest() when the test has passed */
38 #define TEST_PASSED 1
39 /** Value returned by RunSUTAndTest() when the test has failed */
40 #define TEST_FAILED 0
41 /** Value returned by RunSUTAndTest() on a fatal error */
42 #define TEST_ERROR -1
43
44 static SDL_ProcessInfo pinfo;
45 static SDL_ProcessExitStatus sut_exitstatus;
46 static SDLVisualTest_HarnessState state;
47 static SDLVisualTest_Variator variator;
48 static SDLVisualTest_ActionNode* current; /* the current action being performed */
49 static SDL_TimerID action_timer, kill_timer;
50
51 /* returns a char* to be passed as the format argument of a printf-style function. */
52 static char*
usage()53 usage()
54 {
55 return "Usage: \n%s --sutapp xyz"
56 " [--sutargs abc | --parameter-config xyz.parameters"
57 " [--variator exhaustive|random]"
58 " [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]"
59 " [--action-config xyz.actions]"
60 " [--output-dir /path/to/output]"
61 " [--verify-dir /path/to/verify]"
62 " or --config app.config";
63 }
64
65 /* register Ctrl+C handlers */
66 #if defined(__LINUX__) || defined(__CYGWIN__)
67 static void
CtrlCHandlerCallback(int signum)68 CtrlCHandlerCallback(int signum)
69 {
70 SDL_Event event;
71 SDLTest_Log("Ctrl+C received");
72 event.type = SDL_QUIT;
73 SDL_PushEvent(&event);
74 }
75 #endif
76
77 static Uint32
ActionTimerCallback(Uint32 interval,void * param)78 ActionTimerCallback(Uint32 interval, void* param)
79 {
80 SDL_Event event;
81 SDL_UserEvent userevent;
82 Uint32 next_action_time;
83
84 /* push an event to handle the action */
85 userevent.type = SDL_USEREVENT;
86 userevent.code = ACTION_TIMER_EVENT;
87 userevent.data1 = ¤t->action;
88 userevent.data2 = NULL;
89
90 event.type = SDL_USEREVENT;
91 event.user = userevent;
92 SDL_PushEvent(&event);
93
94 /* calculate the new interval and return it */
95 if(current->next)
96 next_action_time = current->next->action.time - current->action.time;
97 else
98 {
99 next_action_time = 0;
100 action_timer = 0;
101 }
102
103 current = current->next;
104 return next_action_time;
105 }
106
107 static Uint32
KillTimerCallback(Uint32 interval,void * param)108 KillTimerCallback(Uint32 interval, void* param)
109 {
110 SDL_Event event;
111 SDL_UserEvent userevent;
112
113 userevent.type = SDL_USEREVENT;
114 userevent.code = KILL_TIMER_EVENT;
115 userevent.data1 = NULL;
116 userevent.data2 = NULL;
117
118 event.type = SDL_USEREVENT;
119 event.user = userevent;
120 SDL_PushEvent(&event);
121
122 kill_timer = 0;
123 return 0;
124 }
125
126 static int
ProcessAction(SDLVisualTest_Action * action,int * sut_running,char * args)127 ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args)
128 {
129 if(!action || !sut_running)
130 return TEST_ERROR;
131
132 switch(action->type)
133 {
134 case SDL_ACTION_KILL:
135 SDLTest_Log("Action: Kill SUT");
136 if(SDL_IsProcessRunning(&pinfo) == 1 &&
137 !SDL_KillProcess(&pinfo, &sut_exitstatus))
138 {
139 SDLTest_LogError("SDL_KillProcess() failed");
140 return TEST_ERROR;
141 }
142 *sut_running = 0;
143 break;
144
145 case SDL_ACTION_QUIT:
146 SDLTest_Log("Action: Quit SUT");
147 if(SDL_IsProcessRunning(&pinfo) == 1 &&
148 !SDL_QuitProcess(&pinfo, &sut_exitstatus))
149 {
150 SDLTest_LogError("SDL_QuitProcess() failed");
151 return TEST_FAILED;
152 }
153 *sut_running = 0;
154 break;
155
156 case SDL_ACTION_LAUNCH:
157 {
158 char* path;
159 char* args;
160 SDL_ProcessInfo action_process;
161 SDL_ProcessExitStatus ps;
162
163 path = action->extra.process.path;
164 args = action->extra.process.args;
165 if(args)
166 {
167 SDLTest_Log("Action: Launch process: %s with arguments: %s",
168 path, args);
169 }
170 else
171 SDLTest_Log("Action: Launch process: %s", path);
172 if(!SDL_LaunchProcess(path, args, &action_process))
173 {
174 SDLTest_LogError("SDL_LaunchProcess() failed");
175 return TEST_ERROR;
176 }
177
178 /* small delay so that the process can do its job */
179 SDL_Delay(1000);
180
181 if(SDL_IsProcessRunning(&action_process) > 0)
182 {
183 SDLTest_LogError("Process %s took too long too complete."
184 " Force killing...", action->extra);
185 if(!SDL_KillProcess(&action_process, &ps))
186 {
187 SDLTest_LogError("SDL_KillProcess() failed");
188 return TEST_ERROR;
189 }
190 }
191 }
192 break;
193
194 case SDL_ACTION_SCREENSHOT:
195 {
196 char path[MAX_PATH_LEN], hash[33];
197
198 SDLTest_Log("Action: Take screenshot");
199 /* can't take a screenshot if the SUT isn't running */
200 if(SDL_IsProcessRunning(&pinfo) != 1)
201 {
202 SDLTest_LogError("SUT has quit.");
203 *sut_running = 0;
204 return TEST_FAILED;
205 }
206
207 /* file name for the screenshot image */
208 SDLVisualTest_HashString(args, hash);
209 SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash);
210 if(!SDLVisualTest_ScreenshotProcess(&pinfo, path))
211 {
212 SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed");
213 return TEST_ERROR;
214 }
215 }
216 break;
217
218 case SDL_ACTION_VERIFY:
219 {
220 int ret;
221
222 SDLTest_Log("Action: Verify screenshot");
223 ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir,
224 state.verify_dir);
225
226 if(ret == -1)
227 {
228 SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed");
229 return TEST_ERROR;
230 }
231 else if(ret == 0)
232 {
233 SDLTest_Log("Verification failed: Images were not equal.");
234 return TEST_FAILED;
235 }
236 else if(ret == 1)
237 SDLTest_Log("Verification successful.");
238 else
239 {
240 SDLTest_Log("Verfication skipped.");
241 return TEST_FAILED;
242 }
243 }
244 break;
245
246 default:
247 SDLTest_LogError("Invalid action type");
248 return TEST_ERROR;
249 break;
250 }
251
252 return TEST_PASSED;
253 }
254
255 static int
RunSUTAndTest(char * sutargs,int variation_num)256 RunSUTAndTest(char* sutargs, int variation_num)
257 {
258 int success, sut_running, return_code;
259 char hash[33];
260 SDL_Event event;
261
262 return_code = TEST_PASSED;
263
264 if(!sutargs)
265 {
266 SDLTest_LogError("sutargs argument cannot be NULL");
267 return_code = TEST_ERROR;
268 goto runsutandtest_cleanup_generic;
269 }
270
271 SDLVisualTest_HashString(sutargs, hash);
272 SDLTest_Log("Hash: %s", hash);
273
274 success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo);
275 if(!success)
276 {
277 SDLTest_Log("Could not launch SUT.");
278 return_code = TEST_ERROR;
279 goto runsutandtest_cleanup_generic;
280 }
281 SDLTest_Log("SUT launch successful.");
282 SDLTest_Log("Process will be killed in %d milliseconds", state.timeout);
283 sut_running = 1;
284
285 /* launch the timers */
286 SDLTest_Log("Performing actions..");
287 current = state.action_queue.front;
288 action_timer = 0;
289 kill_timer = 0;
290 if(current)
291 {
292 action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL);
293 if(!action_timer)
294 {
295 SDLTest_LogError("SDL_AddTimer() failed");
296 return_code = TEST_ERROR;
297 goto runsutandtest_cleanup_timer;
298 }
299 }
300 kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL);
301 if(!kill_timer)
302 {
303 SDLTest_LogError("SDL_AddTimer() failed");
304 return_code = TEST_ERROR;
305 goto runsutandtest_cleanup_timer;
306 }
307
308 /* the timer stops running if the actions queue is empty, and the
309 SUT stops running if it crashes or if we encounter a KILL/QUIT action */
310 while(sut_running)
311 {
312 /* process the actions by using an event queue */
313 while(SDL_PollEvent(&event))
314 {
315 if(event.type == SDL_USEREVENT)
316 {
317 if(event.user.code == ACTION_TIMER_EVENT)
318 {
319 SDLVisualTest_Action* action;
320
321 action = (SDLVisualTest_Action*)event.user.data1;
322
323 switch(ProcessAction(action, &sut_running, sutargs))
324 {
325 case TEST_PASSED:
326 break;
327
328 case TEST_FAILED:
329 return_code = TEST_FAILED;
330 goto runsutandtest_cleanup_timer;
331 break;
332
333 default:
334 SDLTest_LogError("ProcessAction() failed");
335 return_code = TEST_ERROR;
336 goto runsutandtest_cleanup_timer;
337 }
338 }
339 else if(event.user.code == KILL_TIMER_EVENT)
340 {
341 SDLTest_LogError("Maximum timeout reached. Force killing..");
342 return_code = TEST_FAILED;
343 goto runsutandtest_cleanup_timer;
344 }
345 }
346 else if(event.type == SDL_QUIT)
347 {
348 SDLTest_LogError("Received QUIT event. Testharness is quitting..");
349 return_code = TEST_ERROR;
350 goto runsutandtest_cleanup_timer;
351 }
352 }
353 SDL_Delay(1000/ACTION_LOOP_FPS);
354 }
355
356 SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status);
357 if(sut_exitstatus.exit_status == 0)
358 {
359 return_code = TEST_PASSED;
360 goto runsutandtest_cleanup_timer;
361 }
362 else
363 {
364 return_code = TEST_FAILED;
365 goto runsutandtest_cleanup_timer;
366 }
367
368 return_code = TEST_ERROR;
369 goto runsutandtest_cleanup_generic;
370
371 runsutandtest_cleanup_timer:
372 if(action_timer && !SDL_RemoveTimer(action_timer))
373 {
374 SDLTest_Log("SDL_RemoveTimer() failed");
375 return_code = TEST_ERROR;
376 }
377
378 if(kill_timer && !SDL_RemoveTimer(kill_timer))
379 {
380 SDLTest_Log("SDL_RemoveTimer() failed");
381 return_code = TEST_ERROR;
382 }
383 /* runsutandtest_cleanup_process: */
384 if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus))
385 {
386 SDLTest_Log("SDL_KillProcess() failed");
387 return_code = TEST_ERROR;
388 }
389 runsutandtest_cleanup_generic:
390 return return_code;
391 }
392
393 /** Entry point for testharness */
394 int
main(int argc,char * argv[])395 main(int argc, char* argv[])
396 {
397 int i, passed, return_code, failed;
398
399 /* freeing resources, linux style! */
400 return_code = 0;
401
402 if(argc < 2)
403 {
404 SDLTest_Log(usage(), argv[0]);
405 goto cleanup_generic;
406 }
407
408 #if defined(__LINUX__) || defined(__CYGWIN__)
409 signal(SIGINT, CtrlCHandlerCallback);
410 #endif
411
412 /* parse arguments */
413 if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state))
414 {
415 SDLTest_Log(usage(), argv[0]);
416 return_code = 1;
417 goto cleanup_generic;
418 }
419 SDLTest_Log("Parsed harness arguments successfully.");
420
421 /* initialize SDL */
422 if(SDL_Init(SDL_INIT_TIMER) == -1)
423 {
424 SDLTest_LogError("SDL_Init() failed.");
425 SDLVisualTest_FreeHarnessState(&state);
426 return_code = 1;
427 goto cleanup_harness_state;
428 }
429
430 /* create an output directory if none exists */
431 #if defined(__LINUX__) || defined(__CYGWIN__)
432 mkdir(state.output_dir, 0777);
433 #elif defined(__WIN32__)
434 _mkdir(state.output_dir);
435 #else
436 #error "Unsupported platform"
437 #endif
438
439 /* test with sutargs */
440 if(SDL_strlen(state.sutargs))
441 {
442 SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs);
443 if(!state.no_launch)
444 {
445 switch(RunSUTAndTest(state.sutargs, 0))
446 {
447 case TEST_PASSED:
448 SDLTest_Log("Status: PASSED");
449 break;
450
451 case TEST_FAILED:
452 SDLTest_Log("Status: FAILED");
453 break;
454
455 case TEST_ERROR:
456 SDLTest_LogError("Some error occurred while testing.");
457 return_code = 1;
458 goto cleanup_sdl;
459 break;
460 }
461 }
462 }
463
464 if(state.sut_config.num_options > 0)
465 {
466 char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ?
467 "RANDOM" : "EXHAUSTIVE";
468 if(state.num_variations > 0)
469 SDLTest_Log("Testing SUT with variator: %s for %d variations",
470 variator_name, state.num_variations);
471 else
472 SDLTest_Log("Testing SUT with variator: %s and ALL variations",
473 variator_name);
474 /* initialize the variator */
475 if(!SDLVisualTest_InitVariator(&variator, &state.sut_config,
476 state.variator_type, 0))
477 {
478 SDLTest_LogError("Could not initialize variator");
479 return_code = 1;
480 goto cleanup_sdl;
481 }
482
483 /* iterate through all the variations */
484 passed = 0;
485 failed = 0;
486 for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++)
487 {
488 char* args = SDLVisualTest_GetNextVariation(&variator);
489 if(!args)
490 break;
491 SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args);
492
493 if(!state.no_launch)
494 {
495 switch(RunSUTAndTest(args, i + 1))
496 {
497 case TEST_PASSED:
498 SDLTest_Log("Status: PASSED");
499 passed++;
500 break;
501
502 case TEST_FAILED:
503 SDLTest_Log("Status: FAILED");
504 failed++;
505 break;
506
507 case TEST_ERROR:
508 SDLTest_LogError("Some error occurred while testing.");
509 goto cleanup_variator;
510 break;
511 }
512 }
513 }
514 if(!state.no_launch)
515 {
516 /* report stats */
517 SDLTest_Log("Testing complete.");
518 SDLTest_Log("%d/%d tests passed.", passed, passed + failed);
519 }
520 goto cleanup_variator;
521 }
522
523 goto cleanup_sdl;
524
525 cleanup_variator:
526 SDLVisualTest_FreeVariator(&variator);
527 cleanup_sdl:
528 SDL_Quit();
529 cleanup_harness_state:
530 SDLVisualTest_FreeHarnessState(&state);
531 cleanup_generic:
532 return return_code;
533 }
534