• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
2 // -*- Mode: C++ -*-
3 //
4 // Copyright (C) 2016-2022 Red Hat, Inc.
5 //
6 // Author: Dodji Seketeli
7 
8 /// @file
9 ///
10 /// This program runs abidiff between input files and checks that
11 /// the exit code of the abidiff is the one we expect.
12 ///
13 /// The set of input files and reference reports to consider should be
14 /// present in the source distribution.
15 
16 /// This is an aggregate that specifies where a test shall get its
17 /// input from and where it shall write its ouput to.
18 
19 #include <sys/wait.h>
20 #include <cstring>
21 #include <string>
22 #include <vector>
23 #include <fstream>
24 #include <iostream>
25 #include <cstdlib>
26 #include "abg-tools-utils.h"
27 #include "test-utils.h"
28 
29 using abigail::tools_utils::abidiff_status;
30 using abigail::tests::emit_test_status_and_update_counters;
31 using abigail::tests::emit_test_summary;
32 
33 struct InOutSpec
34 {
35   const char*	in_elfv0_path;
36   const char*	in_elfv1_path;
37   const char*	in_suppr_path;
38   const char*   in_elfv0_headers_dirs;
39   const char*   in_elfv1_headers_dirs;
40   const char*	abidiff_options;
41   abidiff_status status;
42   const char*	in_report_path;
43   const char*	out_report_path;
44 };// end struct InOutSpec;
45 
46 InOutSpec in_out_specs[] =
47 {
48   {
49     "data/test-abidiff-exit/test1-voffset-change-v0.o",
50     "data/test-abidiff-exit/test1-voffset-change-v1.o",
51     "",
52     "",
53     "",
54     "--no-default-suppression --no-show-locs",
55     abigail::tools_utils::ABIDIFF_ABI_CHANGE
56     | abigail::tools_utils::ABIDIFF_ABI_INCOMPATIBLE_CHANGE,
57     "data/test-abidiff-exit/test1-voffset-change-report0.txt",
58     "output/test-abidiff-exit/test1-voffset-change-report0.txt"
59   },
60   {
61     "data/test-abidiff-exit/test1-voffset-change-v0.o",
62     "data/test-abidiff-exit/test1-voffset-change-v1.o",
63     "data/test-abidiff-exit/test1-voffset-change.abignore",
64     "",
65     "",
66     "--no-default-suppression --no-show-locs",
67     abigail::tools_utils::ABIDIFF_OK,
68     "data/test-abidiff-exit/test1-voffset-change-report1.txt",
69     "output/test-abidiff-exit/test1-voffset-change-report1.txt"
70   },
71   {
72     "data/test-abidiff-exit/test2-filtered-removed-fns-v0.o",
73     "data/test-abidiff-exit/test2-filtered-removed-fns-v1.o",
74     "",
75     "",
76     "",
77     "--no-default-suppression --no-show-locs",
78     abigail::tools_utils::ABIDIFF_ABI_CHANGE
79     | abigail::tools_utils::ABIDIFF_ABI_INCOMPATIBLE_CHANGE,
80     "data/test-abidiff-exit/test2-filtered-removed-fns-report0.txt",
81     "output/test-abidiff-exit/test2-filtered-removed-fns-report0.txt"
82   },
83   {
84     "data/test-abidiff-exit/test2-filtered-removed-fns-v0.o",
85     "data/test-abidiff-exit/test2-filtered-removed-fns-v1.o",
86     "data/test-abidiff-exit/test2-filtered-removed-fns.abignore",
87     "",
88     "",
89     "--no-default-suppression --no-show-locs",
90     abigail::tools_utils::ABIDIFF_OK,
91     "data/test-abidiff-exit/test2-filtered-removed-fns-report1.txt",
92     "output/test-abidiff-exit/test2-filtered-removed-fns-report1.txt"
93   },
94   {
95     "data/test-abidiff-exit/test-loc-v0.bi",
96     "data/test-abidiff-exit/test-loc-v1.bi",
97     "",
98     "",
99     "",
100     "",
101     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
102     "data/test-abidiff-exit/test-loc-with-locs-report.txt",
103     "output/test-abidiff-exit/test-loc-with-locs-report.txt"
104   },
105   {
106     "data/test-abidiff-exit/test-loc-v0.bi",
107     "data/test-abidiff-exit/test-loc-v1.bi",
108     "",
109     "",
110     "",
111     "--no-show-locs",
112     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
113     "data/test-abidiff-exit/test-loc-without-locs-report.txt",
114     "output/test-abidiff-exit/test-loc-without-locs-report.txt"
115   },
116   {
117     "data/test-abidiff-exit/test-no-stray-comma-v0.o",
118     "data/test-abidiff-exit/test-no-stray-comma-v1.o",
119     "",
120     "",
121     "",
122     "--leaf-changes-only",
123     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
124     "data/test-abidiff-exit/test-no-stray-comma-report.txt",
125     "output/test-abidiff-exit/test-no-stray-comma-report.txt"
126   },
127   {
128     "data/test-abidiff-exit/test-leaf-stats-v0.o",
129     "data/test-abidiff-exit/test-leaf-stats-v1.o",
130     "",
131     "",
132     "",
133     "--no-show-locs --leaf-changes-only",
134     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
135     "data/test-abidiff-exit/test-leaf-stats-report.txt",
136     "output/test-abidiff-exit/test-leaf-stats-report.txt"
137   },
138   {
139     "data/test-abidiff-exit/test-leaf-more-v0.o",
140     "data/test-abidiff-exit/test-leaf-more-v1.o",
141     "",
142     "",
143     "",
144     "--no-show-locs --leaf-changes-only",
145     abigail::tools_utils::ABIDIFF_ABI_CHANGE
146     | abigail::tools_utils::ABIDIFF_ABI_INCOMPATIBLE_CHANGE,
147     "data/test-abidiff-exit/test-leaf-more-report.txt",
148     "output/test-abidiff-exit/test-leaf-more-report.txt"
149   },
150   {
151     "data/test-abidiff-exit/test-leaf-fun-type-v0.o",
152     "data/test-abidiff-exit/test-leaf-fun-type-v1.o",
153     "",
154     "",
155     "",
156     "--no-show-locs --leaf-changes-only",
157     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
158     "data/test-abidiff-exit/test-leaf-fun-type-report.txt",
159     "output/test-abidiff-exit/test-leaf-fun-type-report.txt"
160   },
161   {
162     "data/test-abidiff-exit/test-leaf-redundant-v0.o",
163     "data/test-abidiff-exit/test-leaf-redundant-v1.o",
164     "",
165     "",
166     "",
167     "--leaf-changes-only",
168     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
169     "data/test-abidiff-exit/test-leaf-redundant-report.txt",
170     "output/test-abidiff-exit/test-leaf-redundant-report.txt"
171   },
172   {
173     "data/test-abidiff-exit/test-leaf-peeling-v0.o",
174     "data/test-abidiff-exit/test-leaf-peeling-v1.o",
175     "",
176     "",
177     "",
178     "--leaf-changes-only",
179     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
180     "data/test-abidiff-exit/test-leaf-peeling-report.txt",
181     "output/test-abidiff-exit/test-leaf-peeling-report.txt"
182   },
183   {
184     "data/test-abidiff-exit/test-leaf-cxx-members-v0.o",
185     "data/test-abidiff-exit/test-leaf-cxx-members-v1.o",
186     "",
187     "",
188     "",
189     "--leaf-changes-only",
190     abigail::tools_utils::ABIDIFF_ABI_CHANGE
191     | abigail::tools_utils::ABIDIFF_ABI_INCOMPATIBLE_CHANGE,
192     "data/test-abidiff-exit/test-leaf-cxx-members-report.txt",
193     "output/test-abidiff-exit/test-leaf-cxx-members-report.txt"
194   },
195   {
196     "data/test-abidiff-exit/test-member-size-v0.o",
197     "data/test-abidiff-exit/test-member-size-v1.o",
198     "",
199     "",
200     "",
201     "",
202     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
203     "data/test-abidiff-exit/test-member-size-report0.txt",
204     "output/test-abidiff-exit/test-member-size-report0.txt"
205   },
206   {
207     "data/test-abidiff-exit/test-member-size-v0.o",
208     "data/test-abidiff-exit/test-member-size-v1.o",
209     "",
210     "",
211     "",
212     "--leaf-changes-only",
213     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
214     "data/test-abidiff-exit/test-member-size-report1.txt",
215     "output/test-abidiff-exit/test-member-size-report1.txt"
216   },
217   {
218     "data/test-abidiff-exit/test-decl-struct-v0.o",
219     "data/test-abidiff-exit/test-decl-struct-v1.o",
220     "",
221     "",
222     "",
223     "--harmless",
224     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
225     "data/test-abidiff-exit/test-decl-struct-report.txt",
226     "output/test-abidiff-exit/test-decl-struct-report.txt"
227   },
228   {
229     "data/test-abidiff-exit/test-fun-param-v0.abi",
230     "data/test-abidiff-exit/test-fun-param-v1.abi",
231     "",
232     "",
233     "",
234     "",
235     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
236     "data/test-abidiff-exit/test-fun-param-report.txt",
237     "output/test-abidiff-exit/test-fun-param-report.txt"
238   },
239   {
240     "data/test-abidiff-exit/test-decl-enum-v0.o",
241     "data/test-abidiff-exit/test-decl-enum-v1.o",
242     "",
243     "",
244     "",
245     "--harmless",
246     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
247     "data/test-abidiff-exit/test-decl-enum-report.txt",
248     "output/test-abidiff-exit/test-decl-enum-report.txt"
249   },
250   {
251     "data/test-abidiff-exit/test-decl-enum-v0.o",
252     "data/test-abidiff-exit/test-decl-enum-v1.o",
253     "",
254     "",
255     "",
256     "",
257     abigail::tools_utils::ABIDIFF_OK,
258     "data/test-abidiff-exit/test-decl-enum-report-2.txt",
259     "output/test-abidiff-exit/test-decl-enum-report-2.txt"
260   },
261   {
262     "data/test-abidiff-exit/test-decl-enum-v0.o",
263     "data/test-abidiff-exit/test-decl-enum-v1.o",
264     "",
265     "",
266     "",
267     "--leaf-changes-only",
268     abigail::tools_utils::ABIDIFF_OK,
269     "data/test-abidiff-exit/test-decl-enum-report-3.txt",
270     "output/test-abidiff-exit/test-decl-enum-report-3.txt"
271   },
272   {
273     "data/test-abidiff-exit/test-net-change-v0.o",
274     "data/test-abidiff-exit/test-net-change-v1.o",
275     "",
276     "",
277     "",
278     "--no-default-suppression --no-show-locs",
279     abigail::tools_utils::ABIDIFF_ABI_CHANGE
280     | abigail::tools_utils::ABIDIFF_ABI_INCOMPATIBLE_CHANGE,
281     "data/test-abidiff-exit/test-net-change-report0.txt",
282     "output/test-abidiff-exit/test-net-change-report0.txt"
283   },
284   {
285     "data/test-abidiff-exit/test-net-change-v0.o",
286     "data/test-abidiff-exit/test-net-change-v1.o",
287     "data/test-abidiff-exit/test-net-change.abignore",
288     "",
289     "",
290     "--no-default-suppression --no-show-locs",
291     abigail::tools_utils::ABIDIFF_OK,
292     "data/test-abidiff-exit/test-net-change-report1.txt",
293     "output/test-abidiff-exit/test-net-change-report1.txt"
294   },
295   {
296     "data/test-abidiff-exit/test-net-change-v0.o",
297     "data/test-abidiff-exit/test-net-change-v1.o",
298     "",
299     "",
300     "",
301     "--no-default-suppression --no-show-locs --leaf-changes-only",
302     abigail::tools_utils::ABIDIFF_ABI_CHANGE
303     | abigail::tools_utils::ABIDIFF_ABI_INCOMPATIBLE_CHANGE,
304     "data/test-abidiff-exit/test-net-change-report2.txt",
305     "output/test-abidiff-exit/test-net-change-report2.txt"
306   },
307   {
308     "data/test-abidiff-exit/test-net-change-v0.o",
309     "data/test-abidiff-exit/test-net-change-v1.o",
310     "data/test-abidiff-exit/test-net-change.abignore",
311     "",
312     "",
313     "--no-default-suppression --no-show-locs --leaf-changes-only",
314     abigail::tools_utils::ABIDIFF_OK,
315     "data/test-abidiff-exit/test-net-change-report3.txt",
316     "output/test-abidiff-exit/test-net-change-report3.txt"
317   },
318   {
319     "data/test-abidiff-exit/test-headers-dirs/test-headers-dir-v0.o",
320     "data/test-abidiff-exit/test-headers-dirs/test-headers-dir-v1.o",
321     "",
322     "data/test-abidiff-exit/test-headers-dirs/headers-a",
323     "data/test-abidiff-exit/test-headers-dirs/headers-a",
324     "--no-default-suppression",
325     abigail::tools_utils::ABIDIFF_OK,
326     "data/test-abidiff-exit/test-headers-dirs/test-headers-dir-report-1.txt",
327     "output/test-abidiff-exit/test-headers-dirs/test-headers-dir-report-1.txt"
328   },
329   {
330     "data/test-abidiff-exit/test-headers-dirs/test-headers-dir-v0.o",
331     "data/test-abidiff-exit/test-headers-dirs/test-headers-dir-v1.o",
332     "",
333     "data/test-abidiff-exit/test-headers-dirs/headers-a, "
334     "data/test-abidiff-exit/test-headers-dirs/headers-b",
335     "data/test-abidiff-exit/test-headers-dirs/headers-a, "
336     "data/test-abidiff-exit/test-headers-dirs/headers-b",
337     "--no-default-suppression",
338     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
339     "data/test-abidiff-exit/test-headers-dirs/test-headers-dir-report-2.txt",
340     "output/test-abidiff-exit/test-headers-dirs/test-headers-dir-report-2.txt"
341   },
342   {
343     "data/test-abidiff-exit/qualifier-typedef-array-v0.o",
344     "data/test-abidiff-exit/qualifier-typedef-array-v1.o",
345     "",
346     "",
347     "",
348     "",
349     abigail::tools_utils::ABIDIFF_OK,
350     "data/test-abidiff-exit/qualifier-typedef-array-report-0.txt",
351     "output/test-abidiff-exit/qualifier-typedef-array-report-0.txt"
352   },
353   {
354     "data/test-abidiff-exit/qualifier-typedef-array-v0.o",
355     "data/test-abidiff-exit/qualifier-typedef-array-v1.o",
356     "",
357     "",
358     "",
359     "--harmless",
360     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
361     "data/test-abidiff-exit/qualifier-typedef-array-report-1.txt",
362     "output/test-abidiff-exit/qualifier-typedef-array-report-1.txt"
363   },
364   {
365     "data/test-abidiff-exit/qualifier-typedef-array-v0.o",
366     "data/test-abidiff-exit/qualifier-typedef-array-v1.o",
367     "",
368     "",
369     "",
370     "--leaf-changes-only",
371     abigail::tools_utils::ABIDIFF_OK,
372     "data/test-abidiff-exit/qualifier-typedef-array-report-2.txt",
373     "output/test-abidiff-exit/qualifier-typedef-array-report-2.txt"
374   },
375   {
376     "data/test-abidiff-exit/qualifier-typedef-array-v0.o",
377     "data/test-abidiff-exit/qualifier-typedef-array-v1.o",
378     "",
379     "",
380     "",
381     "--harmless --leaf-changes-only",
382     abigail::tools_utils::ABIDIFF_OK,
383     "data/test-abidiff-exit/qualifier-typedef-array-report-3.txt",
384     "output/test-abidiff-exit/qualifier-typedef-array-report-3.txt"
385   },
386   {
387     "data/test-abidiff-exit/test-non-leaf-array-v0.o",
388     "data/test-abidiff-exit/test-non-leaf-array-v1.o",
389     "",
390     "",
391     "",
392     "--leaf-changes-only",
393     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
394     "data/test-abidiff-exit/test-non-leaf-array-report.txt",
395     "output/test-abidiff-exit/test-non-leaf-array-report.txt"
396   },
397   {
398     "data/test-abidiff-exit/test-crc-v0.abi",
399     "data/test-abidiff-exit/test-crc-v1.abi",
400     "",
401     "",
402     "",
403     "",
404     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
405     "data/test-abidiff-exit/test-crc-report.txt",
406     "output/test-abidiff-exit/test-crc-report.txt"
407   },
408   {
409     "data/test-abidiff-exit/test-missing-alias.abi",
410     "data/test-abidiff-exit/test-missing-alias.abi",
411     "data/test-abidiff-exit/test-missing-alias.suppr",
412     "",
413     "",
414     "",
415     abigail::tools_utils::ABIDIFF_OK,
416     "data/test-abidiff-exit/test-missing-alias-report.txt",
417     "output/test-abidiff-exit/test-missing-alias-report.txt"
418   },
419   {
420     "data/test-abidiff-exit/test-PR28316-v0.o",
421     "data/test-abidiff-exit/test-PR28316-v1.o",
422     "",
423     "",
424     "",
425     "--no-default-suppression --harmless",
426     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
427     "data/test-abidiff-exit/test-PR28316-report.txt",
428     "output/test-abidiff-exit/test-PR28316-report.txt"
429   },
430   {
431     "data/test-abidiff-exit/test-PR29144-v0.o",
432     "data/test-abidiff-exit/test-PR29144-v1.o",
433     "",
434     "",
435     "",
436     "--no-default-suppression --harmless",
437     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
438     "data/test-abidiff-exit/test-PR29144-report.txt",
439     "output/test-abidiff-exit/test-PR29144-report.txt"
440   },
441   {
442     "data/test-abidiff-exit/test-PR29144-v0.o",
443     "data/test-abidiff-exit/test-PR29144-v1.o",
444     "",
445     "",
446     "",
447     "--leaf-changes-only --no-default-suppression --harmless",
448     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
449     "data/test-abidiff-exit/test-PR29144-report-2.txt",
450     "output/test-abidiff-exit/test-PR29144-report-2.txt"
451   },
452   {
453     "data/test-abidiff-exit/ld-2.28-210.so",
454     "data/test-abidiff-exit/ld-2.28-211.so",
455     "",
456     "",
457     "",
458     "--no-default-suppression",
459     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
460     "data/test-abidiff-exit/test-ld-2.28-210.so--ld-2.28-211.so.txt",
461     "output/test-abidiff-exit/test-ld-2.28-210.so--ld-2.28-211.so.txt"
462   },
463   {
464     "data/test-abidiff-exit/test-rhbz2114909-v0.o",
465     "data/test-abidiff-exit/test-rhbz2114909-v1.o",
466     "",
467     "",
468     "",
469     "--no-default-suppression",
470     abigail::tools_utils::ABIDIFF_ABI_CHANGE,
471     "data/test-abidiff-exit/test-rhbz2114909-report-1.txt",
472     "output/test-abidiff-exit/test-rhbz2114909-report-1.txt"
473   },
474   {0, 0, 0 ,0, 0, 0, abigail::tools_utils::ABIDIFF_OK, 0, 0}
475 };
476 
477 /// Prefix the strings in a vector of string.
478 ///
479 /// @param strings the strings to prefix.
480 ///
481 /// @param prefix the prefix to use.
482 static void
do_prefix_strings(std::vector<std::string> & strings,const std::string & prefix)483 do_prefix_strings(std::vector<std::string> &strings,
484 		  const std::string& prefix)
485 {
486   for (std::vector<std::string>::size_type i = 0; i < strings.size(); ++i)
487     strings[i] = prefix + strings[i];
488 }
489 
490 int
main()491 main()
492 {
493   using std::string;
494   using std::vector;
495   using std::cerr;
496   using abigail::tests::get_src_dir;
497   using abigail::tests::get_build_dir;
498   using abigail::tools_utils::ensure_parent_dir_created;
499   using abigail::tools_utils::split_string;
500   using abigail::tools_utils::abidiff_status;
501 
502   unsigned int total_count = 0, passed_count = 0, failed_count = 0;
503 
504   string in_elfv0_path, in_elfv1_path,
505     in_suppression_path, abidiff_options, abidiff, cmd, diff_cmd,
506     ref_diff_report_path, out_diff_report_path;
507   vector<string> in_elfv0_headers_dirs, in_elfv1_headers_dirs;
508   string source_dir_prefix = string(get_src_dir()) + "/tests/";
509   string build_dir_prefix = string(get_build_dir()) + "/tests/";
510 
511     for (InOutSpec* s = in_out_specs; s->in_elfv0_path; ++s)
512       {
513 	bool is_ok = true;
514 	in_elfv0_path = source_dir_prefix + s->in_elfv0_path;
515 	in_elfv1_path = source_dir_prefix + s->in_elfv1_path;
516 	split_string(s->in_elfv0_headers_dirs, ",", in_elfv0_headers_dirs);
517 	split_string(s->in_elfv1_headers_dirs, ",", in_elfv1_headers_dirs);
518 	do_prefix_strings(in_elfv0_headers_dirs, source_dir_prefix);
519 	do_prefix_strings(in_elfv1_headers_dirs, source_dir_prefix);
520 
521 	if (s->in_suppr_path && strcmp(s->in_suppr_path, ""))
522 	  in_suppression_path = source_dir_prefix + s->in_suppr_path;
523 	else
524 	  in_suppression_path.clear();
525 
526 	abidiff_options = s->abidiff_options;
527 	ref_diff_report_path = source_dir_prefix + s->in_report_path;
528 	out_diff_report_path = build_dir_prefix + s->out_report_path;
529 
530 	if (!ensure_parent_dir_created(out_diff_report_path))
531 	  {
532 	    cerr << "could not create parent directory for "
533 		 << out_diff_report_path;
534 	    is_ok = false;
535 	    continue;
536 	  }
537 
538 	abidiff = string(get_build_dir()) + "/tools/abidiff";
539 	if (!abidiff_options.empty())
540 	  abidiff += " " + abidiff_options;
541 
542 	if (!in_elfv0_headers_dirs.empty())
543 	  for (vector<string>::const_iterator s = in_elfv0_headers_dirs.begin();
544 	       s != in_elfv0_headers_dirs.end();
545 	       ++s)
546 	    abidiff += " --headers-dir1 " + *s;
547 
548 	if (!in_elfv1_headers_dirs.empty())
549 	  for (vector<string>::const_iterator s = in_elfv1_headers_dirs.begin();
550 	       s != in_elfv1_headers_dirs.end();
551 	       ++s)
552 	    abidiff += " --headers-dir2 " + *s;
553 
554 	if (!in_suppression_path.empty())
555 	  abidiff += " --suppressions " + in_suppression_path;
556 
557 	cmd = abidiff + " " + in_elfv0_path + " " + in_elfv1_path;
558 	cmd += " > " + out_diff_report_path;
559 
560 	bool abidiff_ok = true;
561 	int code = system(cmd.c_str());
562 	if (!WIFEXITED(code))
563 	  abidiff_ok = false;
564 	else
565 	  {
566 	    abigail::tools_utils::abidiff_status status =
567 	      static_cast<abidiff_status>(WEXITSTATUS(code));
568 	    if (status != s->status)
569 	      {
570 		cerr << "for command '"
571 		     << cmd
572 		     << "', expected abidiff status to be " << s->status
573 		     << " but instead, got " << status << "\n";
574 		abidiff_ok = false;
575 	      }
576 	  }
577 
578 	if (abidiff_ok)
579 	  {
580 	    diff_cmd = "diff -u " + ref_diff_report_path
581 	      + " " + out_diff_report_path;
582 	    if (system(diff_cmd.c_str()))
583 	      is_ok = false;
584 	  }
585 	else
586 	  is_ok = false;
587 
588 	emit_test_status_and_update_counters(is_ok,
589 					     cmd,
590 					     passed_count,
591 					     failed_count,
592 					     total_count);
593       }
594 
595     emit_test_summary(total_count, passed_count, failed_count);
596 
597 
598     return failed_count;
599 }
600