1 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
2 // -*- mode: C++ -*-
3 //
4 // Copyright 2022 Google LLC
5 //
6 // Licensed under the Apache License v2.0 with LLVM Exceptions (the
7 // "License"); you may not use this file except in compliance with the
8 // License. You may obtain a copy of the License at
9 //
10 // https://llvm.org/LICENSE.txt
11 //
12 // Unless required by applicable law or agreed to in writing, software
13 // distributed under the License is distributed on an "AS IS" BASIS,
14 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 // See the License for the specific language governing permissions and
16 // limitations under the License.
17 //
18 // Author: Siddharth Nayyar
19
20 #include <filesystem>
21 #include <fstream>
22 #include <sstream>
23 #include <string>
24
25 #include <catch2/catch.hpp>
26 #include "comparison.h"
27 #include "graph.h"
28 #include "input.h"
29 #include "reader_options.h"
30 #include "reporting.h"
31 #include "runtime.h"
32
33 namespace {
34
35 struct IgnoreTestCase {
36 const std::string name;
37 const stg::InputFormat format0;
38 const std::string file0;
39 const stg::InputFormat format1;
40 const std::string file1;
41 const stg::Ignore ignore;
42 const std::string expected_output;
43 const bool expected_equals;
44 };
45
filename_to_path(const std::string & f)46 std::string filename_to_path(const std::string& f) {
47 return std::filesystem::path("testdata") / f;
48 }
49
Read(stg::Runtime & runtime,stg::Graph & graph,stg::InputFormat format,const std::string & input)50 stg::Id Read(stg::Runtime& runtime, stg::Graph& graph, stg::InputFormat format,
51 const std::string& input) {
52 return stg::Read(runtime, graph, format, filename_to_path(input).c_str(),
53 stg::ReadOptions(), nullptr);
54 }
55
56 TEST_CASE("ignore") {
57 const auto test = GENERATE(
58 IgnoreTestCase(
59 {"symbol type presence change",
60 stg::InputFormat::ABI,
61 "symbol_type_presence_0.xml",
62 stg::InputFormat::ABI,
63 "symbol_type_presence_1.xml",
64 stg::Ignore(),
65 "symbol_type_presence_small_diff",
66 false}),
67 IgnoreTestCase(
68 {"symbol type presence change pruned",
69 stg::InputFormat::ABI,
70 "symbol_type_presence_0.xml",
71 stg::InputFormat::ABI,
72 "symbol_type_presence_1.xml",
73 stg::Ignore(stg::Ignore::SYMBOL_TYPE_PRESENCE),
74 "empty",
75 true}),
76 IgnoreTestCase(
77 {"type declaration status change",
78 stg::InputFormat::ABI,
79 "type_declaration_status_0.xml",
80 stg::InputFormat::ABI,
81 "type_declaration_status_1.xml",
82 stg::Ignore(),
83 "type_declaration_status_small_diff",
84 false}),
85 IgnoreTestCase(
86 {"type declaration status change pruned",
87 stg::InputFormat::ABI,
88 "type_declaration_status_0.xml",
89 stg::InputFormat::ABI,
90 "type_declaration_status_1.xml",
91 stg::Ignore(stg::Ignore::TYPE_DECLARATION_STATUS),
92 "empty",
93 true}),
94 IgnoreTestCase(
95 {"primitive type encoding",
96 stg::InputFormat::STG,
97 "primitive_type_encoding_0.stg",
98 stg::InputFormat::STG,
99 "primitive_type_encoding_1.stg",
100 stg::Ignore(),
101 "primitive_type_encoding_small_diff",
102 false}),
103 IgnoreTestCase(
104 {"primitive type encoding ignored",
105 stg::InputFormat::STG,
106 "primitive_type_encoding_0.stg",
107 stg::InputFormat::STG,
108 "primitive_type_encoding_1.stg",
109 stg::Ignore(stg::Ignore::PRIMITIVE_TYPE_ENCODING),
110 "empty",
111 true}),
112 IgnoreTestCase(
113 {"member size",
114 stg::InputFormat::STG,
115 "member_size_0.stg",
116 stg::InputFormat::STG,
117 "member_size_1.stg",
118 stg::Ignore(),
119 "member_size_small_diff",
120 false}),
121 IgnoreTestCase(
122 {"member size ignored",
123 stg::InputFormat::STG,
124 "member_size_0.stg",
125 stg::InputFormat::STG,
126 "member_size_1.stg",
127 stg::Ignore(stg::Ignore::MEMBER_SIZE),
128 "empty",
129 true}),
130 IgnoreTestCase(
131 {"enum underlying type",
132 stg::InputFormat::STG,
133 "enum_underlying_type_0.stg",
134 stg::InputFormat::STG,
135 "enum_underlying_type_1.stg",
136 stg::Ignore(),
137 "enum_underlying_type_small_diff",
138 false}),
139 IgnoreTestCase(
140 {"enum underlying type ignored",
141 stg::InputFormat::STG,
142 "enum_underlying_type_0.stg",
143 stg::InputFormat::STG,
144 "enum_underlying_type_1.stg",
145 stg::Ignore(stg::Ignore::ENUM_UNDERLYING_TYPE),
146 "empty",
147 true}),
148 IgnoreTestCase(
149 {"qualifier",
150 stg::InputFormat::STG,
151 "qualifier_0.stg",
152 stg::InputFormat::STG,
153 "qualifier_1.stg",
154 stg::Ignore(),
155 "qualifier_small_diff",
156 false}),
157 IgnoreTestCase(
158 {"qualifier ignored",
159 stg::InputFormat::STG,
160 "qualifier_0.stg",
161 stg::InputFormat::STG,
162 "qualifier_1.stg",
163 stg::Ignore(stg::Ignore::QUALIFIER),
164 "empty",
165 true}),
166 IgnoreTestCase(
167 {"CRC change",
168 stg::InputFormat::STG,
169 "crc_change_0.stg",
170 stg::InputFormat::STG,
171 "crc_change_1.stg",
172 stg::Ignore(),
173 "crc_change_small_diff",
174 false}),
175 IgnoreTestCase(
176 {"CRC change ignored",
177 stg::InputFormat::STG,
178 "crc_change_0.stg",
179 stg::InputFormat::STG,
180 "crc_change_1.stg",
181 stg::Ignore(stg::Ignore::SYMBOL_CRC),
182 "empty",
183 true}),
184 IgnoreTestCase(
185 {"interface addition",
186 stg::InputFormat::STG,
187 "interface_addition_0.stg",
188 stg::InputFormat::STG,
189 "interface_addition_1.stg",
190 stg::Ignore(),
191 "interface_addition_small_diff",
192 false}),
193 IgnoreTestCase(
194 {"interface addition ignored",
195 stg::InputFormat::STG,
196 "interface_addition_0.stg",
197 stg::InputFormat::STG,
198 "interface_addition_1.stg",
199 stg::Ignore(stg::Ignore::INTERFACE_ADDITION),
200 "empty",
201 true}),
202 IgnoreTestCase(
203 {"type addition",
204 stg::InputFormat::STG,
205 "type_addition_0.stg",
206 stg::InputFormat::STG,
207 "type_addition_1.stg",
208 stg::Ignore(),
209 "type_addition_small_diff",
210 false}),
211 IgnoreTestCase(
212 {"type addition ignored",
213 stg::InputFormat::STG,
214 "type_addition_0.stg",
215 stg::InputFormat::STG,
216 "type_addition_1.stg",
217 stg::Ignore(stg::Ignore::INTERFACE_ADDITION),
218 "empty",
219 true}),
220 IgnoreTestCase(
221 {"type definition addition",
222 stg::InputFormat::STG,
223 "type_addition_1.stg",
224 stg::InputFormat::STG,
225 "type_addition_2.stg",
226 stg::Ignore(),
227 "type_definition_addition_small_diff",
228 false}),
229 IgnoreTestCase(
230 {"type definition addition ignored",
231 stg::InputFormat::STG,
232 "type_addition_1.stg",
233 stg::InputFormat::STG,
234 "type_addition_2.stg",
235 stg::Ignore(stg::Ignore::TYPE_DEFINITION_ADDITION),
236 "empty",
237 true})
238 );
239
240 SECTION(test.name) {
241 std::ostringstream os;
242 stg::Runtime runtime(os, false);
243
244 // Read inputs.
245 stg::Graph graph;
246 const auto id0 = Read(runtime, graph, test.format0, test.file0);
247 const auto id1 = Read(runtime, graph, test.format1, test.file1);
248
249 // Compute differences.
250 stg::Compare compare{runtime, graph, test.ignore};
251 const auto& [equals, comparison] = compare(id0, id1);
252
253 // Write SMALL reports.
254 std::ostringstream output;
255 if (comparison) {
256 stg::NameCache names;
257 stg::reporting::Options options{stg::reporting::OutputFormat::SMALL, 0};
258 stg::reporting::Reporting reporting{graph, compare.outcomes, options,
259 names};
260 Report(reporting, *comparison, output);
261 }
262
263 // Check comparison outcome and report output.
264 CHECK(equals == test.expected_equals);
265 std::ifstream expected_output_file(filename_to_path(test.expected_output));
266 std::ostringstream expected_output;
267 expected_output << expected_output_file.rdbuf();
268 CHECK(output.str() == expected_output.str());
269 }
270 }
271
272 struct ShortReportTestCase {
273 const std::string name;
274 const std::string xml0;
275 const std::string xml1;
276 const std::string expected_output;
277 };
278
279 TEST_CASE("short report") {
280 const auto test = GENERATE(
281 ShortReportTestCase(
282 {"crc changes", "crc_0.xml", "crc_1.xml", "crc_changes_short_diff"}),
283 ShortReportTestCase({"only crc changes", "crc_only_0.xml",
284 "crc_only_1.xml", "crc_only_changes_short_diff"}),
285 ShortReportTestCase({"offset changes", "offset_0.xml", "offset_1.xml",
286 "offset_changes_short_diff"}),
287 ShortReportTestCase(
288 {"symbols added and removed", "added_removed_symbols_0.xml",
289 "added_removed_symbols_1.xml", "added_removed_symbols_short_diff"}),
290 ShortReportTestCase({"symbols added and removed only",
291 "added_removed_symbols_only_0.xml",
292 "added_removed_symbols_only_1.xml",
293 "added_removed_symbols_only_short_diff"}));
294
295 SECTION(test.name) {
296 std::ostringstream os;
297 stg::Runtime runtime(os, false);
298
299 // Read inputs.
300 stg::Graph graph;
301 const auto id0 = Read(runtime, graph, stg::InputFormat::ABI, test.xml0);
302 const auto id1 = Read(runtime, graph, stg::InputFormat::ABI, test.xml1);
303
304 // Compute differences.
305 stg::Compare compare{runtime, graph, {}};
306 const auto& [equals, comparison] = compare(id0, id1);
307
308 // Write SHORT reports.
309 std::stringstream output;
310 if (comparison) {
311 stg::NameCache names;
312 stg::reporting::Options options{stg::reporting::OutputFormat::SHORT, 2};
313 stg::reporting::Reporting reporting{graph, compare.outcomes, options,
314 names};
315 Report(reporting, *comparison, output);
316 }
317
318 // Check comparison outcome and report output.
319 CHECK(equals == false);
320 std::ifstream expected_output_file(filename_to_path(test.expected_output));
321 std::ostringstream expected_output;
322 expected_output << expected_output_file.rdbuf();
323 CHECK(output.str() == expected_output.str());
324 }
325 }
326
327 TEST_CASE("fidelity diff") {
328 std::ostringstream os;
329 stg::Runtime runtime(os, false);
330
331 // Read inputs.
332 stg::Graph graph;
333 const auto id0 =
334 Read(runtime, graph, stg::InputFormat::STG, "fidelity_diff_0.stg");
335 const auto id1 =
336 Read(runtime, graph, stg::InputFormat::STG, "fidelity_diff_1.stg");
337
338 // Compute fidelity diff.
339 auto fidelity_diff = stg::GetFidelityTransitions(graph, id0, id1);
340
341 // Write fidelity diff report.
342 std::ostringstream report;
343 stg::reporting::FidelityDiff(fidelity_diff, report);
344
345 // Check report.
346 std::ifstream expected_report_file(filename_to_path("fidelity_diff_report"));
347 std::ostringstream expected_report;
348 expected_report << expected_report_file.rdbuf();
349 CHECK(report.str() == expected_report.str());
350 }
351
352 } // namespace
353