• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2022 Code Intelligence GmbH
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //      http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 package com.code_intelligence.jazzer.junit;
16 
17 import static com.google.common.truth.Truth.assertThat;
18 import static com.google.common.truth.Truth8.assertThat;
19 import static org.junit.Assume.assumeFalse;
20 import static org.junit.Assume.assumeTrue;
21 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
22 import static org.junit.platform.testkit.engine.EventConditions.container;
23 import static org.junit.platform.testkit.engine.EventConditions.displayName;
24 import static org.junit.platform.testkit.engine.EventConditions.event;
25 import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully;
26 import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
27 import static org.junit.platform.testkit.engine.EventConditions.test;
28 import static org.junit.platform.testkit.engine.EventConditions.type;
29 import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings;
30 import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED;
31 import static org.junit.platform.testkit.engine.EventType.FINISHED;
32 import static org.junit.platform.testkit.engine.EventType.SKIPPED;
33 import static org.junit.platform.testkit.engine.EventType.STARTED;
34 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf;
35 
36 import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium;
37 import java.io.IOException;
38 import java.nio.charset.StandardCharsets;
39 import java.nio.file.Files;
40 import java.nio.file.Path;
41 import java.nio.file.Paths;
42 import java.util.stream.Stream;
43 import org.junit.Before;
44 import org.junit.Rule;
45 import org.junit.Test;
46 import org.junit.platform.testkit.engine.EngineExecutionResults;
47 import org.junit.platform.testkit.engine.EngineTestKit;
48 import org.junit.platform.testkit.engine.EventType;
49 import org.junit.rules.TemporaryFolder;
50 
51 public class ValueProfileTest {
52   private static final boolean VALUE_PROFILE_ENABLED =
53       "True".equals(System.getenv("JAZZER_VALUE_PROFILE"));
54 
55   private static final String ENGINE = "engine:junit-jupiter";
56   private static final String CLAZZ = "class:com.example.ValueProfileFuzzTest";
57   private static final String VALUE_PROFILE_FUZZ = "test-template:valueProfileFuzz([B)";
58   private static final String INVOCATION = "test-template-invocation:#";
59 
60   @Rule public TemporaryFolder temp = new TemporaryFolder();
61   Path baseDir;
62   Path inputsDirectories;
63 
64   @Before
setup()65   public void setup() throws IOException {
66     baseDir = temp.getRoot().toPath();
67     // Create a fake test resource directory structure with an input directory to verify that
68     // Jazzer uses it and emits a crash file into it.
69     inputsDirectories = baseDir.resolve(Paths.get("src", "test", "resources", "com", "example",
70         "ValueProfileFuzzTestInputs", "valueProfileFuzz"));
71     Files.createDirectories(inputsDirectories);
72   }
73 
executeTests()74   private EngineExecutionResults executeTests() {
75     return EngineTestKit.engine("junit-jupiter")
76         .selectors(selectClass("com.example.ValueProfileFuzzTest"))
77         .configurationParameter(
78             "jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*")
79         .configurationParameter("jazzer.valueprofile", System.getenv("JAZZER_VALUE_PROFILE"))
80         .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString())
81         .execute();
82   }
83 
84   @Test
valueProfileEnabled()85   public void valueProfileEnabled() throws IOException {
86     assumeTrue(VALUE_PROFILE_ENABLED);
87 
88     EngineExecutionResults results = executeTests();
89 
90     results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
91         event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
92         event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
93         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ)),
94             finishedSuccessfully()),
95         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
96         event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
97 
98     results.testEvents().assertEventsMatchExactly(
99         event(type(DYNAMIC_TEST_REGISTERED),
100             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
101         event(type(STARTED),
102             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)),
103             displayName("<empty input>")),
104         event(type(FINISHED),
105             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)),
106             displayName("<empty input>"), finishedSuccessfully()),
107         event(type(DYNAMIC_TEST_REGISTERED),
108             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
109         event(type(STARTED),
110             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)),
111             displayName("empty_seed")),
112         event(type(FINISHED),
113             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)),
114             displayName("empty_seed"), finishedSuccessfully()),
115         event(type(DYNAMIC_TEST_REGISTERED),
116             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
117         event(type(STARTED),
118             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)),
119             displayName("Fuzzing...")),
120         event(type(FINISHED),
121             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)),
122             displayName("Fuzzing..."),
123             finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class))));
124 
125     // Should crash on the exact input "Jazzer", with the crash emitted into the seed corpus.
126     try (Stream<Path> crashFiles = Files.list(baseDir).filter(
127              path -> path.getFileName().toString().startsWith("crash-"))) {
128       assertThat(crashFiles).isEmpty();
129     }
130     try (Stream<Path> seeds = Files.list(inputsDirectories)) {
131       assertThat(seeds).containsExactly(
132           inputsDirectories.resolve("crash-131db69c7fadc408fe5031079dad3a441df09aff"));
133     }
134     assertThat(Files.readAllBytes(
135                    inputsDirectories.resolve("crash-131db69c7fadc408fe5031079dad3a441df09aff")))
136         .isEqualTo("Jazzer".getBytes(StandardCharsets.UTF_8));
137 
138     // Verify that the engine created the generated corpus directory and emitted inputs into it.
139     Path generatedCorpus =
140         baseDir.resolve(Paths.get(".cifuzz-corpus", "com.example.ValueProfileFuzzTest"));
141     assertThat(Files.isDirectory(generatedCorpus)).isTrue();
142     try (Stream<Path> entries = Files.list(generatedCorpus)) {
143       assertThat(entries).isNotEmpty();
144     }
145   }
146 
147   @Test
valueProfileDisabled()148   public void valueProfileDisabled() throws IOException {
149     assumeFalse(VALUE_PROFILE_ENABLED);
150 
151     EngineExecutionResults results = executeTests();
152 
153     results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
154         event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
155         event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
156         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ)),
157             finishedSuccessfully()),
158         event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
159         event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
160 
161     results.testEvents().assertEventsMatchExactly(
162         event(type(DYNAMIC_TEST_REGISTERED),
163             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
164         event(type(STARTED),
165             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)),
166             displayName("<empty input>")),
167         event(type(FINISHED),
168             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)),
169             displayName("<empty input>"), finishedSuccessfully()),
170         event(type(DYNAMIC_TEST_REGISTERED),
171             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
172         event(type(STARTED),
173             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)),
174             displayName("empty_seed")),
175         event(type(FINISHED),
176             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)),
177             displayName("empty_seed"), finishedSuccessfully()),
178         event(type(DYNAMIC_TEST_REGISTERED),
179             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))),
180         event(type(STARTED),
181             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)),
182             displayName("Fuzzing...")),
183         event(type(FINISHED),
184             test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)),
185             displayName("Fuzzing..."), finishedSuccessfully()));
186 
187     // No crash means no crashing input is emitted anywhere.
188     try (Stream<Path> crashFiles = Files.list(baseDir).filter(
189              path -> path.getFileName().toString().startsWith("crash-"))) {
190       assertThat(crashFiles).isEmpty();
191     }
192     try (Stream<Path> seeds = Files.list(inputsDirectories)) {
193       assertThat(seeds).isEmpty();
194     }
195 
196     // Verify that the engine created the generated corpus directory and emitted inputs into it.
197     Path generatedCorpus =
198         baseDir.resolve(Paths.get(".cifuzz-corpus", "com.example.ValueProfileFuzzTest"));
199     assertThat(Files.isDirectory(generatedCorpus)).isTrue();
200     try (Stream<Path> entries = Files.list(generatedCorpus)) {
201       assertThat(entries).isNotEmpty();
202     }
203   }
204 }
205