1 // Copyright 2022 Code Intelligence GmbH 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package com.code_intelligence.jazzer.junit; 16 17 import static com.google.common.truth.Truth.assertThat; 18 import static com.google.common.truth.Truth8.assertThat; 19 import static org.junit.Assume.assumeFalse; 20 import static org.junit.Assume.assumeTrue; 21 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; 22 import static org.junit.platform.testkit.engine.EventConditions.container; 23 import static org.junit.platform.testkit.engine.EventConditions.displayName; 24 import static org.junit.platform.testkit.engine.EventConditions.event; 25 import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; 26 import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; 27 import static org.junit.platform.testkit.engine.EventConditions.test; 28 import static org.junit.platform.testkit.engine.EventConditions.type; 29 import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; 30 import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; 31 import static org.junit.platform.testkit.engine.EventType.FINISHED; 32 import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; 33 import static org.junit.platform.testkit.engine.EventType.SKIPPED; 34 import static org.junit.platform.testkit.engine.EventType.STARTED; 35 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; 36 37 import java.io.IOException; 38 import java.nio.file.Files; 39 import java.nio.file.Path; 40 import java.nio.file.Paths; 41 import java.util.Arrays; 42 import java.util.stream.Stream; 43 import org.junit.Before; 44 import org.junit.Rule; 45 import org.junit.Test; 46 import org.junit.platform.launcher.TagFilter; 47 import org.junit.platform.testkit.engine.EngineExecutionResults; 48 import org.junit.platform.testkit.engine.EngineTestKit; 49 import org.junit.rules.TemporaryFolder; 50 import org.opentest4j.AssertionFailedError; 51 52 public class FuzzingWithCrashTest { 53 private static final String CRASHING_SEED_NAME = "crashing_seed"; 54 // Crashes ByteFuzzTest since 'b' % 2 == 0. 55 private static final byte[] CRASHING_SEED_CONTENT = new byte[] {'b', 'a', 'c'}; 56 private static final String CRASHING_SEED_DIGEST = "5e4dec23c9afa48bd5bee3daa2a0ab66e147012b"; 57 private static final String ENGINE = "engine:junit-jupiter"; 58 private static final String INVOCATION = "test-template-invocation:#"; 59 60 private static final String CLAZZ_NAME = "com.example.ValidFuzzTests"; 61 62 private static final String CLAZZ = "class:" + CLAZZ_NAME; 63 private static final TestMethod BYTE_FUZZ = new TestMethod(CLAZZ_NAME, "byteFuzz([B)"); 64 private static final TestMethod NO_CRASH_FUZZ = new TestMethod(CLAZZ_NAME, "noCrashFuzz([B)"); 65 private static final TestMethod DATA_FUZZ = 66 new TestMethod(CLAZZ_NAME, "dataFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)"); 67 68 @Rule public TemporaryFolder temp = new TemporaryFolder(); 69 Path baseDir; 70 Path inputsDirectory; 71 72 @Before setup()73 public void setup() throws IOException { 74 baseDir = temp.getRoot().toPath(); 75 // Create a fake test resource directory structure with an inputs directory to verify that 76 // Jazzer uses it and emits a crash file into it. 77 inputsDirectory = baseDir.resolve( 78 Paths.get("src", "test", "resources", "com", "example", "ValidFuzzTestsInputs")); 79 // populate the same seed in all test directories 80 for (String method : 81 Arrays.asList(BYTE_FUZZ.getName(), NO_CRASH_FUZZ.getName(), DATA_FUZZ.getName())) { 82 Path methodInputsDirectory = inputsDirectory.resolve(method); 83 Files.createDirectories(methodInputsDirectory); 84 Files.write(methodInputsDirectory.resolve(CRASHING_SEED_NAME), CRASHING_SEED_CONTENT); 85 } 86 } 87 executeTests()88 private EngineExecutionResults executeTests() { 89 return EngineTestKit.engine("junit-jupiter") 90 .selectors(selectClass("com.example.ValidFuzzTests")) 91 .filters(TagFilter.includeTags("jazzer")) 92 .configurationParameter( 93 "jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*") 94 .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) 95 .execute(); 96 } 97 98 @Test fuzzingEnabled()99 public void fuzzingEnabled() throws IOException { 100 assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); 101 102 EngineExecutionResults results = executeTests(); 103 104 results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), 105 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), 106 event(type(STARTED), 107 container(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId()))), 108 event(type(FINISHED), 109 container(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId())), 110 finishedSuccessfully()), 111 event(type(SKIPPED), 112 container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ.getDescriptorId()))), 113 event(type(SKIPPED), 114 container(uniqueIdSubstrings(ENGINE, CLAZZ, DATA_FUZZ.getDescriptorId()))), 115 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), 116 event(type(FINISHED), container(ENGINE), finishedSuccessfully())); 117 118 results.testEvents().assertEventsMatchLooselyInOrder( 119 event(type(DYNAMIC_TEST_REGISTERED), 120 test(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId()))), 121 event(type(STARTED), 122 test(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId(), INVOCATION)), 123 displayName("Fuzzing...")), 124 event(type(FINISHED), 125 test(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId(), INVOCATION)), 126 displayName("Fuzzing..."), 127 finishedWithFailure(instanceOf(AssertionFailedError.class)))); 128 129 // Jazzer first tries the empty input, which doesn't crash the ByteFuzzTest. The second input is 130 // the seed we planted, which is crashing, so verify that a crash file with the same content is 131 // created in our fake seed corpus, but not in the current working directory. 132 try (Stream<Path> crashFiles = Files.list(baseDir).filter( 133 path -> path.getFileName().toString().startsWith("crash-"))) { 134 assertThat(crashFiles).isEmpty(); 135 } 136 137 // the crashing input will be created in the directory for the fuzzed test, in this case 138 // byteFuzz and will not exist in the directories of the other tests 139 Path byteFuzzInputDirectory = inputsDirectory.resolve(BYTE_FUZZ.getName()); 140 try (Stream<Path> seeds = Files.list(byteFuzzInputDirectory)) { 141 assertThat(seeds).containsExactly( 142 byteFuzzInputDirectory.resolve("crash-" + CRASHING_SEED_DIGEST), 143 byteFuzzInputDirectory.resolve(CRASHING_SEED_NAME)); 144 } 145 assertThat(Files.readAllBytes(byteFuzzInputDirectory.resolve("crash-" + CRASHING_SEED_DIGEST))) 146 .isEqualTo(CRASHING_SEED_CONTENT); 147 148 // check that the others only include 1 file 149 for (String method : Arrays.asList(NO_CRASH_FUZZ.getName(), DATA_FUZZ.getName())) { 150 Path methodInputsDirectory = inputsDirectory.resolve(method); 151 try (Stream<Path> seeds = Files.list(methodInputsDirectory)) { 152 assertThat(seeds).containsExactly(methodInputsDirectory.resolve(CRASHING_SEED_NAME)); 153 } 154 } 155 156 // Verify that the engine created the generated corpus directory. As a seed produced the crash, 157 // it should be empty. 158 Path generatedCorpus = 159 baseDir.resolve(Paths.get(".cifuzz-corpus", CLAZZ_NAME, BYTE_FUZZ.getName())); 160 assertThat(Files.isDirectory(generatedCorpus)).isTrue(); 161 try (Stream<Path> entries = Files.list(generatedCorpus)) { 162 assertThat(entries).isEmpty(); 163 } 164 } 165 166 @Test fuzzingDisabled()167 public void fuzzingDisabled() throws IOException { 168 assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); 169 170 EngineExecutionResults results = executeTests(); 171 172 results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), 173 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), 174 event(type(STARTED), 175 container(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId()))), 176 event(type(REPORTING_ENTRY_PUBLISHED), 177 container(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId()))), 178 event(type(FINISHED), 179 container(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId())), 180 finishedSuccessfully()), 181 event(type(STARTED), 182 container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ.getDescriptorId()))), 183 event(type(REPORTING_ENTRY_PUBLISHED), 184 container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ.getDescriptorId()))), 185 event(type(FINISHED), 186 container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ.getDescriptorId()))), 187 event(type(STARTED), 188 container(uniqueIdSubstrings(ENGINE, CLAZZ, DATA_FUZZ.getDescriptorId()))), 189 event(type(REPORTING_ENTRY_PUBLISHED), 190 container(uniqueIdSubstrings(ENGINE, CLAZZ, DATA_FUZZ.getDescriptorId()))), 191 event(type(FINISHED), 192 container(uniqueIdSubstrings(ENGINE, CLAZZ, DATA_FUZZ.getDescriptorId()))), 193 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), 194 event(type(FINISHED), container(ENGINE), finishedSuccessfully())); 195 196 // No fuzzing means no crashes means no new seeds. 197 // Check against all methods' input directories 198 for (String method : 199 Arrays.asList(BYTE_FUZZ.getName(), NO_CRASH_FUZZ.getName(), DATA_FUZZ.getName())) { 200 Path methodInputsDirectory = inputsDirectory.resolve(method); 201 try (Stream<Path> seeds = Files.list(methodInputsDirectory)) { 202 assertThat(seeds).containsExactly(methodInputsDirectory.resolve(CRASHING_SEED_NAME)); 203 } 204 } 205 } 206 } 207