1 // Copyright 2023 Code Intelligence GmbH 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package com.code_intelligence.jazzer.junit; 16 17 import static com.google.common.truth.Truth.assertThat; 18 import static com.google.common.truth.Truth8.assertThat; 19 import static org.junit.Assume.assumeFalse; 20 import static org.junit.Assume.assumeTrue; 21 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; 22 import static org.junit.platform.testkit.engine.EventConditions.container; 23 import static org.junit.platform.testkit.engine.EventConditions.displayName; 24 import static org.junit.platform.testkit.engine.EventConditions.event; 25 import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; 26 import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; 27 import static org.junit.platform.testkit.engine.EventConditions.test; 28 import static org.junit.platform.testkit.engine.EventConditions.type; 29 import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; 30 import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; 31 import static org.junit.platform.testkit.engine.EventType.FINISHED; 32 import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; 33 import static org.junit.platform.testkit.engine.EventType.STARTED; 34 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; 35 36 import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium; 37 import java.io.File; 38 import java.io.IOException; 39 import java.nio.file.Files; 40 import java.nio.file.Path; 41 import java.nio.file.Paths; 42 import java.util.stream.Stream; 43 import org.junit.Before; 44 import org.junit.Rule; 45 import org.junit.Test; 46 import org.junit.platform.testkit.engine.EngineExecutionResults; 47 import org.junit.platform.testkit.engine.EngineTestKit; 48 import org.junit.rules.TemporaryFolder; 49 50 public class CorpusDirectoryTest { 51 private static final String ENGINE = "engine:junit-jupiter"; 52 private static final String CLAZZ = "class:com.example.CorpusDirectoryFuzzTest"; 53 private static final String INPUTS_FUZZ = 54 "test-template:corpusDirectoryFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)"; 55 private static final String INVOCATION = "test-template-invocation:#"; 56 57 @Rule public TemporaryFolder temp = new TemporaryFolder(); 58 Path baseDir; 59 60 @Before setup()61 public void setup() { 62 baseDir = temp.getRoot().toPath(); 63 } 64 65 @Test fuzzingEnabled()66 public void fuzzingEnabled() throws IOException { 67 assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); 68 69 // Create a fake test resource directory structure with an inputs directory to verify that 70 // Jazzer uses it and emits a crash file into it. 71 Path artifactsDirectory = baseDir.resolve(Paths.get("src", "test", "resources", "com", 72 "example", "CorpusDirectoryFuzzTestInputs", "corpusDirectoryFuzz")); 73 Files.createDirectories(artifactsDirectory); 74 75 // An explicitly stated corpus directory should be used to save new corpus entries. 76 Path explicitGeneratedCorpus = baseDir.resolve(Paths.get("corpus")); 77 Files.createDirectories(explicitGeneratedCorpus); 78 79 // The default generated corpus directory should only be used if no explicit corpus directory 80 // is given. 81 Path defaultGeneratedCorpus = baseDir.resolve( 82 Paths.get(".cifuzz-corpus", "com.example.CorpusDirectoryFuzzTest", "corpusDirectoryFuzz")); 83 84 EngineExecutionResults results = 85 EngineTestKit.engine("junit-jupiter") 86 .selectors(selectClass("com.example.CorpusDirectoryFuzzTest")) 87 .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) 88 // Add corpus directory as initial libFuzzer parameter. 89 .configurationParameter("jazzer.internal.arg.0", "fake_test_argv0") 90 .configurationParameter( 91 "jazzer.internal.arg.1", explicitGeneratedCorpus.toAbsolutePath().toString()) 92 .execute(); 93 94 results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), 95 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), 96 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), 97 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ)), 98 finishedSuccessfully()), 99 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), 100 event(type(FINISHED), container(ENGINE), finishedSuccessfully())); 101 102 results.testEvents().assertEventsMatchExactly( 103 event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), 104 event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))), 105 event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)), 106 displayName("<empty input>"), finishedSuccessfully()), 107 event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), 108 event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))), 109 event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)), 110 displayName("seed"), finishedSuccessfully()), 111 event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), 112 event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)), 113 displayName("Fuzzing...")), 114 event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)), 115 displayName("Fuzzing..."), 116 finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class)))); 117 118 // Crash file should be emitted into the artifacts directory and not into corpus directory. 119 assertCrashFileExistsIn(artifactsDirectory); 120 assertNoCrashFileExistsIn(baseDir); 121 assertNoCrashFileExistsIn(explicitGeneratedCorpus); 122 assertNoCrashFileExistsIn(defaultGeneratedCorpus); 123 124 // Verify that corpus files are written to given corpus directory and not generated one. 125 assertThat(Files.list(explicitGeneratedCorpus)).isNotEmpty(); 126 assertThat(Files.list(defaultGeneratedCorpus)).isEmpty(); 127 } 128 129 @Test fuzzingDisabled()130 public void fuzzingDisabled() throws IOException { 131 assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); 132 133 Path corpusDirectory = baseDir.resolve(Paths.get("corpus")); 134 Files.createDirectories(corpusDirectory); 135 Files.createFile(corpusDirectory.resolve("corpus_entry")); 136 137 EngineExecutionResults results = 138 EngineTestKit.engine("junit-jupiter") 139 .selectors(selectClass("com.example.CorpusDirectoryFuzzTest")) 140 .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) 141 // Add corpus directory as initial libFuzzer parameter. 142 .configurationParameter("jazzer.internal.arg.0", "fake_test_argv0") 143 .configurationParameter( 144 "jazzer.internal.arg.1", corpusDirectory.toAbsolutePath().toString()) 145 .execute(); 146 147 results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), 148 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), 149 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), 150 event(type(REPORTING_ENTRY_PUBLISHED), 151 container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), 152 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), 153 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), 154 event(type(FINISHED), container(ENGINE), finishedSuccessfully())); 155 156 // Verify that corpus_entry is not picked up and corpus directory is ignored in regression mode. 157 results.testEvents().assertEventsMatchExactly( 158 event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), 159 event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))), 160 event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)), 161 displayName("<empty input>"), finishedSuccessfully()), 162 event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), 163 event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))), 164 event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)), 165 displayName("seed"), finishedSuccessfully())); 166 } 167 assertCrashFileExistsIn(Path artifactsDirectory)168 private static void assertCrashFileExistsIn(Path artifactsDirectory) throws IOException { 169 try (Stream<Path> crashFiles = 170 Files.list(artifactsDirectory) 171 .filter(path -> path.getFileName().toString().startsWith("crash-"))) { 172 assertThat(crashFiles).isNotEmpty(); 173 } 174 } 175 assertNoCrashFileExistsIn(Path generatedCorpus)176 private static void assertNoCrashFileExistsIn(Path generatedCorpus) throws IOException { 177 try (Stream<Path> crashFiles = 178 Files.list(generatedCorpus) 179 .filter(path -> path.getFileName().toString().startsWith("crash-"))) { 180 assertThat(crashFiles).isEmpty(); 181 } 182 } 183 } 184