1 // Copyright 2022 Code Intelligence GmbH 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package com.code_intelligence.jazzer.junit; 16 17 import static com.google.common.truth.Truth.assertThat; 18 import static com.google.common.truth.Truth.assertWithMessage; 19 import static com.google.common.truth.Truth8.assertThat; 20 import static org.junit.Assume.assumeFalse; 21 import static org.junit.Assume.assumeTrue; 22 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod; 23 import static org.junit.platform.testkit.engine.EventConditions.abortedWithReason; 24 import static org.junit.platform.testkit.engine.EventConditions.container; 25 import static org.junit.platform.testkit.engine.EventConditions.displayName; 26 import static org.junit.platform.testkit.engine.EventConditions.event; 27 import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; 28 import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; 29 import static org.junit.platform.testkit.engine.EventConditions.test; 30 import static org.junit.platform.testkit.engine.EventConditions.type; 31 import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; 32 import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; 33 import static org.junit.platform.testkit.engine.EventType.FINISHED; 34 import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; 35 import static org.junit.platform.testkit.engine.EventType.STARTED; 36 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; 37 38 import java.io.IOException; 39 import java.nio.charset.StandardCharsets; 40 import java.nio.file.Files; 41 import java.nio.file.Path; 42 import java.nio.file.Paths; 43 import java.util.Arrays; 44 import java.util.List; 45 import java.util.stream.Collectors; 46 import java.util.stream.Stream; 47 import org.junit.Rule; 48 import org.junit.Test; 49 import org.junit.platform.testkit.engine.EngineExecutionResults; 50 import org.junit.platform.testkit.engine.EngineTestKit; 51 import org.junit.rules.TemporaryFolder; 52 import org.opentest4j.TestAbortedException; 53 54 public class AutofuzzTest { 55 @Rule public TemporaryFolder temp = new TemporaryFolder(); 56 57 @Test fuzzingEnabled()58 public void fuzzingEnabled() throws IOException { 59 assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); 60 61 Path baseDir = temp.getRoot().toPath(); 62 // Create a fake test resource directory structure to verify that Jazzer uses it and emits a 63 // crash file into it. 64 Path testResourceDir = baseDir.resolve("src").resolve("test").resolve("resources"); 65 Files.createDirectories(testResourceDir); 66 Path inputsDirectory = testResourceDir.resolve("com") 67 .resolve("example") 68 .resolve("AutofuzzFuzzTestInputs") 69 .resolve("autofuzz"); 70 71 EngineExecutionResults results = 72 EngineTestKit.engine("junit-jupiter") 73 .selectors(selectMethod( 74 "com.example.AutofuzzFuzzTest#autofuzz(java.lang.String,com.example.AutofuzzFuzzTest$IntHolder)")) 75 .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) 76 .execute(); 77 78 final String engine = "engine:junit-jupiter"; 79 final String clazz = "class:com.example.AutofuzzFuzzTest"; 80 final String autofuzz = 81 "test-template:autofuzz(java.lang.String, com.example.AutofuzzFuzzTest$IntHolder)"; 82 final String invocation = "test-template-invocation:#"; 83 84 results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(engine)), 85 event(type(STARTED), container(uniqueIdSubstrings(engine, clazz))), 86 event(type(STARTED), container(uniqueIdSubstrings(engine, clazz, autofuzz))), 87 event(type(FINISHED), container(uniqueIdSubstrings(engine, clazz, autofuzz)), 88 finishedSuccessfully()), 89 event(type(FINISHED), container(uniqueIdSubstrings(engine, clazz)), finishedSuccessfully()), 90 event(type(FINISHED), container(engine), finishedSuccessfully())); 91 92 results.testEvents().assertEventsMatchExactly(event(type(DYNAMIC_TEST_REGISTERED)), 93 event(type(STARTED)), 94 event(test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation + 1)), 95 displayName("<empty input>"), 96 abortedWithReason(instanceOf(TestAbortedException.class))), 97 event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(engine, clazz, autofuzz))), 98 event(type(STARTED), test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation + 2)), 99 displayName("Fuzzing...")), 100 event(type(FINISHED), test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation + 2)), 101 displayName("Fuzzing..."), finishedWithFailure(instanceOf(RuntimeException.class)))); 102 103 // Should crash on an input that contains "jazzer", with the crash emitted into the 104 // automatically created inputs directory. 105 Path crashingInput; 106 try (Stream<Path> crashFiles = 107 Files.list(inputsDirectory) 108 .filter(path -> path.getFileName().toString().startsWith("crash-"))) { 109 List<Path> crashFilesList = crashFiles.collect(Collectors.toList()); 110 assertWithMessage("Expected crashing input in " + baseDir).that(crashFilesList).hasSize(1); 111 crashingInput = crashFilesList.get(0); 112 } 113 assertThat(new String(Files.readAllBytes(crashingInput), StandardCharsets.UTF_8)) 114 .contains("jazzer"); 115 116 try (Stream<Path> seeds = Files.list(baseDir).filter(Files::isRegularFile)) { 117 assertThat(seeds).isEmpty(); 118 } 119 120 // Verify that the engine created the generated corpus directory. Since the crash was not found 121 // on a seed, it should not be empty. 122 Path generatedCorpus = 123 baseDir.resolve(".cifuzz-corpus").resolve("com.example.AutofuzzFuzzTest"); 124 assertThat(Files.isDirectory(generatedCorpus)).isTrue(); 125 try (Stream<Path> entries = Files.list(generatedCorpus)) { 126 assertThat(entries).isNotEmpty(); 127 } 128 } 129 130 @Test fuzzingDisabled()131 public void fuzzingDisabled() { 132 assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); 133 134 EngineExecutionResults results = 135 EngineTestKit.engine("junit-jupiter") 136 .selectors(selectMethod( 137 "com.example.AutofuzzWithCorpusFuzzTest#autofuzzWithCorpus(java.lang.String,int)")) 138 .execute(); 139 140 final String engine = "engine:junit-jupiter"; 141 final String clazz = "class:com.example.AutofuzzWithCorpusFuzzTest"; 142 final String autofuzzWithCorpus = "test-template:autofuzzWithCorpus(java.lang.String, int)"; 143 144 results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(engine)), 145 event(type(STARTED), container(uniqueIdSubstrings(engine, clazz))), 146 event(type(STARTED), container(uniqueIdSubstrings(engine, clazz, autofuzzWithCorpus))), 147 // "No fuzzing has been performed..." 148 event(type(REPORTING_ENTRY_PUBLISHED), 149 container(uniqueIdSubstrings(engine, clazz, autofuzzWithCorpus))), 150 event(type(FINISHED), container(uniqueIdSubstrings(engine, clazz, autofuzzWithCorpus)), 151 finishedSuccessfully()), 152 event(type(FINISHED), container(uniqueIdSubstrings(engine, clazz)), finishedSuccessfully()), 153 event(type(FINISHED), container(engine), finishedSuccessfully())); 154 155 results.testEvents().assertEventsMatchExactly(event(type(DYNAMIC_TEST_REGISTERED)), 156 event(type(STARTED)), 157 event(test("autofuzzWithCorpus", "<empty input>"), finishedSuccessfully()), 158 event(type(DYNAMIC_TEST_REGISTERED)), event(type(STARTED)), 159 event(test("autofuzzWithCorpus", "crashing_input"), 160 finishedWithFailure(instanceOf(RuntimeException.class)))); 161 } 162 } 163