1 // Copyright 2022 Code Intelligence GmbH 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package com.code_intelligence.jazzer.junit; 16 17 import static com.google.common.truth.Truth.assertThat; 18 import static com.google.common.truth.Truth8.assertThat; 19 import static org.junit.Assume.assumeFalse; 20 import static org.junit.Assume.assumeTrue; 21 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod; 22 import static org.junit.platform.testkit.engine.EventConditions.container; 23 import static org.junit.platform.testkit.engine.EventConditions.displayName; 24 import static org.junit.platform.testkit.engine.EventConditions.event; 25 import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; 26 import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; 27 import static org.junit.platform.testkit.engine.EventConditions.test; 28 import static org.junit.platform.testkit.engine.EventConditions.type; 29 import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; 30 import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; 31 import static org.junit.platform.testkit.engine.EventType.FINISHED; 32 import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; 33 import static org.junit.platform.testkit.engine.EventType.SKIPPED; 34 import static org.junit.platform.testkit.engine.EventType.STARTED; 35 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; 36 37 import com.google.common.truth.Truth8; 38 import java.io.IOException; 39 import java.nio.file.Files; 40 import java.nio.file.Path; 41 import java.nio.file.Paths; 42 import java.util.stream.IntStream; 43 import java.util.stream.Stream; 44 import org.assertj.core.api.Condition; 45 import org.junit.Before; 46 import org.junit.Rule; 47 import org.junit.Test; 48 import org.junit.platform.testkit.engine.EngineExecutionResults; 49 import org.junit.platform.testkit.engine.EngineTestKit; 50 import org.junit.platform.testkit.engine.Event; 51 import org.junit.platform.testkit.engine.EventType; 52 import org.junit.platform.testkit.engine.Events; 53 import org.junit.rules.TemporaryFolder; 54 import org.opentest4j.AssertionFailedError; 55 56 public class FuzzingWithoutCrashTest { 57 private static final String ENGINE = "engine:junit-jupiter"; 58 private static final String CLAZZ = "class:com.example.ValidFuzzTests"; 59 private static final String NO_CRASH_FUZZ = "test-template:noCrashFuzz([B)"; 60 private static final String INVOCATION = "test-template-invocation:#"; 61 @Rule public TemporaryFolder temp = new TemporaryFolder(); 62 Path baseDir; 63 64 @Before setup()65 public void setup() { 66 baseDir = temp.getRoot().toPath(); 67 } 68 executeTests()69 private EngineExecutionResults executeTests() { 70 return EngineTestKit.engine("junit-jupiter") 71 .selectors(selectMethod("com.example.ValidFuzzTests#noCrashFuzz(byte[])")) 72 .configurationParameter( 73 "jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*") 74 .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) 75 .execute(); 76 } 77 78 @Test fuzzingEnabled()79 public void fuzzingEnabled() throws IOException { 80 assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); 81 82 EngineExecutionResults results = executeTests(); 83 84 results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), 85 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), 86 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), 87 // Warning because the inputs directory hasn't been found in the source tree. 88 event(type(REPORTING_ENTRY_PUBLISHED), 89 container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), 90 // Warning because the inputs directory has been found on the classpath, but only in a JAR. 91 event(type(REPORTING_ENTRY_PUBLISHED), 92 container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), 93 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ)), 94 finishedSuccessfully()), 95 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), 96 event(type(FINISHED), container(ENGINE), finishedSuccessfully())); 97 98 results.testEvents().assertEventsMatchLooselyInOrder( 99 event( 100 type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), 101 event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ, INVOCATION)), 102 displayName("Fuzzing...")), 103 event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ, INVOCATION)), 104 displayName("Fuzzing..."), finishedSuccessfully())); 105 106 // Verify that the engine created the generated corpus directory. As the fuzz test produces 107 // coverage (but no crash), it should not be empty. 108 Path generatedCorpus = 109 baseDir.resolve(Paths.get(".cifuzz-corpus", "com.example.ValidFuzzTests", "noCrashFuzz")); 110 assertThat(Files.isDirectory(generatedCorpus)).isTrue(); 111 try (Stream<Path> entries = Files.list(generatedCorpus)) { 112 assertThat(entries).isNotEmpty(); 113 } 114 } 115 116 @Test fuzzingDisabled()117 public void fuzzingDisabled() { 118 assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); 119 120 EngineExecutionResults results = executeTests(); 121 122 results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), 123 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), 124 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), 125 event(type(REPORTING_ENTRY_PUBLISHED), 126 container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), 127 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), 128 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), 129 event(type(FINISHED), container(ENGINE), finishedSuccessfully())); 130 131 results.testEvents().assertEventsMatchExactly( 132 IntStream.rangeClosed(1, 6) 133 .boxed() 134 .flatMap(i 135 -> Stream.of(event(type(DYNAMIC_TEST_REGISTERED), 136 test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), 137 event(type(STARTED), 138 test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ, INVOCATION + i))), 139 event(type(FINISHED), 140 test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ, INVOCATION + i)), 141 finishedSuccessfully()))) 142 .toArray(Condition[] ::new)); 143 144 // Verify that the generated corpus directory hasn't been created. 145 Path generatedCorpus = 146 baseDir.resolve(Paths.get(".cifuzz-corpus", "com.example.ValidFuzzTests", "noCrashFuzz")); 147 assertThat(Files.notExists(generatedCorpus)).isTrue(); 148 } 149 } 150