1 // Copyright 2022 Code Intelligence GmbH 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package com.code_intelligence.jazzer.junit; 16 17 import static org.junit.Assume.assumeFalse; 18 import static org.junit.Assume.assumeTrue; 19 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; 20 import static org.junit.platform.testkit.engine.EventConditions.container; 21 import static org.junit.platform.testkit.engine.EventConditions.displayName; 22 import static org.junit.platform.testkit.engine.EventConditions.event; 23 import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; 24 import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; 25 import static org.junit.platform.testkit.engine.EventConditions.reportEntry; 26 import static org.junit.platform.testkit.engine.EventConditions.test; 27 import static org.junit.platform.testkit.engine.EventConditions.type; 28 import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; 29 import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; 30 import static org.junit.platform.testkit.engine.EventType.FINISHED; 31 import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; 32 import static org.junit.platform.testkit.engine.EventType.STARTED; 33 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; 34 35 import java.io.IOException; 36 import java.nio.file.Files; 37 import java.nio.file.Path; 38 import java.nio.file.Paths; 39 import org.assertj.core.api.Condition; 40 import org.junit.Before; 41 import org.junit.Rule; 42 import org.junit.Test; 43 import org.junit.platform.engine.reporting.ReportEntry; 44 import org.junit.platform.testkit.engine.EngineExecutionResults; 45 import org.junit.platform.testkit.engine.EngineTestKit; 46 import org.junit.platform.testkit.engine.Event; 47 import org.junit.rules.TemporaryFolder; 48 49 public class MutatorTest { 50 private static final String ENGINE = "engine:junit-jupiter"; 51 private static final String CLASS_NAME = "com.example.MutatorFuzzTest"; 52 private static final String CLAZZ = "class:" + CLASS_NAME; 53 private static final String LIFECYCLE_FUZZ = "test-template:mutatorFuzz(java.util.List)"; 54 private static final String INVOCATION = "test-template-invocation:#"; 55 private static final String INVALID_SIGNATURE_ENTRY = 56 "Some files in the seed corpus do not match the fuzz target signature.\n" 57 + "This indicates that they were generated with a different signature and may cause issues reproducing previous findings."; 58 59 @Rule public TemporaryFolder temp = new TemporaryFolder(); 60 private Path baseDir; 61 62 @Before setup()63 public void setup() throws IOException { 64 baseDir = temp.getRoot().toPath(); 65 Path inputsDirectory = baseDir.resolve(Paths.get( 66 "src", "test", "resources", "com", "example", "MutatorFuzzTestInputs", "mutatorFuzz")); 67 Files.createDirectories(inputsDirectory); 68 Files.write(inputsDirectory.resolve("invalid"), "invalid input".getBytes()); 69 } 70 executeTests()71 private EngineExecutionResults executeTests() { 72 System.setProperty("jazzer.experimental_mutator", "true"); 73 return EngineTestKit.engine("junit-jupiter") 74 .selectors(selectClass(CLASS_NAME)) 75 .configurationParameter("jazzer.instrument", "com.example.**") 76 .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) 77 .execute(); 78 } 79 80 @Test fuzzingEnabled()81 public void fuzzingEnabled() { 82 assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); 83 84 EngineExecutionResults results = executeTests(); 85 86 results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), 87 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), 88 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), 89 // Invalid corpus input warning 90 event(type(REPORTING_ENTRY_PUBLISHED), 91 container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)), 92 new Condition<>( 93 Event.byPayload(ReportEntry.class, 94 (it) -> it.getKeyValuePairs().values().contains(INVALID_SIGNATURE_ENTRY)), 95 "has invalid signature entry reporting entry")), 96 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)), 97 finishedSuccessfully()), 98 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), 99 event(type(FINISHED), container(ENGINE), finishedSuccessfully())); 100 101 results.testEvents().assertEventsMatchExactly( 102 event(type(DYNAMIC_TEST_REGISTERED), 103 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1))), 104 event(type(STARTED), 105 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), 106 displayName("<empty input>")), 107 event(type(FINISHED), 108 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), 109 displayName("<empty input>"), finishedSuccessfully()), 110 event(type(DYNAMIC_TEST_REGISTERED), 111 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2))), 112 event(type(STARTED), 113 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)), 114 displayName("invalid")), 115 event(type(FINISHED), 116 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)), 117 displayName("invalid"), finishedSuccessfully()), 118 event( 119 type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), 120 event(type(STARTED), 121 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 3)), 122 displayName("Fuzzing...")), 123 event(type(FINISHED), 124 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 3)), 125 displayName("Fuzzing..."), finishedWithFailure(instanceOf(AssertionError.class)))); 126 } 127 128 @Test fuzzingDisabled()129 public void fuzzingDisabled() { 130 assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); 131 132 EngineExecutionResults results = executeTests(); 133 134 results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), 135 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), 136 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), 137 // Deactivated fuzzing warning 138 event(type(REPORTING_ENTRY_PUBLISHED), 139 container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), 140 // Invalid corpus input warning 141 event(type(REPORTING_ENTRY_PUBLISHED), 142 container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), 143 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), 144 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), 145 event(type(FINISHED), container(ENGINE), finishedSuccessfully())); 146 147 results.testEvents().assertEventsMatchExactly( 148 event(type(DYNAMIC_TEST_REGISTERED), 149 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1))), 150 event(type(STARTED), 151 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), 152 displayName("<empty input>")), 153 event(type(FINISHED), 154 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), 155 displayName("<empty input>"), finishedSuccessfully()), 156 event(type(DYNAMIC_TEST_REGISTERED), 157 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2))), 158 event(type(STARTED), 159 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)), 160 displayName("invalid")), 161 event(type(FINISHED), 162 test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)), 163 displayName("invalid"), finishedSuccessfully())); 164 } 165 } 166