1 // Copyright 2022 Code Intelligence GmbH 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package com.code_intelligence.jazzer.junit; 16 17 import static com.google.common.truth.Truth8.assertThat; 18 import static org.junit.Assume.assumeTrue; 19 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectPackage; 20 import static org.junit.platform.testkit.engine.EventConditions.container; 21 import static org.junit.platform.testkit.engine.EventConditions.displayName; 22 import static org.junit.platform.testkit.engine.EventConditions.event; 23 import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; 24 import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; 25 import static org.junit.platform.testkit.engine.EventConditions.test; 26 import static org.junit.platform.testkit.engine.EventConditions.type; 27 import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; 28 import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; 29 import static org.junit.platform.testkit.engine.EventType.FINISHED; 30 import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; 31 import static org.junit.platform.testkit.engine.EventType.STARTED; 32 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; 33 import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message; 34 35 import com.code_intelligence.jazzer.api.FuzzerSecurityIssueCritical; 36 import com.code_intelligence.jazzer.api.FuzzerSecurityIssueHigh; 37 import com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow; 38 import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium; 39 import java.io.ByteArrayOutputStream; 40 import java.io.PrintStream; 41 import java.nio.charset.StandardCharsets; 42 import java.util.Arrays; 43 import org.junit.Test; 44 import org.junit.platform.testkit.engine.EngineExecutionResults; 45 import org.junit.platform.testkit.engine.EngineTestKit; 46 import org.opentest4j.AssertionFailedError; 47 48 public class RegressionTestTest { 49 private static final String ENGINE = "engine:junit-jupiter"; 50 private static final String BYTE_FUZZ_TEST = "class:com.example.ByteFuzzTest"; 51 private static final String VALID_FUZZ_TESTS = "class:com.example.ValidFuzzTests"; 52 private static final String INVALID_FUZZ_TESTS = "class:com.example.InvalidFuzzTests"; 53 private static final String AUTOFUZZ_WITH_CORPUS_FUZZ_TEST = 54 "class:com.example.AutofuzzWithCorpusFuzzTest"; 55 private static final String BYTE_FUZZ = "test-template:byteFuzz([B)"; 56 private static final String NO_CRASH_FUZZ = "test-template:noCrashFuzz([B)"; 57 private static final String DATA_FUZZ = 58 "test-template:dataFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)"; 59 private static final String INVALID_PARAMETER_COUNT_FUZZ = 60 "test-template:invalidParameterCountFuzz()"; 61 private static final String AUTOFUZZ_WITH_CORPUS = 62 "test-template:autofuzzWithCorpus(java.lang.String, int)"; 63 private static final String INVOCATION = "test-template-invocation:#"; 64 executeTests()65 private static EngineExecutionResults executeTests() { 66 return EngineTestKit.engine("junit-jupiter") 67 .selectors(selectPackage("com.example")) 68 .configurationParameter( 69 "jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*") 70 .execute(); 71 } 72 73 @Test regressionTestEnabled()74 public void regressionTestEnabled() { 75 assumeTrue(System.getenv("JAZZER_FUZZ") == null); 76 77 // Record Jazzer's stderr. 78 PrintStream stderr = System.err; 79 ByteArrayOutputStream recordedStderr = new ByteArrayOutputStream(); 80 System.setErr(new PrintStream(recordedStderr)); 81 82 EngineExecutionResults results = executeTests(); 83 System.setErr(stderr); 84 85 // Verify that Jazzer doesn't print any warning or errors. 86 String[] stderrLines = 87 new String(recordedStderr.toByteArray(), StandardCharsets.UTF_8).split("\n"); 88 for (String line : stderrLines) { 89 System.err.println(line); 90 } 91 assertThat(Arrays.stream(stderrLines) 92 .filter(line -> line.startsWith("WARN:") || line.startsWith("ERROR:"))) 93 .isEmpty(); 94 95 results.containerEvents().debug().assertEventsMatchLoosely( 96 event(type(STARTED), container(ENGINE)), 97 event( 98 type(STARTED), container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, NO_CRASH_FUZZ))), 99 event(type(REPORTING_ENTRY_PUBLISHED), 100 container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, NO_CRASH_FUZZ))), 101 event(type(FINISHED), 102 container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, NO_CRASH_FUZZ)), 103 finishedSuccessfully()), 104 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ))), 105 event(type(REPORTING_ENTRY_PUBLISHED), 106 container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ))), 107 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ)), 108 finishedSuccessfully()), 109 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS)), 110 finishedSuccessfully()), 111 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST))), 112 event(type(STARTED), 113 container( 114 uniqueIdSubstrings(ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS))), 115 event(type(REPORTING_ENTRY_PUBLISHED), 116 container( 117 uniqueIdSubstrings(ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS))), 118 event(type(FINISHED), 119 container( 120 uniqueIdSubstrings(ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS)), 121 finishedSuccessfully()), 122 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST)), 123 finishedSuccessfully()), 124 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST))), 125 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ))), 126 event(type(REPORTING_ENTRY_PUBLISHED), 127 container(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ))), 128 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ)), 129 finishedSuccessfully()), 130 event(type(STARTED), container(uniqueIdSubstrings(ENGINE, INVALID_FUZZ_TESTS))), 131 event(type(STARTED), 132 container( 133 uniqueIdSubstrings(ENGINE, INVALID_FUZZ_TESTS, INVALID_PARAMETER_COUNT_FUZZ))), 134 event(type(FINISHED), 135 container(uniqueIdSubstrings(ENGINE, INVALID_FUZZ_TESTS, INVALID_PARAMETER_COUNT_FUZZ)), 136 finishedWithFailure(instanceOf(IllegalArgumentException.class), 137 message("Methods annotated with @FuzzTest must take at least one parameter"))), 138 event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, INVALID_FUZZ_TESTS)), 139 finishedSuccessfully()), 140 event(type(FINISHED), container(ENGINE), finishedSuccessfully())); 141 142 results.testEvents().debug().assertEventsMatchLoosely( 143 event(type(DYNAMIC_TEST_REGISTERED), 144 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 145 displayName("<empty input>")), 146 event(type(STARTED), 147 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 148 displayName("<empty input>")), 149 event(type(FINISHED), 150 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 151 displayName("<empty input>"), 152 finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class))), 153 event(type(DYNAMIC_TEST_REGISTERED), 154 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 155 displayName("no_crash")), 156 event(type(STARTED), 157 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 158 displayName("no_crash")), 159 event(type(FINISHED), 160 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 161 displayName("no_crash"), finishedSuccessfully()), 162 event(type(DYNAMIC_TEST_REGISTERED), 163 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 164 displayName("assert")), 165 event(type(STARTED), 166 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 167 displayName("assert")), 168 event(type(FINISHED), 169 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 170 displayName("assert"), finishedWithFailure(instanceOf(AssertionFailedError.class))), 171 event(type(DYNAMIC_TEST_REGISTERED), 172 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 173 displayName("honeypot")), 174 event(type(STARTED), 175 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 176 displayName("honeypot")), 177 event(type(FINISHED), 178 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 179 displayName("honeypot"), 180 finishedWithFailure(instanceOf(FuzzerSecurityIssueHigh.class))), 181 event(type(DYNAMIC_TEST_REGISTERED), 182 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 183 displayName("sanitizer_internal_class")), 184 event(type(STARTED), 185 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 186 displayName("sanitizer_internal_class")), 187 event(type(FINISHED), 188 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 189 displayName("sanitizer_internal_class"), 190 finishedWithFailure(instanceOf(FuzzerSecurityIssueCritical.class))), 191 event(type(DYNAMIC_TEST_REGISTERED), 192 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 193 displayName("sanitizer_user_class")), 194 event(type(STARTED), 195 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 196 displayName("sanitizer_user_class")), 197 event(type(FINISHED), 198 test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), 199 displayName("sanitizer_user_class"), 200 finishedWithFailure(instanceOf(FuzzerSecurityIssueLow.class))), 201 event(type(DYNAMIC_TEST_REGISTERED), 202 test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), 203 displayName("<empty input>")), 204 event(type(STARTED), 205 test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), 206 displayName("<empty input>")), 207 event(type(FINISHED), 208 test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), 209 displayName("<empty input>"), finishedSuccessfully()), 210 event(type(DYNAMIC_TEST_REGISTERED), 211 test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), 212 displayName("succeeds")), 213 event(type(STARTED), 214 test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), 215 displayName("succeeds")), 216 event(type(FINISHED), 217 test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), 218 displayName("succeeds"), finishedSuccessfully()), 219 event(type(DYNAMIC_TEST_REGISTERED), 220 test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), 221 displayName("fails")), 222 event(type(STARTED), 223 test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), 224 displayName("fails")), 225 event(type(FINISHED), 226 test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), 227 displayName("fails"), finishedWithFailure(instanceOf(AssertionFailedError.class))), 228 event(type(DYNAMIC_TEST_REGISTERED), 229 test(uniqueIdSubstrings( 230 ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), 231 displayName("<empty input>")), 232 event(type(STARTED), 233 test(uniqueIdSubstrings( 234 ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), 235 displayName("<empty input>")), 236 event(type(FINISHED), 237 test(uniqueIdSubstrings( 238 ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), 239 displayName("<empty input>"), finishedSuccessfully()), 240 event(type(DYNAMIC_TEST_REGISTERED), 241 test(uniqueIdSubstrings( 242 ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), 243 displayName("crashing_input")), 244 event(type(STARTED), 245 test(uniqueIdSubstrings( 246 ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), 247 displayName("crashing_input")), 248 event(type(FINISHED), 249 test(uniqueIdSubstrings( 250 ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), 251 displayName("crashing_input"), 252 finishedWithFailure(instanceOf(RuntimeException.class)))); 253 } 254 } 255