1 // Copyright 2022 Code Intelligence GmbH 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package com.code_intelligence.jazzer.junit; 16 17 import java.lang.annotation.ElementType; 18 import java.lang.annotation.Retention; 19 import java.lang.annotation.RetentionPolicy; 20 import java.lang.annotation.Target; 21 import org.junit.jupiter.api.Tag; 22 import org.junit.jupiter.api.extension.ExtendWith; 23 import org.junit.jupiter.api.parallel.ResourceAccessMode; 24 import org.junit.jupiter.api.parallel.ResourceLock; 25 import org.junit.jupiter.api.parallel.Resources; 26 import org.junit.jupiter.params.ParameterizedTest; 27 import org.junit.jupiter.params.provider.ArgumentsSource; 28 29 /** 30 * A parameterized test with parameters generated automatically by the Java fuzzer <a 31 * href="https://github.com/CodeIntelligenceTesting/jazzer">Jazzer</a>. 32 * 33 * <h2>Test parameters</h2> 34 * 35 * <p>Methods annotated with {@link FuzzTest} can take either of the following types of parameters: 36 * 37 * <dl> 38 * <dt>{@code byte[]}</dt> 39 * <dd>Raw byte input mutated by the fuzzer. Use this signature when your fuzz test naturally 40 * handles raw bytes (e.g. when fuzzing a binary format parser). This is the most efficient, but 41 * also the least convenient way to write a fuzz test.</dd> 42 * 43 * <dt>{@link com.code_intelligence.jazzer.api.FuzzedDataProvider}</dt> 44 * <dd>Provides convenience methods that generate instances of commonly used Java types from the raw 45 * fuzzer input. This is generally the best way to write fuzz tests.</dd> 46 * 47 * <dt>any non-zero number of parameters of any type</dt> 48 * <dd>In this case, Jazzer will rely on reflection and class path scanning to instantiate concrete 49 * arguments. While convenient and a good way to get started, fuzz tests using this feature will 50 * generally be less efficient than fuzz tests using any of the other possible signatures. Due to 51 * the reliance on class path scanning, any change to the class path may also render previous 52 * findings unreproducible.</dd> 53 * </dl> 54 * 55 * <h2>Test modes</h2> 56 * 57 * A fuzz test can be run in two modes: fuzzing and regression testing. 58 * 59 * <h3>Fuzzing</h3> 60 * <p>When the environment variable {@code JAZZER_FUZZ} is set to any non-empty value, fuzz tests 61 * run in "fuzzing" mode. In this mode, the method annotated with {@link FuzzTest} are invoked 62 * repeatedly with inputs that Jazzer generates and mutates based on feedback obtained from 63 * instrumentation it applies to the test and every class loaded by it. 64 * 65 * <p>When an assertion in the test fails, an exception is thrown but not caught, or Jazzer's 66 * instrumentation detects a security issue (e.g. SQL injection or insecure deserialization), the 67 * fuzz test is reported as failed and the input is collected in the inputs directory for the test 68 * class (see "Regression testing" for details). 69 * 70 * <p>When no issue has been found after the configured {@link FuzzTest#maxDuration()}, the test 71 * passes. 72 * 73 * <p>Only a single fuzz test per test run will be executed in fuzzing mode. All other fuzz tests 74 * will be skipped. 75 * 76 * <h3>Regression testing</h3> 77 * <p>By default, a fuzz test is executed as a regular JUnit {@link ParameterizedTest} running on a 78 * fixed set of inputs. It can be run together with regular unit tests and used to verify that past 79 * findings remain fixed. In IDEs with JUnit 5 integration, it can also be used to conveniently 80 * debug individual findings. 81 * 82 * <p>Fuzz tests are always executed on the empty input as well as all input files contained in the 83 * resource directory called {@code <TestClassName>Inputs} in the current package. For example, 84 * all fuzz tests contained in the class {@code com.example.MyFuzzTests} would run on all files 85 * under {@code src/test/resources/com/example/MyFuzzTestsInputs}. 86 */ 87 @Target({ElementType.METHOD}) 88 @Retention(RetentionPolicy.RUNTIME) 89 @AgentConfiguringArgumentsProviderArgumentsSource 90 @ArgumentsSource(SeedArgumentsProvider.class) 91 @FuzzingArgumentsProviderArgumentsSource 92 @ExtendWith(FuzzTestExtensions.class) 93 // {0} is expanded to the basename of the seed by the ArgumentProvider. 94 @ParameterizedTest(name = "{0}") 95 @Tag("jazzer") 96 // Fuzz tests can't run in parallel with other fuzz tests since the last finding is kept in a global 97 // variable. 98 // Fuzz tests also can't run in parallel with other non-fuzz tests since method hooks are enabled 99 // conditionally based on a global variable. 100 @ResourceLock(value = Resources.GLOBAL, mode = ResourceAccessMode.READ_WRITE) 101 public @interface FuzzTest { 102 /** 103 * A duration string such as "1h 2m 30s" indicating for how long the fuzz test should be executed 104 * during fuzzing. 105 * 106 * <p>This option has no effect during regression testing. 107 */ maxDuration()108 String maxDuration() default "5m"; 109 } 110 111 // Internal use only. 112 // These wrappers are needed only because the container annotation for @ArgumentsSource, 113 // @ArgumentsSources, can't be applied to annotations. 114 @Target({ElementType.ANNOTATION_TYPE}) 115 @Retention(RetentionPolicy.RUNTIME) 116 @ArgumentsSource(AgentConfiguringArgumentsProvider.class) 117 @interface AgentConfiguringArgumentsProviderArgumentsSource {} 118 119 @Target({ElementType.ANNOTATION_TYPE}) 120 @Retention(RetentionPolicy.RUNTIME) 121 @ArgumentsSource(FuzzingArgumentsProvider.class) 122 @interface FuzzingArgumentsProviderArgumentsSource {} 123