• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2022 Code Intelligence GmbH
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.example;
18 
19 import com.code_intelligence.jazzer.api.FuzzedDataProvider;
20 import com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow;
21 import com.code_intelligence.jazzer.third_party.org.jacoco.core.data.ExecutionData;
22 import com.code_intelligence.jazzer.third_party.org.jacoco.core.data.ExecutionDataReader;
23 import com.code_intelligence.jazzer.third_party.org.jacoco.core.data.ExecutionDataStore;
24 import com.code_intelligence.jazzer.third_party.org.jacoco.core.data.SessionInfoStore;
25 import java.io.FileInputStream;
26 import java.io.IOException;
27 import java.nio.file.Files;
28 import java.nio.file.Paths;
29 import java.util.ArrayList;
30 import java.util.List;
31 import java.util.stream.Collectors;
32 import java.util.stream.IntStream;
33 
34 /**
35  * Test of coverage report and dump.
36  *
37  * Internally, JaCoCo is used to gather coverage information to guide the fuzzer to cover new
38  * branches. This information can be dumped in the JaCoCo format and used to generate reports later
39  * on. The dump only contains classes with at least one coverage data point. A JaCoCo report will
40  * also include completely uncovered files based on the available classes in the stated jar files
41  * in the report command.
42  *
43  * A human-readable coverage report can be generated directly by Jazzer. It contains information
44  * on file level about all classes that should have been instrumented according to the
45  * instrumentation_includes and instrumentation_exclude filters.
46  */
47 @SuppressWarnings({"unused", "UnusedReturnValue"})
48 public final class CoverageFuzzer {
49   // Not used during fuzz run, so not included in the dump
50   public static class ClassNotToCover {
51     private final int i;
ClassNotToCover(int i)52     public ClassNotToCover(int i) {
53       this.i = i;
54     }
getI()55     public int getI() {
56       return i;
57     }
58   }
59 
60   // Used in the fuzz run and included in the dump
61   public static class ClassToCover {
62     private final int i;
63 
ClassToCover(int i)64     public ClassToCover(int i) {
65       if (i < 0 || i > 1000) {
66         throw new IllegalArgumentException(String.format("Invalid repeat number \"%d\"", i));
67       }
68       this.i = i;
69     }
70 
repeat(String str)71     public String repeat(String str) {
72       if (str != null && str.length() >= 3 && str.length() <= 10) {
73         return IntStream.range(0, i).mapToObj(i -> str).collect(Collectors.joining());
74       }
75       throw new IllegalArgumentException(String.format("Invalid str \"%s\"", str));
76     }
77   }
78 
fuzzerTestOneInput(FuzzedDataProvider data)79   public static void fuzzerTestOneInput(FuzzedDataProvider data) {
80     try {
81       ClassToCover classToCover = new ClassToCover(data.consumeInt());
82       String repeated = classToCover.repeat(data.consumeRemainingAsAsciiString());
83       if (repeated.equals("foofoofoo")) {
84         throw new FuzzerSecurityIssueLow("Finished coverage fuzzer test");
85       }
86     } catch (IllegalArgumentException ignored) {
87     }
88   }
89 
fuzzerTearDown()90   public static void fuzzerTearDown() throws IOException {
91     assertCoverageReport();
92     assertCoverageDump();
93   }
94 
assertCoverageReport()95   private static void assertCoverageReport() throws IOException {
96     List<String> coverage = Files.readAllLines(Paths.get(System.getenv("COVERAGE_REPORT_FILE")));
97     List<List<String>> sections = new ArrayList<>(4);
98     sections.add(new ArrayList<>());
99     coverage.forEach(l -> {
100       if (l.isEmpty()) {
101         sections.add(new ArrayList<>());
102       } else {
103         sections.get(sections.size() - 1).add(l);
104       }
105     });
106 
107     List<String> branchCoverage = sections.get(0);
108     assertEquals(2, branchCoverage.size());
109     List<String> lineCoverage = sections.get(1);
110     assertEquals(2, lineCoverage.size());
111     List<String> incompleteCoverage = sections.get(2);
112     assertEquals(2, incompleteCoverage.size());
113     List<String> missedCoverage = sections.get(3);
114     assertEquals(2, missedCoverage.size());
115 
116     assertNotNull(
117         branchCoverage.stream()
118             .filter(l -> l.startsWith(CoverageFuzzer.class.getSimpleName()))
119             .findFirst()
120             .orElseThrow(() -> new IllegalStateException("Could not find branch coverage")));
121 
122     assertNotNull(
123         lineCoverage.stream()
124             .filter(l -> l.startsWith(CoverageFuzzer.class.getSimpleName()))
125             .findFirst()
126             .orElseThrow(() -> new IllegalStateException("Could not find line coverage")));
127 
128     assertNotNull(
129         incompleteCoverage.stream()
130             .filter(l -> l.startsWith(CoverageFuzzer.class.getSimpleName()))
131             .findFirst()
132             .orElseThrow(() -> new IllegalStateException("Could not find incomplete coverage")));
133 
134     String missed =
135         missedCoverage.stream()
136             .filter(l -> l.startsWith(CoverageFuzzer.class.getSimpleName()))
137             .findFirst()
138             .orElseThrow(() -> new IllegalStateException("Could not find missed coverage"));
139     List<String> missingLines = IntStream.rangeClosed(63, 79)
140                                     .mapToObj(i -> " " + i)
141                                     .filter(missed::contains)
142                                     .collect(Collectors.toList());
143     if (!missingLines.isEmpty()) {
144       throw new IllegalStateException(String.format(
145           "Missing coverage for ClassToCover on lines %s", String.join(", ", missingLines)));
146     }
147   }
148 
assertCoverageDump()149   private static void assertCoverageDump() throws IOException {
150     ExecutionDataStore executionDataStore = new ExecutionDataStore();
151     SessionInfoStore sessionInfoStore = new SessionInfoStore();
152     try (FileInputStream bais = new FileInputStream(System.getenv("COVERAGE_DUMP_FILE"))) {
153       ExecutionDataReader reader = new ExecutionDataReader(bais);
154       reader.setExecutionDataVisitor(executionDataStore);
155       reader.setSessionInfoVisitor(sessionInfoStore);
156       reader.read();
157     }
158     assertEquals(2, executionDataStore.getContents().size());
159 
160     ExecutionData coverageFuzzerCoverage = new ExecutionData(0, "", 0);
161     ExecutionData classToCoverCoverage = new ExecutionData(0, "", 0);
162     for (ExecutionData content : executionDataStore.getContents()) {
163       if (content.getName().endsWith("ClassToCover")) {
164         classToCoverCoverage = content;
165       } else {
166         coverageFuzzerCoverage = content;
167       }
168     }
169 
170     assertEquals("com/example/CoverageFuzzer", coverageFuzzerCoverage.getName());
171     assertEquals(7, countHits(coverageFuzzerCoverage.getProbes()));
172 
173     assertEquals("com/example/CoverageFuzzer$ClassToCover", classToCoverCoverage.getName());
174     assertEquals(11, countHits(classToCoverCoverage.getProbes()));
175   }
176 
countHits(boolean[] probes)177   private static int countHits(boolean[] probes) {
178     int count = 0;
179     for (boolean probe : probes) {
180       if (probe)
181         count++;
182     }
183     return count;
184   }
185 
assertEquals(T expected, T actual)186   private static <T> void assertEquals(T expected, T actual) {
187     if (!expected.equals(actual)) {
188       throw new IllegalStateException(
189           String.format("Expected \"%s\", got \"%s\"", expected, actual));
190     }
191   }
192 
assertNotNull(T actual)193   private static <T> void assertNotNull(T actual) {
194     if (actual == null) {
195       throw new IllegalStateException("Expected none null value, got null");
196     }
197   }
198 }
199