Lines Matching +full:ls +full:- +full:files
3 # american fuzzy lop++ - corpus minimization tool
4 # ---------------------------------------------
14 # https://www.apache.org/licenses/LICENSE-2.0
16 # This tool tries to find the smallest subset of files in the input directory
20 # - Screening large corpora of input files before using them as a seed for
21 # afl-fuzz. The tool will remove functionally redundant files and likely
24 # (In this case, you probably also want to consider running afl-tmin on
25 # the individual files later on to reduce their size.)
27 # - Minimizing the corpus generated organically by afl-fuzz, perhaps when
28 # planning to feed it to more resource-intensive tools. The tool achieves
32 # Note that the tool doesn't modify the files themselves. For that, you want
33 # afl-tmin.
39 echo "corpus minimization tool for afl-fuzz by Michal Zalewski"
46 # Process command-line options...
81 EXTRA_PAR="$EXTRA_PAR -e"
87 EXTRA_PAR="$EXTRA_PAR -O"
91 EXTRA_PAR="$EXTRA_PAR -Q"
95 EXTRA_PAR="$EXTRA_PAR -U"
106 shift $((OPTIND-1))
110 if [ "$TARGET_BIN" = "" -o "$IN_DIR" = "" -o "$OUT_DIR" = "" ]; then
113 Usage: $0 [ options ] -- /path/to/target_app [ ... ]
117 -i dir - input directory with the starting corpus
118 -o dir - output directory for minimized files
122 -f file - location read by the fuzzed program (stdin)
123 -m megs - memory limit for child process ($MEM_LIMIT MB)
124 -t msec - run time limit for child process (none)
125 -O - use binary-only instrumentation (FRIDA mode)
126 -Q - use binary-only instrumentation (QEMU mode)
127 -U - use unicorn-based instrumentation (Unicorn mode)
131 -C - keep crashing inputs, reject everything else
132 -e - solve for edge coverage only, ignore hit counts
139 AFL_PATH: last resort location to find the afl-showmap binary
150 # echo "$IN_DIR" | grep -qE '^(/var)?/tmp/'
153 # echo "$TARGET_BIN" | grep -qE '^(/var)?/tmp/'
156 # echo "$OUT_DIR" | grep -qE '^(/var)?/tmp/'
159 # echo "$STDIN_FILE" | grep -qE '^(/var)?/tmp/'
162 # echo "$PWD" | grep -qE '^(/var)?/tmp/'
165 # if [ "$T1" = "0" -o "$T2" = "0" -o "$T3" = "0" -o "$T4" = "0" -o "$T5" = "0" ]; then
166 # echo "[-] Error: do not use this script in /tmp or /var/tmp." 1>&2
172 # If @@ is specified, but there's no -f, let's come up with a temporary input
179 if echo "$*" | grep -qF '@@'; then
189 if [ "$MEM_LIMIT" -lt "5" ]; then
190 echo "[-] Error: dangerously low memory limit." 1>&2
198 if [ "$TIMEOUT" -lt "10" ]; then
199 echo "[-] Error: dangerously low timeout." 1>&2
205 if [ ! -f "$TARGET_BIN" -o ! -x "$TARGET_BIN" ]; then
209 if [ ! -f "$TNEW" -o ! -x "$TNEW" ]; then
210 echo "[-] Error: binary '$TARGET_BIN' not found or not executable." 1>&2
218 if [ "$AFL_SKIP_BIN_CHECK" = "" -a "$QEMU_MODE" = "" -a "$FRIDA_MODE" = "" -a "$UNICORN_MODE" = "" …
220 if ! grep -qF "__AFL_SHM_ID" "$TARGET_BIN"; then
221 echo "[-] Error: binary '$TARGET_BIN' doesn't appear to be instrumented." 1>&2
227 if [ ! -d "$IN_DIR" ]; then
228 echo "[-] Error: directory '$IN_DIR' not found." 1>&2
232 test -d "$IN_DIR/default" && IN_DIR="$IN_DIR/default"
233 test -d "$IN_DIR/queue" && IN_DIR="$IN_DIR/queue"
235 find "$OUT_DIR" -name 'id[:_]*' -maxdepth 1 -exec rm -- {} \; 2>/dev/null
236 rm -rf "$TRACE_DIR" 2>/dev/null
240 if [ -d "$OUT_DIR" ]; then
241 echo "[-] Error: directory '$OUT_DIR' exists and is not empty - delete it first." 1>&2
245 mkdir -m 700 -p "$TRACE_DIR" || exit 1
248 rm -f "$STDIN_FILE" || exit 1
252 SHOWMAP=`command -v afl-showmap 2>/dev/null`
254 if [ -z "$SHOWMAP" ]; then
255 TMP="${0%/afl-cmin.bash}/afl-showmap"
256 if [ -x "$TMP" ]; then
261 if [ -z "$SHOWMAP" -a -x "./afl-showmap" ]; then
262 SHOWMAP="./afl-showmap"
264 if [ -n "$AFL_PATH" ]; then
265 SHOWMAP="$AFL_PATH/afl-showmap"
269 if [ ! -x "$SHOWMAP" ]; then
270 echo "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." 1>&2
271 rm -rf "$TRACE_DIR"
275 IN_COUNT=$((`ls -- "$IN_DIR" 2>/dev/null | wc -l`))
279 rm -rf "$TRACE_DIR"
283 FIRST_FILE=`ls "$IN_DIR" | head -1`
287 if [ -d "$IN_DIR/$FIRST_FILE" ]; then
288 echo "[-] Error: The target directory contains subdirectories - please fix." 1>&2
289 rm -rf "$TRACE_DIR"
293 # Check for the more efficient way to copy files...
301 # Make sure that we can actually get anything out of afl-showmap before we
308 …AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_…
313 …AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_…
317 FIRST_COUNT=$((`grep -c . "$TRACE_DIR/.run_test"`))
319 if [ "$FIRST_COUNT" -gt "0" ]; then
325 echo "[-] Error: no instrumentation output detected (perhaps crash or timeout)." 1>&2
326 test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR"
337 echo "[*] Obtaining traces for input files in '$IN_DIR'..."
345 ls "$IN_DIR" | while read -r fn; do
350 … "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$fn"
356 ls "$IN_DIR" | while read -r fn; do
363 …"$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -H "$STDIN_FILE" -- "$@…
379 # datasets. The reasoning here is that we won't be able to avoid the files
385 ls "$IN_DIR" | sed "s#^#$TRACE_DIR/#" | tr '\n' '\0' | xargs -0 -n 1 cat | \
386 sort | uniq -c | sort -k 1,1 -n >"$TRACE_DIR/.all_uniq"
388 TUPLE_COUNT=$((`grep -c . "$TRACE_DIR/.all_uniq"`))
390 echo "[+] Found $TUPLE_COUNT unique tuples across $IN_COUNT files."
393 # STEP 3: SELECTING CANDIDATE FILES #
406 ls -rS "$IN_DIR" | while read -r fn; do
421 # At this point, we have a file of tuple-file pairs, sorted by file size
422 # in ascending order (as a consequence of ls -rS). By doing sort keyed
423 # only by tuple (-k 1,1) and configured to output only the first line for
424 # every key (-s -u), we end up with the smallest file for each tuple.
428 sort -k1,1 -s -u "$TRACE_DIR/.candidate_list" | \
431 if [ ! -s "$TRACE_DIR/.candidate_script" ]; then
432 echo "[-] Error: no traces obtained from test cases, check syntax!" 1>&2
433 test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR"
448 # tuples associated with the newly-added file to the "already have" list. The
451 echo "[*] Processing candidates and writing output files..."
457 while read -r cnt tuple; do
464 grep -q "^$tuple\$" "$TRACE_DIR/.already_have" && continue
468 # echo "tuple nr $CUR ($tuple cnt=$cnt) -> $FN" >> "$TRACE_DIR/.log"
472 sort -u "$TRACE_DIR/$FN" "$TRACE_DIR/.already_have" >"$TRACE_DIR/.tmp"
473 mv -f "$TRACE_DIR/.tmp" "$TRACE_DIR/.already_have"
482 OUT_COUNT=`ls -- "$OUT_DIR" | wc -l`
488 echo "[+] Narrowed down to $OUT_COUNT files, saved in '$OUT_DIR'."
491 test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR"