• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2020 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5/**
6 * @fileoverview Tests for differential fuzzing.
7 */
8
9'use strict';
10
11const assert = require('assert');
12const program = require('commander');
13const sinon = require('sinon');
14
15const helpers = require('./helpers.js');
16const scriptMutator = require('../script_mutator.js');
17const sourceHelpers = require('../source_helpers.js');
18const random = require('../random.js');
19
20const { DifferentialFuzzMutator, DifferentialFuzzSuppressions } = require(
21    '../mutators/differential_fuzz_mutator.js');
22const { DifferentialScriptMutator } = require(
23    '../differential_script_mutator.js');
24
25const sandbox = sinon.createSandbox();
26
27function testMutators(settings, mutatorClass, inputFile, expectedFile) {
28  const source = helpers.loadTestData('differential_fuzz/' + inputFile);
29
30  const mutator = new mutatorClass(settings);
31  mutator.mutate(source);
32
33  const mutated = sourceHelpers.generateCode(source);
34  helpers.assertExpectedResult(
35      'differential_fuzz/' + expectedFile, mutated);
36}
37
38describe('Differential fuzzing', () => {
39  beforeEach(() => {
40    // Zero settings for all mutators.
41    this.settings = scriptMutator.defaultSettings();
42    for (const key of Object.keys(this.settings)) {
43      this.settings[key] = 0.0;
44    }
45    // By default, deterministically use all mutations of differential
46    // fuzzing.
47    this.settings['DIFF_FUZZ_EXTRA_PRINT'] = 1.0;
48    this.settings['DIFF_FUZZ_TRACK_CAUGHT'] = 1.0;
49
50    // Fake fuzzer being called with --input_dir flag.
51    this.oldInputDir = program.input_dir;
52    program.input_dir = helpers.BASE_DIR;
53  });
54
55  afterEach(() => {
56    sandbox.restore();
57    program.input_dir = this.oldInputDir;
58  });
59
60  it('applies suppressions', () => {
61    // This selects the first random variable when replacing .arguments.
62    sandbox.stub(random, 'single').callsFake(a => a[0]);
63    testMutators(
64        this.settings,
65        DifferentialFuzzSuppressions,
66        'suppressions.js',
67        'suppressions_expected.js');
68  });
69
70  it('adds extra printing', () => {
71    testMutators(
72        this.settings,
73        DifferentialFuzzMutator,
74        'mutations.js',
75        'mutations_expected.js');
76  });
77
78  it('does no extra printing', () => {
79    this.settings['DIFF_FUZZ_EXTRA_PRINT'] = 0.0;
80    testMutators(
81        this.settings,
82        DifferentialFuzzMutator,
83        'exceptions.js',
84        'exceptions_expected.js');
85  });
86
87  it('runs end to end', () => {
88    // Don't choose any zeroed settings or IGNORE_DEFAULT_PROB in try-catch
89    // mutator. Choose using original flags with >= 2%.
90    const chooseOrigFlagsProb = 0.2;
91    sandbox.stub(random, 'choose').callsFake((p) => p >= chooseOrigFlagsProb);
92
93    // Fake build directory from which two json configurations for flags are
94    // loaded.
95    const env = {
96      APP_DIR: 'test_data/differential_fuzz',
97      GENERATE: process.env.GENERATE,
98    };
99    sandbox.stub(process, 'env').value(env);
100
101    // Fake loading resources and instead load one fixed fake file for each.
102    sandbox.stub(sourceHelpers, 'loadResource').callsFake(() => {
103      return helpers.loadTestData('differential_fuzz/fake_resource.js');
104    });
105
106    // Load input files.
107    const files = [
108      'differential_fuzz/input1.js',
109      'differential_fuzz/input2.js',
110    ];
111    const sources = files.map(helpers.loadTestData);
112
113    // Apply top-level fuzzing, with all probabilistic configs switched off.
114    this.settings['DIFF_FUZZ_EXTRA_PRINT'] = 0.0;
115    this.settings['DIFF_FUZZ_TRACK_CAUGHT'] = 0.0;
116    const mutator = new DifferentialScriptMutator(
117        this.settings, helpers.DB_DIR);
118    const mutated = mutator.mutateMultiple(sources);
119    helpers.assertExpectedResult(
120        'differential_fuzz/combined_expected.js', mutated.code);
121
122    // Flags for v8_foozzie.py are calculated from v8_fuzz_experiments.json and
123    // v8_fuzz_flags.json in test_data/differential_fuzz.
124    const expectedFlags = [
125      '--first-config=ignition',
126      '--second-config=ignition_turbo',
127      '--second-d8=d8',
128      '--second-config-extra-flags=--foo1',
129      '--second-config-extra-flags=--foo2',
130      '--first-config-extra-flags=--flag1',
131      '--second-config-extra-flags=--flag1',
132      '--first-config-extra-flags=--flag2',
133      '--second-config-extra-flags=--flag2',
134      '--first-config-extra-flags=--flag3',
135      '--second-config-extra-flags=--flag3',
136      '--first-config-extra-flags=--flag4',
137      '--second-config-extra-flags=--flag4'
138    ];
139    assert.deepEqual(expectedFlags, mutated.flags);
140  });
141});
142