1// Copyright 2020 the V8 project authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5/** 6 * @fileoverview Corpus 7 */ 8 9const program = require('commander'); 10const fs = require('fs'); 11const path = require('path'); 12 13const exceptions = require('./exceptions.js'); 14const random = require('./random.js'); 15const sourceHelpers = require('./source_helpers.js'); 16 17function* walkDirectory(directory, filter) { 18 // Generator for recursively walk a directory. 19 for (const filePath of fs.readdirSync(directory)) { 20 const currentPath = path.join(directory, filePath); 21 const stat = fs.lstatSync(currentPath); 22 if (stat.isFile()) { 23 if (!filter || filter(currentPath)) { 24 yield currentPath; 25 } 26 continue; 27 } 28 29 if (stat.isDirectory()) { 30 for (let childFilePath of walkDirectory(currentPath, filter)) { 31 yield childFilePath; 32 } 33 } 34 } 35} 36 37class Corpus { 38 // Input corpus. 39 constructor(inputDir, corpusName, extraStrict=false) { 40 this.inputDir = inputDir; 41 this.extraStrict = extraStrict; 42 43 // Filter for permitted JS files. 44 function isPermittedJSFile(absPath) { 45 return (absPath.endsWith('.js') && 46 !exceptions.isTestSkippedAbs(absPath)); 47 } 48 49 // Cache relative paths of all files in corpus. 50 this.skippedFiles = []; 51 this.softSkippedFiles = []; 52 this.permittedFiles = []; 53 const directory = path.join(inputDir, corpusName); 54 for (const absPath of walkDirectory(directory, isPermittedJSFile)) { 55 const relPath = path.relative(this.inputDir, absPath); 56 if (exceptions.isTestSkippedRel(relPath)) { 57 this.skippedFiles.push(relPath); 58 } else if (exceptions.isTestSoftSkippedAbs(absPath) || 59 exceptions.isTestSoftSkippedRel(relPath)) { 60 this.softSkippedFiles.push(relPath); 61 } else { 62 this.permittedFiles.push(relPath); 63 } 64 } 65 random.shuffle(this.softSkippedFiles); 66 random.shuffle(this.permittedFiles); 67 } 68 69 // Relative paths of all files in corpus. 70 *relFiles() { 71 for (const relPath of this.permittedFiles) { 72 yield relPath; 73 } 74 for (const relPath of this.softSkippedFiles) { 75 yield relPath; 76 } 77 } 78 79 // Relative paths of all files in corpus including generated skipped. 80 *relFilesForGenSkipped() { 81 for (const relPath of this.relFiles()) { 82 yield relPath; 83 } 84 for (const relPath of this.skippedFiles) { 85 yield relPath; 86 } 87 } 88 89 /** 90 * Returns "count" relative test paths, randomly selected from soft-skipped 91 * and permitted files. Permitted files have a 4 times higher chance to 92 * be chosen. 93 */ 94 getRandomTestcasePaths(count) { 95 return random.twoBucketSample( 96 this.softSkippedFiles, this.permittedFiles, 4, count); 97 } 98 99 loadTestcase(relPath, strict, label) { 100 const start = Date.now(); 101 try { 102 const source = sourceHelpers.loadSource(this.inputDir, relPath, strict); 103 if (program.verbose) { 104 const duration = Date.now() - start; 105 console.log(`Parsing ${relPath} ${label} took ${duration} ms.`); 106 } 107 return source; 108 } catch (e) { 109 console.log(`WARNING: failed to ${label} parse ${relPath}`); 110 console.log(e); 111 } 112 return undefined; 113 } 114 115 *loadTestcases(relPaths) { 116 for (const relPath of relPaths) { 117 if (this.extraStrict) { 118 // When re-generating the files marked sloppy, we additionally test if 119 // the file parses in strict mode. 120 this.loadTestcase(relPath, true, 'strict'); 121 } 122 const source = this.loadTestcase(relPath, false, 'sloppy'); 123 if (source) { 124 yield source; 125 } 126 } 127 } 128 129 getRandomTestcases(count) { 130 return Array.from(this.loadTestcases(this.getRandomTestcasePaths(count))); 131 } 132 133 getAllTestcases() { 134 return this.loadTestcases(this.relFilesForGenSkipped()); 135 } 136} 137 138module.exports = { 139 Corpus: Corpus, 140 walkDirectory: walkDirectory, 141} 142