/** * Command line application to test GMS and unit tests with puppeteer. * node run-wasm-gm-tests --js_file ../../out/wasm_gm_tests/wasm_gm_tests.js --wasm_file ../../out/wasm_gm_tests/wasm_gm_tests.wasm --known_hashes /tmp/gold2/tests/hashes.txt --output /tmp/gold2/tests/ --use_gpu --timeout 180 */ const puppeteer = require('puppeteer'); const express = require('express'); const path = require('path'); const bodyParser = require('body-parser'); const fs = require('fs'); const commandLineArgs = require('command-line-args'); const commandLineUsage = require('command-line-usage'); const opts = [ { name: 'js_file', typeLabel: '{underline file}', description: '(required) The path to wasm_gm_tests.js.' }, { name: 'wasm_file', typeLabel: '{underline file}', description: '(required) The path to wasm_gm_tests.wasm.' }, { name: 'known_hashes', typeLabel: '{underline file}', description: '(required) The hashes that should not be written to disk.' }, { name: 'output', typeLabel: '{underline file}', description: '(required) The directory to write the output JSON and images to.', }, { name: 'resources', typeLabel: '{underline file}', description: '(required) The directory that test images are stored in.', }, { name: 'use_gpu', description: 'Whether we should run in non-headless mode with GPU.', type: Boolean, }, { name: 'enable_simd', description: 'enable execution of wasm SIMD operations in chromium', type: Boolean }, { name: 'port', description: 'The port number to use, defaults to 8081.', type: Number, }, { name: 'help', alias: 'h', type: Boolean, description: 'Print this usage guide.' }, { name: 'timeout', description: 'Number of seconds to allow test to run.', type: Number, }, { name: 'manual_mode', description: 'If set, tests will not run automatically.', type: Boolean, }, { name: 'batch_size', description: 'Number of gms (or unit tests) to run in a batch. The main thread ' + 'of the page is only unlocked between batches. Default: 50. Use 1 for debugging.', type: Number, } ]; const usage = [ { header: 'Measuring correctness of Skia WASM code', content: 'Command line application to capture images drawn from tests', }, { header: 'Options', optionList: opts, }, ]; // Parse and validate flags. const options = commandLineArgs(opts); if (!options.port) { options.port = 8081; } if (!options.timeout) { options.timeout = 60; } if (!options.batch_size) { options.batch_size = 50; } if (options.help) { console.log(commandLineUsage(usage)); process.exit(0); } if (!options.output) { console.error('You must supply an output directory.'); console.log(commandLineUsage(usage)); process.exit(1); } if (!options.js_file) { console.error('You must supply path to wasm_gm_tests.js.'); console.log(commandLineUsage(usage)); process.exit(1); } if (!options.wasm_file) { console.error('You must supply path to wasm_gm_tests.wasm.'); console.log(commandLineUsage(usage)); process.exit(1); } if (!options.known_hashes) { console.error('You must supply path to known_hashes.txt'); console.log(commandLineUsage(usage)); process.exit(1); } if (!options.resources) { console.error('You must supply resources directory'); console.log(commandLineUsage(usage)); process.exit(1); } const resourceBaseDir = path.resolve(options.resources) // This executes recursively and synchronously. const recursivelyListFiles = (dir) => { const absolutePaths = []; const files = fs.readdirSync(dir); files.forEach((file) => { const filepath = path.join(dir, file); const stats = fs.statSync(filepath); if (stats.isDirectory()) { absolutePaths.push(...recursivelyListFiles(filepath)); } else if (stats.isFile()) { absolutePaths.push(path.relative(resourceBaseDir, filepath)); } }); return absolutePaths; }; const resourceListing = recursivelyListFiles(options.resources); console.log('Saw resources', resourceListing); const driverHTML = fs.readFileSync('run-wasm-gm-tests.html', 'utf8'); const testJS = fs.readFileSync(options.js_file, 'utf8'); const testWASM = fs.readFileSync(options.wasm_file, 'binary'); const knownHashes = fs.readFileSync(options.known_hashes, 'utf8'); // This express webserver will serve the HTML file running the benchmark and any additional assets // needed to run the tests. const app = express(); app.get('/', (req, res) => res.send(driverHTML)); app.use('/static/resources/', express.static(resourceBaseDir)); console.log('resources served from', resourceBaseDir); // This allows the server to receive POST requests of up to 10MB for image/png and read the body // as raw bytes, housed in a buffer. app.use(bodyParser.raw({ type: 'image/png', limit: '10mb' })); app.get('/static/hashes.txt', (req, res) => res.send(knownHashes)); app.get('/static/resource_listing.json', (req, res) => res.send(JSON.stringify(resourceListing))); app.get('/static/wasm_gm_tests.js', (req, res) => res.send(testJS)); app.get('/static/wasm_gm_tests.wasm', function(req, res) { // Set the MIME type so it can be streamed efficiently. res.type('application/wasm'); res.send(new Buffer(testWASM, 'binary')); }); app.post('/write_png', (req, res) => { const md5 = req.header('X-MD5-Hash'); if (!md5) { res.sendStatus(400); return; } const data = req.body; const newFile = path.join(options.output, md5 + '.png'); fs.writeFileSync(newFile, data, { encoding: 'binary', }); res.sendStatus(200); }); const server = app.listen(options.port, () => console.log('- Local web server started.')); const hash = options.use_gpu? '#gpu': '#cpu'; const targetURL = `http://localhost:${options.port}/${hash}`; const viewPort = {width: 1000, height: 1000}; // Drive chrome to load the web page from the server we have running. async function driveBrowser() { console.log('- Launching chrome for ' + options.input); const browser_args = [ '--no-sandbox', '--disable-setuid-sandbox', '--window-size=' + viewPort.width + ',' + viewPort.height, // The following two params allow Chrome to run at an unlimited fps. Note, if there is // already a chrome instance running, these arguments will have NO EFFECT, as the existing // Chrome instance will be used instead of puppeteer spinning up a new one. '--disable-frame-rate-limit', '--disable-gpu-vsync', ]; if (options.enable_simd) { browser_args.push('--enable-features=WebAssemblySimd'); } if (options.use_gpu) { browser_args.push('--ignore-gpu-blacklist'); browser_args.push('--ignore-gpu-blocklist'); browser_args.push('--enable-gpu-rasterization'); } const headless = !options.use_gpu; console.log("Running with headless: " + headless + " args: " + browser_args); let browser; let page; try { browser = await puppeteer.launch({ headless: headless, args: browser_args, executablePath: options.chromium_executable_path }); page = await browser.newPage(); await page.setViewport(viewPort); } catch (e) { console.log('Could not open the browser.', e); process.exit(1); } console.log("Loading " + targetURL); let failed = []; try { await page.goto(targetURL, { timeout: 60000, waitUntil: 'networkidle0' }); if (options.manual_mode) { console.log('Manual mode detected. Will hang'); // Wait a very long time, with the web server running. await page.waitForFunction(`window._abort_manual_mode`, { timeout: 1000000000, polling: 1000, }); } // Page is mostly loaded, wait for test harness page to report itself ready. Some resources // may still be loading. console.log('Waiting 30s for test harness to be ready'); await page.waitForFunction(`(window._testsReady === true) || window._error`, { timeout: 30000, }); const err = await page.evaluate('window._error'); if (err) { const log = await page.evaluate('window._log'); console.info(log); console.error(`ERROR: ${err}`); process.exit(1); } // There is a button with id #start_tests to click (this also makes manual debugging easier). await page.click('#start_tests'); // Rather than wait a long time for things to finish, we send progress updates every 50 tests. let batch = options.batch_size; while (true) { console.log(`Waiting ${options.timeout}s for ${options.batch_size} tests to complete`); await page.waitForFunction(`(window._testsProgress >= ${batch}) || window._testsDone || window._error`, { timeout: options.timeout*1000, }); const progress = await page.evaluate(() => { return { err: window._error, done: window._testsDone, count: window._testsProgress, }; }); if (progress.err) { const log = await page.evaluate('window._log'); console.info(log); console.error(`ERROR: ${progress.err}`); process.exit(1); } if (progress.done) { console.log(`Completed ${progress.count} tests. Finished.`); break; } console.log(`In Progress; completed ${progress.count} tests.`) batch = progress.count + options.batch_size; } const goldResults = await page.evaluate('window._results'); failed = await(page.evaluate('window._failed')); const log = await page.evaluate('window._log'); console.info(log); const jsonFile = path.join(options.output, 'gold_results.json'); fs.writeFileSync(jsonFile, JSON.stringify(goldResults)); } catch(e) { console.log('Timed out while loading, drawing, or writing to disk.', e); if (page) { const log = await page.evaluate('window._log'); console.error(log); } await browser.close(); await new Promise((resolve) => server.close(resolve)); process.exit(1); } await browser.close(); await new Promise((resolve) => server.close(resolve)); if (failed.length > 0) { console.error('Failed tests', failed); process.exit(1); } else { process.exit(0); } } driveBrowser();