1return // dont run this test for now since tape is weird and broken on 0.10 2 3var fs = require('fs') 4var JSONStream = require('../') 5var file = process.argv[2] || '/tmp/JSONStream-test-large.json' 6var size = Number(process.argv[3] || 100000) 7var tape = require('tape') 8// if (process.title !== 'browser') { 9 tape('out of mem', function (t) { 10 t.plan(1) 11 ////////////////////////////////////////////////////// 12 // Produces a random number between arg1 and arg2 13 ////////////////////////////////////////////////////// 14 var randomNumber = function (min, max) { 15 var number = Math.floor(Math.random() * (max - min + 1) + min); 16 return number; 17 }; 18 19 ////////////////////////////////////////////////////// 20 // Produces a random string of a length between arg1 and arg2 21 ////////////////////////////////////////////////////// 22 var randomString = function (min, max) { 23 24 // add several spaces to increase chanses of creating 'words' 25 var chars = ' 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; 26 var result = ''; 27 28 var randomLength = randomNumber(min, max); 29 30 for (var i = randomLength; i > 0; --i) { 31 result += chars[Math.round(Math.random() * (chars.length - 1))]; 32 } 33 return result; 34 }; 35 36 ////////////////////////////////////////////////////// 37 // Produces a random JSON document, as a string 38 ////////////////////////////////////////////////////// 39 var randomJsonDoc = function () { 40 41 var doc = { 42 "CrashOccurenceID": randomNumber(10000, 50000), 43 "CrashID": randomNumber(1000, 10000), 44 "SiteName": randomString(10, 25), 45 "MachineName": randomString(10, 25), 46 "Date": randomString(26, 26), 47 "ProcessDuration": randomString(18, 18), 48 "ThreadIdentityName": null, 49 "WindowsIdentityName": randomString(15, 40), 50 "OperatingSystemName": randomString(35, 65), 51 "DetailedExceptionInformation": randomString(100, 800) 52 }; 53 54 doc = JSON.stringify(doc); 55 doc = doc.replace(/\,/g, ',\n'); // add new lines after each attribute 56 return doc; 57 }; 58 59 ////////////////////////////////////////////////////// 60 // generates test data 61 ////////////////////////////////////////////////////// 62 var generateTestData = function (cb) { 63 64 console.log('generating large data file...'); 65 66 var stream = fs.createWriteStream(file, { 67 encoding: 'utf8' 68 }); 69 70 var i = 0; 71 var max = size; 72 var writing = false 73 var split = ',\n'; 74 var doc = randomJsonDoc(); 75 stream.write('['); 76 77 function write () { 78 if(writing) return 79 writing = true 80 while(++i < max) { 81 if(Math.random() < 0.001) 82 console.log('generate..', i + ' / ' + size) 83 if(!stream.write(doc + split)) { 84 writing = false 85 return stream.once('drain', write) 86 } 87 } 88 stream.write(doc + ']') 89 stream.end(); 90 console.log('END') 91 } 92 write() 93 stream.on('close', cb) 94 }; 95 96 ////////////////////////////////////////////////////// 97 // Shows that parsing 100000 instances using JSONStream fails 98 // 99 // After several seconds, you will get this crash 100 // FATAL ERROR: JS Allocation failed - process out of memory 101 ////////////////////////////////////////////////////// 102 var testJSONStreamParse_causesOutOfMem = function (done) { 103 var items = 0 104 console.log('parsing data files using JSONStream...'); 105 106 var parser = JSONStream.parse([true]); 107 var stream = fs.createReadStream(file); 108 stream.pipe(parser); 109 110 parser.on('data', function (data) { 111 items++ 112 if(Math.random() < 0.01) console.log(items, '...') 113 }); 114 115 parser.on('end', function () { 116 t.equal(items, size) 117 }); 118 119 }; 120 121 ////////////////////////////////////////////////////// 122 // main 123 ////////////////////////////////////////////////////// 124 125 fs.stat(file, function (err, stat) { 126 console.log(stat) 127 if(err) 128 generateTestData(testJSONStreamParse_causesOutOfMem); 129 else 130 testJSONStreamParse_causesOutOfMem() 131 }) 132 133 }) 134 135// } 136