Merge pull request #1 from zeke/streaming

Streaming

README.md

@@ -1,11 +1,9 @@
-# all-the-package-names 
+# all-the-package-names
 
-A perpetually out-of-date list of all the 166,000+ public package names on npm
+A perpetually out-of-date list of all the 263663+ public package names on npm
 
 ## Installation
 
-Download node at [nodejs.org](http://nodejs.org) and install it, if you haven't already.
-
 ```sh
 npm install all-the-package-names --save
 ```
@@ -22,12 +20,12 @@ names.indexOf('crazy-new-package-name') > -1
 // => false
 
 names.length
-// => 143086
+// => 206271
 
 names.filter(name => name.includes('banana'))
 // => [ 'banana', 'banana-banana', 'banana-split', ...]
 
-// note: example uses iojs with --harmony_arrow_functions enabled
+// note: example uses node4+ for arrow functions
 ```
 
 ## License

extract.js

@@ -1,11 +1,20 @@
 #!/usr/bin/env node
 
 const fs = require('fs')
-const packages = require('./skimdb.json').rows
+const JSONStream = require('JSONStream')
+const es = require('event-stream')
 const compact = require('lodash').compact
 
-var names = packages.map(function (pkg) {
-  return pkg.value.name;
-})
+var names = []
 
-fs.writeFileSync('./names.json', JSON.stringify(compact(names), null, 2))
+fs.createReadStream('./skimdb.json')
+  .pipe(JSONStream.parse('rows.*'))
+  .pipe(es.mapSync(function (pkg) {
+    names.push(pkg.value.name)
+    process.stdout.write('.')
+    return pkg
+  }))
+  .on('end', function(){
+    fs.writeFileSync('./names.json', JSON.stringify(compact(names), null, 2))
+    console.log('wrote names.json')
+  })