From 0190ce6df9a423117f7853c20cec07455201080e Mon Sep 17 00:00:00 2001 From: Maurice Butler Date: Fri, 18 Dec 2015 16:08:34 +1000 Subject: [PATCH 1/2] changed method of loading npm packages --- index.js | 1 + load-npm-data.js | 79 ++++++++++++++++++++++++++++++++---------------- package.json | 9 +++--- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/index.js b/index.js index d5158b4..50b3b49 100644 --- a/index.js +++ b/index.js @@ -17,6 +17,7 @@ const fs = require('fs') "allPackagesOutput" : "/path/to/allpackages.json" , "repositoriesOutput" : "/path/to/repositories.json" , "githubOutput" : "/path/to/githubusers.json" + , "aussieOutput" : "/path/to/aussieOutput.json" , "githubAuthToken" : "yourgithubauthtoken" } diff --git a/load-npm-data.js b/load-npm-data.js index 8610d8e..5001bba 100644 --- a/load-npm-data.js +++ b/load-npm-data.js @@ -1,48 +1,75 @@ // if your GitHub location field matches this then we'll guess you're Aussie const GITHUB_REPO_REGEX = /github.com[:\/]([\.\-\w]+)\/([^$\/\.]+)/ - -const npm = require('npm') + , NPM_ALL_PACKAGES_URL = 'https://skimdb.npmjs.com/registry/_all_docs' + , NPM_SINGLE_PACKAGE_URL = 'https://registry.npmjs.org/{packageId}/latest' + , request = require('request').defaults({json:true}) + , async = require('async'); function matchGitHubRepo (npmPackage, repo) { var match = repo && typeof repo.url == 'string' - && repo.url.match(GITHUB_REPO_REGEX) + && repo.url.match(GITHUB_REPO_REGEX); return match && { githubUser : match[1] , githubRepo : match[2] , npmPackage : npmPackage - } + }; } -// load the list of all npm libs with 'repo' pointing to GitHub -function loadNpmData (callback) { - var repositories = [] - , allPackages = [] +function getPackageData(repositories, allPackages, packageData, callback){ + request(NPM_SINGLE_PACKAGE_URL.replace('{packageId}', packageData.id), function (err, response, data) { + if (err) { + // log and continue usually just a timeout, possibly needs retry logic + console.log('error getting data for package: ' + packageData.id, err.message); + return callback(); + } + + // Bad maintainers property there are MANY just skip for much speed increase + if(!data.maintainers || !Array.isArray(data.maintainers)){ + return callback(); + } + + var repo = matchGitHubRepo(data.name, data.repository); + + if (repo){ + repositories.push(repo); + } - npm.load(function (err) { - if (err) return callback(err) + allPackages.push({ + name : data.name + , maintainers : (data.maintainers || []).map(function (m) { return m && m.name }) + , githubUser : repo ? repo.githubUser : null + , githubRepo : repo ? repo.githubRepo : null + , description : data.description + }); + + callback(); + }); +} + +function getAllPackages(callback){ + var repositories = [] + , allPackages = []; - npm.registry.get('/-/all', function (err, data) { - if (err) return callback(err) + // https://github.com/npm/npm-registry-couchapp/issues/162 + request(NPM_ALL_PACKAGES_URL, function(err, response, body){ + if (err) { + return callback(err); + } - Object.keys(data).forEach(function (k) { - var repo = matchGitHubRepo(data[k].name, data[k].repository) - if (repo) - repositories.push(repo) + if(!body || !body.rows){ + body = { rows: [] }; + } - allPackages.push({ - name : data[k].name - , maintainers : (data[k].maintainers || []).map(function (m) { return m.name }) - , githubUser : repo ? repo.githubUser : null - , githubRepo : repo ? repo.githubRepo : null - , description : data[k].description - }) - }) + async.mapLimit(body.rows, 10, getPackageData.bind(null, repositories, allPackages), function(err){ + if (err) { + return callback(err); + } callback(null, { repositories: repositories, allPackages: allPackages }) }) - }) + }); } -module.exports = loadNpmData \ No newline at end of file +module.exports = getAllPackages \ No newline at end of file diff --git a/package.json b/package.json index ac44b1a..dc24041 100644 --- a/package.json +++ b/package.json @@ -7,10 +7,9 @@ "author": "", "license": "MIT", "dependencies": { - "npm": "~1.2.18", - "async": "~0.2.7", - "request": "~2.20.0", - "function-rate-limit": "0.0.1" + "async": "^1.5.0", + "function-rate-limit": "0.0.1", + "request": "^2.67.0" }, "private": true -} \ No newline at end of file +} From 6a466cec950d416ef06fcf99eb498e510db1f66d Mon Sep 17 00:00:00 2001 From: Maurice Butler Date: Fri, 18 Dec 2015 17:15:05 +1000 Subject: [PATCH 2/2] style fixes --- load-npm-data.js | 34 +++++++++++++++++----------------- package.json | 4 ++-- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/load-npm-data.js b/load-npm-data.js index 5001bba..e7e8e88 100644 --- a/load-npm-data.js +++ b/load-npm-data.js @@ -8,32 +8,32 @@ const GITHUB_REPO_REGEX = /github.com[:\/]([\.\-\w]+)\/([^$\/\.]+)/ function matchGitHubRepo (npmPackage, repo) { var match = repo && typeof repo.url == 'string' - && repo.url.match(GITHUB_REPO_REGEX); + && repo.url.match(GITHUB_REPO_REGEX) return match && { githubUser : match[1] , githubRepo : match[2] , npmPackage : npmPackage - }; + } } -function getPackageData(repositories, allPackages, packageData, callback){ +function getPackageData (repositories, allPackages, packageData, callback) { request(NPM_SINGLE_PACKAGE_URL.replace('{packageId}', packageData.id), function (err, response, data) { if (err) { // log and continue usually just a timeout, possibly needs retry logic - console.log('error getting data for package: ' + packageData.id, err.message); - return callback(); + console.log('error getting data for package: ' + packageData.id, err.message) + return callback() } // Bad maintainers property there are MANY just skip for much speed increase - if(!data.maintainers || !Array.isArray(data.maintainers)){ - return callback(); + if (!data.maintainers || !Array.isArray(data.maintainers)) { + return callback() } var repo = matchGitHubRepo(data.name, data.repository); - if (repo){ - repositories.push(repo); + if (repo) { + repositories.push(repo) } allPackages.push({ @@ -42,15 +42,15 @@ function getPackageData(repositories, allPackages, packageData, callback){ , githubUser : repo ? repo.githubUser : null , githubRepo : repo ? repo.githubRepo : null , description : data.description - }); + }) - callback(); - }); + callback() + }) } -function getAllPackages(callback){ +function getAllPackages (callback) { var repositories = [] - , allPackages = []; + , allPackages = [] // https://github.com/npm/npm-registry-couchapp/issues/162 request(NPM_ALL_PACKAGES_URL, function(err, response, body){ @@ -58,18 +58,18 @@ function getAllPackages(callback){ return callback(err); } - if(!body || !body.rows){ + if (!body || !body.rows) { body = { rows: [] }; } - async.mapLimit(body.rows, 10, getPackageData.bind(null, repositories, allPackages), function(err){ + async.mapLimit(body.rows, 10, getPackageData.bind(null, repositories, allPackages), function (err) { if (err) { return callback(err); } callback(null, { repositories: repositories, allPackages: allPackages }) }) - }); + }) } module.exports = getAllPackages \ No newline at end of file diff --git a/package.json b/package.json index dc24041..8d59053 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,9 @@ "author": "", "license": "MIT", "dependencies": { - "async": "^1.5.0", + "async": "~1.5.0", "function-rate-limit": "0.0.1", - "request": "^2.67.0" + "request": "~2.67.0" }, "private": true }