diff --git a/.travis.yml b/.travis.yml index 4555503..d7d0aa9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ language: node_js node_js: - "10" +services: + - redis-server diff --git a/app/get-page.js b/app/get-page.js index 0d733bf..3bc7dac 100644 --- a/app/get-page.js +++ b/app/get-page.js @@ -1,4 +1,5 @@ const got = require('got') +const {redisGet, redisSet} = require('../redis') const DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; allOrigins/${global.AO_VERSION}; +http://allorigins.ml/)` @@ -6,7 +7,7 @@ module.exports = getPage function getPage ({url, format, requestMethod}) { if (format === 'info' || requestMethod === 'HEAD') { - return getPageInfo(url) + return getPageInfo(url) } else if (format === 'raw') { return getRawPage(url, requestMethod) } @@ -19,10 +20,10 @@ async function getPageInfo (url) { if (error) return processError(error) return { - 'url': url, - 'content_type': response.headers['content-type'], - 'content_length': +(response.headers['content-length']) || -1, - 'http_code': response.statusCode + url: url, + content_type: response.headers['content-type'], + content_length: +response.headers['content-length'] || -1, + http_code: response.statusCode, } } @@ -31,7 +32,11 @@ async function getRawPage (url, requestMethod) { if (error) return processError(error) const contentLength = Buffer.byteLength(content) - return {content, contentType: response.headers['content-type'], contentLength} + return { + content, + contentType: response.headers['content-type'], + contentLength, + } } async function getPageContents (url, requestMethod) { @@ -42,25 +47,47 @@ async function getPageContents (url, requestMethod) { return { contents: content.toString(), status: { - 'url': url, - 'content_type': response.headers['content-type'], - 'content_length': contentLength, - 'http_code': response.statusCode, - } + url: url, + content_type: response.headers['content-type'], + content_length: contentLength, + http_code: response.statusCode, + }, } } async function request (url, requestMethod) { try { + let response const options = { - 'method': requestMethod, - 'encoding': null, - 'headers': {'user-agent': process.env.USER_AGENT || DEFAULT_USER_AGENT} + method: requestMethod, + encoding: null, + headers: { + 'user-agent': process.env.USER_AGENT || DEFAULT_USER_AGENT, + }, } + const dat = await redisGet(url + requestMethod) + if (dat) { + const {body, etag: e} = JSON.parse(dat) - const response = await got(url, options) - if (options.method === 'HEAD') return {response} + options.headers['if-none-match'] = e + + response = await got(url, options) + response.body = Buffer.from(body) + return processContent(response) + } + + response = await got(url, options) + if (options.method === 'HEAD') return {response} + if (response.headers.etag) { + redisSet( + url + requestMethod, + JSON.stringify({ + etag: response.headers.etag, + body: response.body.toString(), + }) + ) + } return processContent(response) } catch (error) { return {error} @@ -68,7 +95,7 @@ async function request (url, requestMethod) { } async function processContent (response) { - const res = {'response': response, 'content': response.body} + const res = {response: response, content: response.body} return res } @@ -82,9 +109,10 @@ async function processError (e) { return { contents: body.toString(), status: { - url, http_code, - 'content_type': headers['content-type'], - 'content_length': contentLength - } + url, + http_code, + content_type: headers['content-type'], + content_length: contentLength, + }, } } diff --git a/app/process-request.js b/app/process-request.js index f28067f..a9d1a0d 100644 --- a/app/process-request.js +++ b/app/process-request.js @@ -1,51 +1,52 @@ -const getPage = require('./get-page') +const getPage = require('./get-page') module.exports = processRequest async function processRequest (req, res) { - if (req.method === 'OPTIONS') { - return res.end() - } - - const startTime = new Date() - const params = parseParams(req) - const page = await getPage(params) - return createResponse(page, params, res, startTime) + if (req.method === 'OPTIONS') { + return res.end() + } + + const startTime = new Date() + const params = parseParams(req) + const page = await getPage(params) + return createResponse(page, params, res, startTime) } function parseParams (req) { - const params = { - requestMethod: req.method, - ...req.query, - ...req.params - } - params.requestMethod = parseRequestMethod(params.requestMethod) - params.format = (params.format || 'json').toLowerCase() - return params + const params = { + requestMethod: req.method, + ...req.query, + ...req.params + } + params.requestMethod = parseRequestMethod(params.requestMethod) + params.format = (params.format || 'json').toLowerCase() + return params } function parseRequestMethod (method) { - method = (method || '').toUpperCase() + method = (method || '').toUpperCase() - if (['HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'].includes(method)) { - return method - } - return 'GET' + if (['HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'].includes(method)) { + return method + } + return 'GET' } function createResponse (page, params, res, startTime) { - if (params.format === 'raw' && !(page.status || {}).error) { - res.set('Content-Length', page.contentLength) - res.set('Content-Type', page.contentType) - return res.send(page.content) - } + if (params.format === 'raw' && !(page.status || {}).error) { + res.set('Content-Length', page.contentLength) + res.set('Content-Type', page.contentType) + return res.send(page.content) + } - if (params.charset) res.set('Content-Type', `application/json; charset=${params.charset}`) - else res.set('Content-Type', 'application/json') + if (params.charset) { + res.set('Content-Type', `application/json; charset=${params.charset}`) + } else res.set('Content-Type', 'application/json') - if (page.status) page.status.response_time = (new Date() - startTime) - else page.response_time = (new Date() - startTime) + if (page.status) page.status.response_time = new Date() - startTime + else page.response_time = new Date() - startTime - if (params.callback) return res.jsonp(page) - return res.send(JSON.stringify(page)) + if (params.callback) return res.jsonp(page) + return res.send(JSON.stringify(page)) } diff --git a/package-lock.json b/package-lock.json index c64552e..723cc3c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -483,9 +483,9 @@ } }, "acorn": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.2.0.tgz", - "integrity": "sha512-apwXVmYVpQ34m/i71vrApRrRKCWQnZZF1+npOD0WV5xZFfwWOmKGQ2RWlfdy9vWITsenisM8M0Qeq8agcFHNiQ==", + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.3.1.tgz", + "integrity": "sha512-tLc0wSnatxAQHVHUapaHdz72pi9KUyHjq5KyHjGg9Y8Ifdc79pTh2XvI6I1/chZbnM7QtNKzh66ooDogPZSleA==", "dev": true }, "acorn-globals": { @@ -1037,9 +1037,9 @@ } }, "cli-width": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", - "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", "dev": true }, "cliui": { @@ -1348,6 +1348,11 @@ "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", "dev": true }, + "denque": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", + "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -1593,9 +1598,9 @@ } }, "eslint-visitor-keys": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.2.0.tgz", - "integrity": "sha512-WFb4ihckKil6hu3Dp798xdzSfddwKKU3+nGniKF6HfeW6OLd2OUDEPP7TcHtB5+QXOKg2s6B2DaMPE1Nn/kxKQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", "dev": true }, "espree": { @@ -2928,21 +2933,21 @@ "dev": true }, "inquirer": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.1.0.tgz", - "integrity": "sha512-5fJMWEmikSYu0nv/flMc475MhGbB7TSPd/2IpFV4I4rMklboCH2rQjYY5kKiYGHqUF9gvaambupcJFFG9dvReg==", + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.3.0.tgz", + "integrity": "sha512-K+LZp6L/6eE5swqIcVXrxl21aGDU4S50gKH0/d96OMQnSBCyGyZl/oZhbkVmdp5sBoINHd4xZvFSARh2dk6DWA==", "dev": true, "requires": { "ansi-escapes": "^4.2.1", - "chalk": "^3.0.0", + "chalk": "^4.1.0", "cli-cursor": "^3.1.0", - "cli-width": "^2.0.0", + "cli-width": "^3.0.0", "external-editor": "^3.0.3", "figures": "^3.0.0", "lodash": "^4.17.15", "mute-stream": "0.0.8", "run-async": "^2.4.0", - "rxjs": "^6.5.3", + "rxjs": "^6.6.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0", "through": "^2.3.6" @@ -2974,9 +2979,9 @@ } }, "chalk": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", - "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", "dev": true, "requires": { "ansi-styles": "^4.1.0", @@ -4747,6 +4752,35 @@ "util.promisify": "^1.0.0" } }, + "redis": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz", + "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==", + "requires": { + "denque": "^1.4.1", + "redis-commands": "^1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0" + } + }, + "redis-commands": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz", + "integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", @@ -4943,9 +4977,9 @@ "dev": true }, "rxjs": { - "version": "6.5.5", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz", - "integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==", + "version": "6.6.0", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.0.tgz", + "integrity": "sha512-3HMA8z/Oz61DUHe+SdOiQyzIf4tOx5oQHmMir7IZEu6TMqCLHT4LRcmNaUS0NwOz8VLvmmBduMsoaUvMaIiqzg==", "dev": true, "requires": { "tslib": "^1.9.0" diff --git a/package.json b/package.json index 77965b7..b66bcd5 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ }, "dependencies": { "express": "^4.17.1", - "got": "^9.6.0" + "got": "^9.6.0", + "redis": "^3.0.2" }, "devDependencies": { "eslint": "^6.8.0", diff --git a/redis.js b/redis.js new file mode 100644 index 0000000..b5b1230 --- /dev/null +++ b/redis.js @@ -0,0 +1,16 @@ +const redis = require("redis"); +const redisUrl = process.env.REDIS_URL || "redis://127.0.0.1:6379"; +const client = redis.createClient(redisUrl); +const util = require("util"); +client.get = util.promisify(client.get); + +exports.redisGet = async (key) => { + return client.get(key); +}; + +exports.redisSet = (key, val) => { + client.set(key, val, "EX", 60 * 60 * 24); +}; +exports.closeInstance = () => { + client.quit(() => {}); +}; diff --git a/test/app.test.js b/test/app.test.js index 759028b..71143c3 100644 --- a/test/app.test.js +++ b/test/app.test.js @@ -1,23 +1,25 @@ const request = require('supertest') const nock = require('nock') const app = require('../app.js') - - +const redis = require('../redis') beforeAll(() => { nock('http://example.com') - .persist() - .get('/test.html') - .reply(200, 'Hi, allOrigins!') + .persist() + .get('/test.html') + .reply(200, 'Hi, allOrigins!') - .get('/not-found.html') - .reply(404, 'not found!') + .get('/not-found.html') + .reply(404, 'not found!') - .post('/test.html') - .reply(200, "Hi, allOrigins! It's a POST!") + .post('/test.html') + .reply(200, "Hi, allOrigins! It's a POST!") - .head('/test.html') - .reply(204, undefined, {'Content-Type': 'text/html', 'Content-Length': 'invalid'}) + .head('/test.html') + .reply(204, undefined, { + 'Content-Type': 'text/html', + 'Content-Length': 'invalid', + }) }) test('global.AO_VERSION is defined', () => { @@ -49,7 +51,9 @@ test('Test POST to /get endpoint', async (done) => { }) test('Test /get request to not found url', async (done) => { - const res = await request(app).get('/get?url=http://example.com/not-found.html') + const res = await request(app).get( + '/get?url=http://example.com/not-found.html' + ) expect(res.statusCode).toBe(200) @@ -93,9 +97,15 @@ test('Test OPTIONS request', async (done) => { expect(res.statusCode).toBe(200) // 'cause we accept requests from allOrigins :D expect(res.headers['access-control-allow-origin']).toBe(RANDOM_ORIGIN) - expect(res.headers['access-control-allow-methods']).toBe('OPTIONS, GET, POST, PATCH, PUT, DELETE') + expect(res.headers['access-control-allow-methods']).toBe( + 'OPTIONS, GET, POST, PATCH, PUT, DELETE' + ) expect(res.body.contents).toBeUndefined() done() }) + +afterAll(() => { + redis.closeInstance() +})