Caching added

.travis.yml

@@ -1,3 +1,5 @@
 language: node_js
 node_js:
   - "10"
+services:
+  - redis-server

app/get-page.js

@@ -1,12 +1,13 @@
 const got = require('got')
+const {redisGet, redisSet} = require('../redis')
 
 const DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; allOrigins/${global.AO_VERSION}; +http://allorigins.ml/)`
 
 module.exports = getPage
 
 function getPage ({url, format, requestMethod}) {
   if (format === 'info' || requestMethod === 'HEAD') {
-     return getPageInfo(url)
+    return getPageInfo(url)
   } else if (format === 'raw') {
     return getRawPage(url, requestMethod)
   }
@@ -19,10 +20,10 @@ async function getPageInfo (url) {
   if (error) return processError(error)
 
   return {
-    'url': url,
-    'content_type': response.headers['content-type'],
-    'content_length': +(response.headers['content-length']) || -1,
-    'http_code': response.statusCode
+    url: url,
+    content_type: response.headers['content-type'],
+    content_length: +response.headers['content-length'] || -1,
+    http_code: response.statusCode,
   }
 }
 
@@ -31,7 +32,11 @@ async function getRawPage (url, requestMethod) {
   if (error) return processError(error)
 
   const contentLength = Buffer.byteLength(content)
-  return {content, contentType: response.headers['content-type'], contentLength}
+  return {
+    content,
+    contentType: response.headers['content-type'],
+    contentLength,
+  }
 }
 
 async function getPageContents (url, requestMethod) {
@@ -42,33 +47,55 @@ async function getPageContents (url, requestMethod) {
   return {
     contents: content.toString(),
     status: {
-      'url': url,
-      'content_type': response.headers['content-type'],
-      'content_length': contentLength,
-      'http_code': response.statusCode,
-    }
+      url: url,
+      content_type: response.headers['content-type'],
+      content_length: contentLength,
+      http_code: response.statusCode,
+    },
   }
 }
 
 async function request (url, requestMethod) {
   try {
+    let response
     const options = {
-      'method': requestMethod,
-      'encoding': null,
-      'headers': {'user-agent': process.env.USER_AGENT || DEFAULT_USER_AGENT}
+      method: requestMethod,
+      encoding: null,
+      headers: {
+        'user-agent': process.env.USER_AGENT || DEFAULT_USER_AGENT,
+      },
     }
+    const dat = await redisGet(url + requestMethod)
+    if (dat) {
+      const {body, etag: e} = JSON.parse(dat)
 
-    const response = await got(url, options)
-    if (options.method === 'HEAD') return {response}
+      options.headers['if-none-match'] = e
+
+      response = await got(url, options)
+      response.body = Buffer.from(body)
+      return processContent(response)
+    }
+
+    response = await got(url, options)
 
+    if (options.method === 'HEAD') return {response}
+    if (response.headers.etag) {
+      redisSet(
+        url + requestMethod,
+        JSON.stringify({
+          etag: response.headers.etag,
+          body: response.body.toString(),
+        })
+      )
+    }
     return processContent(response)
   } catch (error) {
     return {error}
   }
 }
 
 async function processContent (response) {
-  const res = {'response': response, 'content': response.body}
+  const res = {response: response, content: response.body}
   return res
 }
 
@@ -82,9 +109,10 @@ async function processError (e) {
   return {
     contents: body.toString(),
     status: {
-      url, http_code,
-      'content_type': headers['content-type'],
-      'content_length': contentLength
-    }
+      url,
+      http_code,
+      content_type: headers['content-type'],
+      content_length: contentLength,
+    },
   }
 }

app/process-request.js

@@ -1,51 +1,52 @@
-const getPage  = require('./get-page')
+const getPage = require('./get-page')
 
 module.exports = processRequest
 
 async function processRequest (req, res) {
-  if (req.method === 'OPTIONS') {
-    return res.end()
-  }
-
-  const startTime = new Date()
-  const params = parseParams(req)
-  const page = await getPage(params)
-  return createResponse(page, params, res, startTime)
+    if (req.method === 'OPTIONS') {
+        return res.end()
+    }
+
+    const startTime = new Date()
+    const params = parseParams(req)
+    const page = await getPage(params)
+    return createResponse(page, params, res, startTime)
 }
 
 function parseParams (req) {
-  const params = {
-    requestMethod: req.method,
-    ...req.query,
-    ...req.params
-  }
-  params.requestMethod = parseRequestMethod(params.requestMethod)
-  params.format = (params.format || 'json').toLowerCase()
-  return params
+    const params = {
+        requestMethod: req.method,
+        ...req.query,
+        ...req.params
+    }
+    params.requestMethod = parseRequestMethod(params.requestMethod)
+    params.format = (params.format || 'json').toLowerCase()
+    return params
 }
 
 function parseRequestMethod (method) {
-  method = (method || '').toUpperCase()
+    method = (method || '').toUpperCase()
 
-  if (['HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'].includes(method)) {
-    return method
-  }
-  return 'GET'
+    if (['HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'].includes(method)) {
+        return method
+    }
+    return 'GET'
 }
 
 function createResponse (page, params, res, startTime) {
-  if (params.format === 'raw' && !(page.status || {}).error) {
-    res.set('Content-Length', page.contentLength)
-    res.set('Content-Type', page.contentType)
-    return res.send(page.content)
-  }
+    if (params.format === 'raw' && !(page.status || {}).error) {
+        res.set('Content-Length', page.contentLength)
+        res.set('Content-Type', page.contentType)
+        return res.send(page.content)
+    }
 
-  if (params.charset) res.set('Content-Type', `application/json; charset=${params.charset}`)
-  else res.set('Content-Type', 'application/json')
+    if (params.charset) {
+        res.set('Content-Type', `application/json; charset=${params.charset}`)
+    } else res.set('Content-Type', 'application/json')
 
-  if (page.status) page.status.response_time = (new Date() - startTime)
-  else page.response_time = (new Date() - startTime)
+    if (page.status) page.status.response_time = new Date() - startTime
+    else page.response_time = new Date() - startTime
 
-  if (params.callback) return res.jsonp(page)
-  return res.send(JSON.stringify(page))
+    if (params.callback) return res.jsonp(page)
+    return res.send(JSON.stringify(page))
 }

package.json

@@ -12,7 +12,8 @@
   },
   "dependencies": {
     "express": "^4.17.1",
-    "got": "^9.6.0"
+    "got": "^9.6.0",
+    "redis": "^3.0.2"
   },
   "devDependencies": {
     "eslint": "^6.8.0",

redis.js

@@ -0,0 +1,16 @@
+const redis = require("redis");
+const redisUrl = process.env.REDIS_URL || "redis://127.0.0.1:6379";
+const client = redis.createClient(redisUrl);
+const util = require("util");
+client.get = util.promisify(client.get);
+
+exports.redisGet = async (key) => {
+  return client.get(key);
+};
+
+exports.redisSet = (key, val) => {
+  client.set(key, val, "EX", 60 * 60 * 24);
+};
+exports.closeInstance = () => {
+  client.quit(() => {});
+};