Caching added

.travis.yml

@@ -1,3 +1,5 @@
 language: node_js
 node_js:
   - "10"
+services:
+  - redis-server

app/get-page.js

@@ -1,12 +1,13 @@
 const got = require('got')
+const {redisGet, redisSet} = require('../redis')
 
 const DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; allOrigins/${global.AO_VERSION}; +http://allorigins.ml/)`
 
 module.exports = getPage
 
 function getPage ({url, format, requestMethod}) {
   if (format === 'info' || requestMethod === 'HEAD') {
-     return getPageInfo(url)
+    return getPageInfo(url)
   } else if (format === 'raw') {
     return getRawPage(url, requestMethod)
   }
@@ -19,10 +20,10 @@ async function getPageInfo (url) {
   if (error) return processError(error)
 
   return {
-    'url': url,
-    'content_type': response.headers['content-type'],
-    'content_length': +(response.headers['content-length']) || -1,
-    'http_code': response.statusCode
+    url: url,
+    content_type: response.headers['content-type'],
+    content_length: +response.headers['content-length'] || -1,
+    http_code: response.statusCode,
   }
 }
 
@@ -31,7 +32,11 @@ async function getRawPage (url, requestMethod) {
   if (error) return processError(error)
 
   const contentLength = Buffer.byteLength(content)
-  return {content, contentType: response.headers['content-type'], contentLength}
+  return {
+    content,
+    contentType: response.headers['content-type'],
+    contentLength,
+  }
 }
 
 async function getPageContents (url, requestMethod) {
@@ -42,33 +47,60 @@ async function getPageContents (url, requestMethod) {
   return {
     contents: content.toString(),
     status: {
-      'url': url,
-      'content_type': response.headers['content-type'],
-      'content_length': contentLength,
-      'http_code': response.statusCode,
-    }
+      url: url,
+      content_type: response.headers['content-type'],
+      content_length: contentLength,
+      http_code: response.statusCode,
+    },
   }
 }
 
 async function request (url, requestMethod) {
   try {
-    const options = {
-      'method': requestMethod,
-      'encoding': null,
-      'headers': {'user-agent': process.env.USER_AGENT || DEFAULT_USER_AGENT}
+    let options, response
+    const dat = await redisGet(url + requestMethod)
+    if (dat) {
+      const body = JSON.parse(dat).body
+      const e = JSON.parse(dat).etag
+
+      options = {
+        method: requestMethod,
+        encoding: null,
+        headers: {
+          'user-agent': process.env.USER_AGENT || DEFAULT_USER_AGENT,
+          'if-none-match': e,
+        },
+      }
+      response = await got(url, options)
+      response.body = Buffer.from(body)
+      return processContent(response)
+    }
+    options = {
+      method: requestMethod,
+      encoding: null,
+      headers: {'user-agent': process.env.USER_AGENT || DEFAULT_USER_AGENT},
     }
 
-    const response = await got(url, options)
-    if (options.method === 'HEAD') return {response}
+    response = await got(url, options)
 
+    if (options.method === 'HEAD') return {response}
+    if (response.headers.etag) {
+      redisSet(
+        url + requestMethod,
+        JSON.stringify({
+          etag: response.headers.etag,
+          body: response.body.toString(),
+        })
+      )
+    }
     return processContent(response)
   } catch (error) {
     return {error}
   }
 }
 
 async function processContent (response) {
-  const res = {'response': response, 'content': response.body}
+  const res = {response: response, content: response.body}
   return res
 }
 
@@ -82,9 +114,10 @@ async function processError (e) {
   return {
     contents: body.toString(),
     status: {
-      url, http_code,
-      'content_type': headers['content-type'],
-      'content_length': contentLength
-    }
+      url,
+      http_code,
+      content_type: headers['content-type'],
+      content_length: contentLength,
+    },
   }
 }

app/process-request.js

@@ -1,4 +1,4 @@
-const getPage  = require('./get-page')
+const getPage = require('./get-page')
 
 module.exports = processRequest
 
@@ -17,7 +17,7 @@ function parseParams (req) {
   const params = {
     requestMethod: req.method,
     ...req.query,
-    ...req.params
+    ...req.params,
   }
   params.requestMethod = parseRequestMethod(params.requestMethod)
   params.format = (params.format || 'json').toLowerCase()
@@ -40,11 +40,12 @@ function createResponse (page, params, res, startTime) {
     return res.send(page.content)
   }
 
-  if (params.charset) res.set('Content-Type', `application/json; charset=${params.charset}`)
-  else res.set('Content-Type', 'application/json')
+  if (params.charset) {
+res.set('Content-Type', `application/json; charset=${params.charset}`)
+} else res.set('Content-Type', 'application/json')
 
-  if (page.status) page.status.response_time = (new Date() - startTime)
-  else page.response_time = (new Date() - startTime)
+  if (page.status) page.status.response_time = new Date() - startTime
+  else page.response_time = new Date() - startTime
 
   if (params.callback) return res.jsonp(page)
   return res.send(JSON.stringify(page))

package.json

@@ -12,7 +12,8 @@
   },
   "dependencies": {
     "express": "^4.17.1",
-    "got": "^9.6.0"
+    "got": "^9.6.0",
+    "redis": "^3.0.2"
   },
   "devDependencies": {
     "eslint": "^6.8.0",

redis.js

@@ -0,0 +1,16 @@
+const redis = require("redis");
+const redisUrl = "redis://127.0.0.1:6379";
+const client = redis.createClient(redisUrl);
+const util = require("util");
+client.get = util.promisify(client.get);
+
+exports.redisGet = async (key) => {
+  return client.get(key);
+};
+
+exports.redisSet = (key, val) => {
+  client.set(key, val, "EX", 60 * 60 * 24);
+};
+exports.closeInstance = () => {
+  client.quit(() => {});
+};