Cookies improvements (#9)

* don't set cookies as extra headers * fix typo * use puppeteer-proxy (based on got) instead of node-fetch * minor improvements * version++
ispras · Nov 24, 2020 · 34b66d8 · 34b66d8
1 parent 20c1ecd
commit 34b66d8
Show file tree

Hide file tree

Showing 12 changed files with 472 additions and 480 deletions.
diff --git a/helpers/fetcher.js b/helpers/fetcher.js
diff --git a/helpers/utils.js b/helpers/utils.js
@@ -1,4 +1,5 @@
-const fetch = require('./fetcher');
+const { proxyRequest } = require('puppeteer-proxy');
+const PAGE_PROXY_URL_KEY = 'puppeteer-service-proxy-url'
 
 async function findContextInBrowser(browser, contextId) {
 
@@ -67,37 +68,12 @@ async function newPage(context) {
     await page.setRequestInterception(true);
 
     // This is request interception in order to make request through proxies
-    page.on('request', async interceptedRequest => {
-        const schemaType = new URL(interceptedRequest.url()).protocol;
-
-        if ('puppeteer-service-proxy-url' in interceptedRequest.headers() && ['http:', 'https:'].indexOf(schemaType) !== -1) {
-            const options = {
-                method: interceptedRequest.method(),
-                headers: interceptedRequest.headers(),
-                body: interceptedRequest.postData(),
-            };
-
-            let proxy = options.headers['puppeteer-service-proxy-url'];
-            delete options.headers['puppeteer-service-proxy-url'];
-
-            fetch(interceptedRequest.url(), options, proxy)
-                .then(async (response) => {
-                    interceptedRequest.respond({
-                        status: response.statusCode,
-                        contentType: response.headers['content-type'],
-                        headers: response.headers,
-                        body: response.body,
-                    });
-                })
-                .catch((err) => {
-                    interceptedRequest.respond({
-                        status: 404,
-                        body: err.stack,
-                    });
-                });
-
+    page.on('request', async request => {
+        const { [PAGE_PROXY_URL_KEY]: proxyUrl } = page;
+        if (proxyUrl) {
+            proxyRequest({ page, proxyUrl, request });
         } else {
-            interceptedRequest.continue();
+            request.continue();
         }
     });
 
@@ -126,23 +102,35 @@ exports.getBrowserPage = async function getBrowserPage(browser, contextId, pageI
     }
 };
 
-exports.perfomAction = async function perfomAction(request, action) {
-    let lock = request.app.get('lock');
-    let page = await exports.getBrowserPage(request.app.get('browser'), request.query.contextId, request.query.pageId);
+exports.performAction = async function performAction(request, action) {
+    const { contextId, pageId } = request.query;
+    const lock = request.app.get('lock');
+    const page = await exports.getBrowserPage(request.app.get('browser'), contextId, pageId);
     return lock.acquire(await page._target._targetId, async () => {
-
-        let extra_headers = {};
+        let extraHeaders = {};
 
         if ('body' in request && 'headers' in request.body) {
-            extra_headers = { ...request.body.headers };
+            extraHeaders = { ...request.body.headers };
         }
 
         if ('body' in request && 'proxy' in request.body) {
-            extra_headers['puppeteer-service-proxy-url'] = request.body.proxy
+            // TODO maybe we should map page ids to proxies instead
+            page[PAGE_PROXY_URL_KEY] = request.body.proxy;
+        }
+
+        if ('cookie' in extraHeaders) {
+            // TODO set cookies from request body like headers
+            const url = request.body.url || page.url()
+            const cookies = extraHeaders.cookie.split(';').map(s => {
+                const [name, value] = s.trim().split(/=(.*)/, 2);
+                return { name, value, url };
+            });
+            delete extraHeaders.cookie;
+            await page.setCookie(...cookies);
         }
 
-        if (Object.keys(extra_headers).length !== 0) {
-            await page.setExtraHTTPHeaders(extra_headers);
+        if (Object.keys(extraHeaders).length !== 0) {
+            await page.setExtraHTTPHeaders(extraHeaders);
         }
 
         return await action(page, request);

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "scrapy-puppeteer-service",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "private": true,
   "scripts": {
     "start": "node ./bin/www"
@@ -12,9 +12,8 @@
     "debug": "~2.6.9",
     "express": "~4.16.1",
     "morgan": "~1.9.1",
-    "node-fetch": "^2.6.1",
     "npm-run-all": "^4.1.5",
-    "proxy-agent": "^3.1.1",
-    "puppeteer": "^1.19.0"
+    "puppeteer": "^3.1.0",
+    "puppeteer-proxy": "^2.1.1"
   }
 }
diff --git a/routes/action.js b/routes/action.js
@@ -36,7 +36,7 @@ router.post('/', async function (req, res, next) {
             throw new Error("Invalid action function");
         }
 
-        let response = await utils.perfomAction(req, action);
+        let response = await utils.performAction(req, action);
         res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response);
     } catch (e) {

diff --git a/routes/click.js b/routes/click.js
@@ -43,7 +43,7 @@ router.post('/', async function (req, res, next) {
     }
 
     try {
-        let response = await utils.perfomAction(req, action);
+        let response = await utils.performAction(req, action);
         res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response)
     } catch (e) {

diff --git a/routes/goback.js b/routes/goback.js
@@ -18,7 +18,7 @@ router.post('/', async function (req, res, next) {
     }
 
     try {
-        let response = await utils.perfomAction(req, action);
+        let response = await utils.performAction(req, action);
         res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response)
     } catch (e) {

diff --git a/routes/goforward.js b/routes/goforward.js
@@ -18,7 +18,7 @@ router.post('/', async function (req, res, next) {
     }
 
     try {
-        let response = await utils.perfomAction(req, action);
+        let response = await utils.performAction(req, action);
         res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response)
     } catch (e) {

diff --git a/routes/goto.js b/routes/goto.js
@@ -31,7 +31,7 @@ router.post('/', async function (req, res, next) {
     }
 
     try {
-        let response = await utils.perfomAction(req, action);
+        let response = await utils.performAction(req, action);
         res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response);
     } catch (e) {

diff --git a/routes/har.js b/routes/har.js
@@ -14,7 +14,7 @@ router.post('/', async function (req, res, next) {
     }
 
     try {
-        let response = await utils.perfomAction(req, action);
+        let response = await utils.performAction(req, action);
         res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response);
     } catch (e) {

diff --git a/routes/screenshot.js b/routes/screenshot.js
@@ -18,7 +18,7 @@ async function action(page, request) {
 router.post('/', async function (req, res, next) {
 
     try {
-        let response = await utils.perfomAction(req, action);
+        let response = await utils.performAction(req, action);
         res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response);
     } catch (e) {

diff --git a/routes/scroll.js b/routes/scroll.js
@@ -29,7 +29,7 @@ async function action(page, request) {
 router.post('/', async function (req, res, next) {
 
     try {
-        let response = await utils.perfomAction(req, action);
+        let response = await utils.performAction(req, action);
         res.header('scrapy-puppeteer-service-context-id', response.contextId);
         res.send(response);
     } catch (e) {