New Action in Scrapy-puppeteer-service (RecaptchaSolver) added (#31)

* Added new action (recaptcha_solver)

* Fixed dependencies and removed extra code

* Implemented RecaptchaSolver, added comments and description. Changed 2captcha token name. Started to think about fingerprints in browser.

* Implemented RecaptchaSolver, added comments and description. Changed 2captcha token name. Added StealthPlugin

* HotFix: fixed token_2captcha environment variable name

* Deleted extra properties, handled "no tekoen_2captcha" error (scapy-puppeteer)

Author: MatthewZMSU

app.js

@@ -1,5 +1,8 @@
 const express = require('express');
-const puppeteer = require('puppeteer');
+const puppeteer = require('puppeteer-extra')
+
+const RecaptchaPlugin = require('puppeteer-extra-plugin-recaptcha')
+const StealthPlugin = require('puppeteer-extra-plugin-stealth')
 const cookieParser = require('cookie-parser');
 const logger = require('morgan');
 const bodyParser = require('body-parser');
@@ -13,6 +16,7 @@ const clickRouter = require('./routes/click');
 const actionRouter = require('./routes/action');
 const scrollRouter = require('./routes/scroll');
 const screenshotRouter = require('./routes/screenshot');
+const recaptchaSolverRouter = require('./routes/recaptcha_solver')
 const mhtmlRouter = require('./routes/mhtml');
 const harRouter = require('./routes/har');
 const closeContextRouter = require('./routes/close_context');
@@ -23,8 +27,35 @@ const HEADLESS = (process.env.HEADLESS || "true").toLowerCase() === "true";
 const CONNECT_TIMEOUT = parseInt(process.env.CONNECT_TIMEOUT) || 180000;
 const VIEWPORT_WIDTH = parseInt(process.env.VIEWPORT_WIDTH) || 1280;
 const VIEWPORT_HEIGHT = parseInt(process.env.VIEWPORT_HEIGHT) || 720;
+const TOKEN_2CAPTCHA = process.env.TOKEN_2CAPTCHA;
+const STEALTH_BROWSING = (process.env.STEALTH_BROWSING || "true").toLowerCase() === "true";
 
 async function setupBrowser() {
+    try {
+        if (TOKEN_2CAPTCHA) {  // If token is given then RecapcthaPlugin is activated
+            puppeteer.use(
+                RecaptchaPlugin({
+                    provider: {
+                        id: '2captcha',
+                        token: TOKEN_2CAPTCHA
+                    }
+                })
+            )
+        }
+    } catch (error) {
+        console.error('Failed to proceed 2captcha token:', error);
+        process.exit(1);
+    }
+
+    try {
+        if (STEALTH_BROWSING) {  // Activate or not StealthPlugin
+            puppeteer.use(StealthPlugin());
+        }
+    } catch (error) {
+        console.error('Failed to enable StealthPlugin:', error);
+        process.exit(1);
+    }
+
     try {
         //TODO add more params for puppeteer launch
         const browser = await puppeteer.launch(
@@ -36,7 +67,6 @@ async function setupBrowser() {
         browser.on('disconnected', setupBrowser);
         app.set('browser', browser);
     } catch (error) {
-        console.error('Failed to start browser:', error);
         process.exit(1);
     }
 }
@@ -60,6 +90,7 @@ app.use('/click', clickRouter);
 app.use('/action', actionRouter);
 app.use('/scroll', scrollRouter);
 app.use('/screenshot', screenshotRouter);
+app.use('/recaptcha_solver', recaptchaSolverRouter);
 app.use('/mhtml', mhtmlRouter);
 app.use('/har', harRouter);
 app.use('/close_context', closeContextRouter);

helpers/utils.js

@@ -128,6 +128,7 @@ exports.getBrowserPage = async function getBrowserPage(browser, request) {
         return newPage(context);
     }
     const { origin: proxyServer, username, password } = new URL(proxy);
+
     const context = await browser.createIncognitoBrowserContext({ proxyServer });
     context[PROXY_URL_KEY] = proxy;
     const page = await newPage(context);

package.json

@@ -11,9 +11,13 @@
     "cookie-parser": "~1.4.6",
     "debug": "~4.3.4",
     "express": "~4.17.3",
+    "fingerprint-injector": "^2.1.30",
     "morgan": "~1.10.0",
     "npm-run-all": "^4.1.5",
     "puppeteer": "^20.1.2",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-recaptcha": "^3.6.8",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
     "puppeteer-proxy": "^2.1.2"
   }
 }

routes/recaptcha_solver.js

@@ -0,0 +1,74 @@
+const express = require('express')
+const router = express.Router()
+const utils = require('../helpers/utils')
+
+const DEFAULT_TIMEOUT = 1000;  // 1 second
+
+/*
+ * This module introduces new ability to puppeteer-service.
+ * It is capable of solving recaptchas on the given web-page.
+ * If there is no recaptcha on the page nothing bad will happen.
+ * If there is recaptcha it solves it and then inserts the special code
+ * into the page automatically.
+ *
+ * Returns useful information about recaptcha_solving.
+ * For more information about return value visit
+ * https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-recaptcha#result-object
+ */
+
+/**
+ *
+ * @param page - page with possible recaptcha.
+ * @param request - request to the page.
+ */
+
+async function action(page, request) {
+
+    let recaptcha_data;
+
+    if (request.body.solve_recaptcha) {
+        recaptcha_data = await page.solveRecaptchas();
+    }
+    else {
+        recaptcha_data = await page.findRecaptchas();
+    }
+
+    const waitOptions = request.body.waitOptions || { timeout: DEFAULT_TIMEOUT };
+    return {
+        ...await utils.formResponse(page, request.query.closePage, waitOptions),
+        recaptcha_data: recaptcha_data,
+    }
+}
+
+router.post('/', async function (req, res, next) {
+    if (!req.query.contextId || !req.query.pageId) {
+        res.status(400);
+        res.send("No page in request");
+        next();
+        return;
+    }
+
+    if (!process.env.TOKEN_2CAPTCHA) {
+        res.status("501");
+        res.send("TOKEN_2CAPTCHA is not provided!");
+        next();
+        return;
+    }
+
+    if (!("solve_recaptcha" in req.body)) {
+        res.status("400");
+        res.send("No solve_recaptcha parameter in request");
+        next();
+        return;
+    }
+
+    try {
+        let response = await utils.performAction(req, action);
+        res.header('scrapy-puppeteer-service-context-id', response.contextId);
+        res.send(response)
+    } catch (e) {
+        next(e);
+    }
+});
+
+module.exports = router;