Update to latest Puppeteer (#26)

* update to pptr 19.11.1

* allow to explicitly specify selector, xpath or timeout to wait for in wait options

* use pptr 20.1.2

Author: mxsnq

README.md

@@ -56,8 +56,32 @@ This method allow to goto a page with a specific url in puppeteer.
 Params: 
 
 url - the url which puppeteer should navigate to.      
-navigationOptions - [possible options to use for request.](https://github.com/GoogleChrome/puppeteer/blob/v1.20.0/docs/api.md#pagegotourl-options)      
-waitOptions - [wait for selector or timeout](https://github.com/puppeteer/puppeteer/blob/v1.20.0/docs/api.md#pagewaitforselectororfunctionortimeout-options-args) after navigation completes, same as in click or scroll.
+navigationOptions - [possible options to use for request.](https://pptr.dev/api/puppeteer.page.goto#remarks)      
+waitOptions - [wait for selector](https://pptr.dev/api/puppeteer.page.waitforselector), [xpath](https://pptr.dev/api/puppeteer.page.waitforxpath), or timeout after navigation completes.
+
+Example request body
+```json5
+{
+   "url": "https://example.com", // <string> URL to navigate page to. The url should include scheme, e.g. https://.
+   "navigationOptions": { // Navigation parameters which might have the following properties:
+       "timeout": 30000, // <number> Maximum navigation time in milliseconds, defaults to 30 seconds, pass 0 to disable timeout.
+       // "waitUntil": <string|Array<string>> When to consider navigation succeeded, defaults to load. Given an array of event strings, navigation is considered to be successful after all events have been fired. Events can be either:
+         // load - consider navigation to be finished when the load event is fired.
+         // domcontentloaded - consider navigation to be finished when the DOMContentLoaded event is fired.
+         // networkidle0 - consider navigation to be finished when there are no more than 0 network connections for at least 500 ms.
+         // networkidle2 - consider navigation to be finished when there are no more than 2 network connections for at least 500 ms.
+       // "referer": <string> Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders().
+   },
+   "waitOptions": { // Wait for element or timeout after navigation completes
+       // "timeout": <number> Wait for given timeout in milliseconds
+       "selector": "span.target", // <string> Wait for element by selector (see https://pptr.dev/api/puppeteer.page.waitforselector)
+       // "xpath": <string> Wait for element by xpath (see https://pptr.dev/api/puppeteer.page.waitforxpath)
+       "options": { // <object> Options to wait for elements (see https://pptr.dev/api/puppeteer.waitforselectoroptions)
+           "timeout": 10000
+       } 
+   }
+}
+```
 
 ### **/back** and **/forward**
 This methods helps to navigate back and forward to see previously seen pages.
@@ -77,9 +101,8 @@ Example request body:
         "delay": 0 //<number> Time to wait between mousedown and mouseup in milliseconds. Defaults to 0.
     },
     "waitOptions": {
-        // if selectorOrTimeout is a string, then the first argument is treated as a selector or xpath, depending on whether or not it starts with '//', and the method is a shortcut for page.waitForSelector or page.waitForXPath
-        // if selectorOrTimeout is a number, then the first argument is treated as a timeout in milliseconds and the method returns a promise which resolves after the timeout
-        "selectorOrTimeout": 5, //default timeout is 1000ms
+        // selector, xpath or timeout, same as in the goto method
+        "timeout": 5000, //default timeout is 1000ms
     },
     "navigationOptions": { // use if click triggers navigation to other page; same as in goXXX methods
         "waitUntil": "domcontentloaded",    
@@ -96,9 +119,8 @@ Example request body:
 {
     "selector": "", //<string> A selector to search for element to click. If there are multiple elements satisfying the selector, the first will be clicked.
     "waitOptions": {
-        // if selectorOrTimeout is a string, then the first argument is treated as a selector or xpath, depending on whether or not it starts with '//', and the method is a shortcut for page.waitForSelector or page.waitForXPath
-        // if selectorOrTimeout is a number, then the first argument is treated as a timeout in milliseconds and the method returns a promise which resolves after the timeout
-        "selectorOrTimeout": 5, //default timeout is 1000ms
+      // selector, xpath or timeout, same as in the goto method
+      "timeout": 5000, //default timeout is 1000ms
     }
  }
 ```
@@ -116,8 +138,8 @@ Simple example request body of goto:
 async function action(page, request) {
     await page.goto(request.query.uri);
     let response = { //return response that you want to see as result
-        context_id: page.browserContext()._id,
-        page_id: await page._target._targetId,
+        context_id: page.browserContext().id,
+        page_id: page.target()._targetId,
         html: await page.content(),
         cookies: await page.cookies()
     };

helpers/utils.js

@@ -2,18 +2,17 @@ const { proxyRequest } = require('puppeteer-proxy');
 const PROXY_URL_KEY = 'puppeteer-service-proxy-url'
 
 async function findContextInBrowser(browser, contextId) {
-
-    for (let context of browser.browserContexts()) {
-        if (contextId === await context._id) {
+    for (const context of browser.browserContexts()) {
+        if (contextId === context.id) {
             return context;
         }
     }
     throw "Context not found";
 }
 
 async function findPageInContext(context, pageId) {
-    for (let page of await context.pages()) {
-        if (pageId === await page._target._targetId) {
+    for (const page of await context.pages()) {
+        if (pageId === page.target()._targetId) {
             return page;
         }
     }
@@ -22,31 +21,54 @@ async function findPageInContext(context, pageId) {
 
 exports.closeContexts = async function closeContexts(browser, contextIds) {
     // TODO shared locks on contexts and exclusive on pages?
-    let close_promises = [];
-    for (let context of browser.browserContexts()) {
-        if (contextIds.includes(context._id)) {
-            close_promises.push(context.close());
+    const closePromises = [];
+    for (const context of browser.browserContexts()) {
+        if (contextIds.includes(context.id)) {
+            closePromises.push(context.close());
         }
     }
-    await Promise.all(close_promises);
+    await Promise.all(closePromises);
 };
 
 async function wait(page, waitFor) {
-    if (waitFor instanceof Object) {
-        const { selectorOrTimeout, options } = waitFor;
-        if (selectorOrTimeout) {
-            await page.waitFor(selectorOrTimeout, options);
+    let { selector, xpath, timeout, options } = waitFor;
+
+    // for compatibility with old waitFor interface
+    const { selectorOrTimeout } = waitFor;
+    if (selectorOrTimeout) {
+        if (!isNaN(selectorOrTimeout)) {
+            timeout = selectorOrTimeout;
+        } else if (typeof selectorOrTimeout === 'string') {
+            if (selectorOrTimeout.startsWith('//')) {
+                xpath = selectorOrTimeout;
+            } else {
+                selector = selectorOrTimeout;
+            }
         }
-    } else if (waitFor) {
-        await page.waitFor(waitFor);
+    }
+
+    if ([selector, xpath, timeout].filter(Boolean).length > 1) {
+        throw "Wait options must contain either a selector, an xpath or a timeout";
+    }
+
+    if (selector) {
+        return page.waitForSelector(selector, options);
+    }
+    if (xpath) {
+        return page.waitForXPath(xpath, options);
+    }
+    if (timeout) {
+        return new Promise(resolve => setTimeout(resolve, timeout));
     }
 }
 
 exports.formResponse = async function formResponse(page, closePage, waitFor) {
-    await wait(page, waitFor);
+    if (waitFor) {
+        await wait(page, waitFor);
+    }
 
-    let response = {
-        contextId: page.browserContext()._id,
+    const response = {
+        contextId: page.browserContext().id,
         html: await page.content(),
         cookies: await page.cookies(),
     };
@@ -56,7 +78,7 @@ exports.formResponse = async function formResponse(page, closePage, waitFor) {
     }
 
     if (!page.isClosed()) {
-        response.pageId = await page._target._targetId;
+        response.pageId = page.target()._targetId;
     }
 
     return response;
@@ -118,7 +140,7 @@ exports.getBrowserPage = async function getBrowserPage(browser, request) {
 exports.performAction = async function performAction(request, action) {
     const lock = request.app.get('lock');
     const page = await exports.getBrowserPage(request.app.get('browser'), request);
-    return lock.acquire(await page._target._targetId, async () => {
+    return lock.acquire(page.target()._targetId, async () => {
         let extraHeaders = {};
 
         if ('body' in request && 'headers' in request.body) {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "scrapy-puppeteer-service",
-  "version": "0.0.10",
+  "version": "0.1.0",
   "private": true,
   "scripts": {
     "start": "node ./bin/www"
@@ -9,11 +9,11 @@
     "async-lock": "^1.3.0",
     "body-parser": "^1.19.1",
     "cookie-parser": "~1.4.6",
-    "debug": "~4.3.3",
+    "debug": "~4.3.4",
     "express": "~4.17.3",
     "morgan": "~1.10.0",
     "npm-run-all": "^4.1.5",
-    "puppeteer": "^13.1.2",
+    "puppeteer": "^20.1.2",
     "puppeteer-proxy": "^2.1.2"
   }
 }

routes/action.js

@@ -10,8 +10,8 @@ const router = express.Router();
  *      some actions with page in puppeteer syntax
  *      ...
  *      return {
- *          context_id: page.browserContext()._id,
- *          page_id: await page._target._targetId,
+ *          context_id: page.browserContext().id,
+ *          page_id: page.target()._targetId,
             html: await page.content(),
             cookies: await page.cookies()
  *      };

routes/click.js

@@ -14,7 +14,8 @@ async function action(page, request) {
     } else {
         await page.click(request.body.selector, request.body.clickOptions);
     }
-    return utils.formResponse(page, request.query.closePage, request.body.waitOptions || DEFAULT_TIMEOUT);
+    const waitOptions = request.body.waitOptions || { timeout: DEFAULT_TIMEOUT };
+    return utils.formResponse(page, request.query.closePage, waitOptions);
 }
 
 /**
@@ -25,11 +26,7 @@ async function action(page, request) {
         "clickCount", //<number> defaults to 1.
         "delay" //<number> Time to wait between mousedown and mouseup in milliseconds. Defaults to 0.
     },
-    "waitOptions": {
-        // if selectorOrTimeout is a string, then the first argument is treated as a selector or xpath, depending on whether or not it starts with '//', and the method is a shortcut for page.waitForSelector or page.waitForXPath
-        // if selectorOrTimeout is a number, then the first argument is treated as a timeout in milliseconds and the method returns a promise which resolves after the timeout
-        "selectorOrTimeout":... default 1,
-    },
+    "waitOptions": {...}, // same as in goto action, defaults to 1s timeout
     "navigationOptions": {...} // same as in goto action
  }
  */

routes/goto.js

@@ -19,7 +19,13 @@ async function action(page, request) {
 //          networkidle2 - consider navigation to be finished when there are no more than 2 network connections for at least 500 ms.
 //     "referer" <string> Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders().
 // },
-// "waitOptions": {...} same as in click action
+// "waitOptions": {
+//     "timeout": <number> Wait for given timeout in milliseconds
+//     "selector": <string> Wait for element by selector (see https://pptr.dev/api/puppeteer.page.waitforselector)
+//     "xpath": <string> Wait for element by xpath (see https://pptr.dev/api/puppeteer.page.waitforxpath)
+//     "options": <object> Options to wait for elements (see https://pptr.dev/api/puppeteer.waitforselectoroptions)
+// }
+// }
 //
 router.post('/', async function (req, res, next) {
 

routes/scroll.js

@@ -13,18 +13,15 @@ async function action(page, request) {
             window.scrollBy(0, document.body.scrollHeight)
         });
     }
-    return utils.formResponse(page, request.query.closePage, request.body.waitOptions || DEFAULT_TIMEOUT);
+    const waitOptions = request.body.waitOptions || { timeout: DEFAULT_TIMEOUT};
+    return utils.formResponse(page, request.query.closePage, waitOptions);
 }
 
 // Method that scrolls page to a certain selector.
 // Example body:
 //  body = {
 //     "selector": "", //<string> A selector to search for element to scroll
-//     "waitOptions": {
-//         // if selectorOrFunctionOrTimeout is a string, then the first argument is treated as a selector or xpath, depending on whether or not it starts with '//', and the method is a shortcut for page.waitForSelector or page.waitForXPath
-//         // if selectorOrFunctionOrTimeout is a number, then the first argument is treated as a timeout in milliseconds and the method returns a promise which resolves after the timeout
-//         "selectorOrTimeout":...,
-//     }
+//     "waitOptions": {...} // same as in goto action, defaults to 1s timeout
 //  }
 router.post('/', async function (req, res, next) {