From c4ee4cd27a997d38610f7a44e95c3ccea171b475 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 24 May 2024 11:32:04 +0000 Subject: [PATCH 1/7] (deps) bump puppeteer to v22.10.0 --- package-lock.json | 66 +++++++++++++++++++++++------------------------ package.json | 2 +- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index 373ddb19..479ae0fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "decamelize": "^5.0.0", "fast-stats": "0.0.6", "js-yaml": "^4.0.0", - "puppeteer": "^22.7.1" + "puppeteer": "^22.10.0" }, "bin": { "phantomas": "bin/phantomas.js" @@ -3226,9 +3226,9 @@ } }, "node_modules/devtools-protocol": { - "version": "0.0.1273771", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1273771.tgz", - "integrity": "sha512-QDbb27xcTVReQQW/GHJsdQqGKwYBE7re7gxehj467kKP2DKuYBUj6i2k5LRiAC66J1yZG/9gsxooz/s9pcm0Og==" + "version": "0.0.1286932", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1286932.tgz", + "integrity": "sha512-wu58HMQll9voDjR4NlPyoDEw1syfzaBNHymMMZ/QOXiHRNluOnDgu9hp1yHOKYoMlxCh4lSSiugLITe6Fvu1eA==" }, "node_modules/diff-sequences": { "version": "28.1.1", @@ -6811,15 +6811,15 @@ } }, "node_modules/puppeteer": { - "version": "22.7.1", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-22.7.1.tgz", - "integrity": "sha512-JBCBCwQ9+dyPp5haqeecgv0N0vgWFx44woUeKJaPeJT8CU3RXrd8F/tqJQbuAmcWlbMhYJSlTJkIFrwVAs6BNA==", + "version": "22.10.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-22.10.0.tgz", + "integrity": "sha512-ZOkZd6a6t0BdKcWb0wAYHWQqCfdlN1PPnXOmg/XNrbo6gJhYWFX4qCNb6ahSn8TpAqBqLCoD4Q010F7GwOM7mA==", "hasInstallScript": true, "dependencies": { "@puppeteer/browsers": "2.2.3", "cosmiconfig": "9.0.0", - "devtools-protocol": "0.0.1273771", - "puppeteer-core": "22.7.1" + "devtools-protocol": "0.0.1286932", + "puppeteer-core": "22.10.0" }, "bin": { "puppeteer": "lib/esm/puppeteer/node/cli.js" @@ -6829,15 +6829,15 @@ } }, "node_modules/puppeteer-core": { - "version": "22.7.1", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.7.1.tgz", - "integrity": "sha512-jD7T7yN7PWGuJmNT0TAEboA26s0VVnvbgCxqgQIF+eNQW2u71ENaV2JwzSJiCHO+e72H4Ue6AgKD9USQ8xAcOQ==", + "version": "22.10.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.10.0.tgz", + "integrity": "sha512-I54J4Vy4I07UHsgB1QSmuFoF7KNQjJWcvFBPhtY+ezMdBfwgGDr8dzYrJa11aPgP9kxIUHjhktcMmmfJkOAtTw==", "dependencies": { "@puppeteer/browsers": "2.2.3", "chromium-bidi": "0.5.19", "debug": "4.3.4", - "devtools-protocol": "0.0.1273771", - "ws": "8.16.0" + "devtools-protocol": "0.0.1286932", + "ws": "8.17.0" }, "engines": { "node": ">=18" @@ -8075,9 +8075,9 @@ } }, "node_modules/ws": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz", - "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==", + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", + "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", "engines": { "node": ">=10.0.0" }, @@ -10614,9 +10614,9 @@ "dev": true }, "devtools-protocol": { - "version": "0.0.1273771", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1273771.tgz", - "integrity": "sha512-QDbb27xcTVReQQW/GHJsdQqGKwYBE7re7gxehj467kKP2DKuYBUj6i2k5LRiAC66J1yZG/9gsxooz/s9pcm0Og==" + "version": "0.0.1286932", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1286932.tgz", + "integrity": "sha512-wu58HMQll9voDjR4NlPyoDEw1syfzaBNHymMMZ/QOXiHRNluOnDgu9hp1yHOKYoMlxCh4lSSiugLITe6Fvu1eA==" }, "diff-sequences": { "version": "28.1.1", @@ -13263,26 +13263,26 @@ "dev": true }, "puppeteer": { - "version": "22.7.1", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-22.7.1.tgz", - "integrity": "sha512-JBCBCwQ9+dyPp5haqeecgv0N0vgWFx44woUeKJaPeJT8CU3RXrd8F/tqJQbuAmcWlbMhYJSlTJkIFrwVAs6BNA==", + "version": "22.10.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-22.10.0.tgz", + "integrity": "sha512-ZOkZd6a6t0BdKcWb0wAYHWQqCfdlN1PPnXOmg/XNrbo6gJhYWFX4qCNb6ahSn8TpAqBqLCoD4Q010F7GwOM7mA==", "requires": { "@puppeteer/browsers": "2.2.3", "cosmiconfig": "9.0.0", - "devtools-protocol": "0.0.1273771", - "puppeteer-core": "22.7.1" + "devtools-protocol": "0.0.1286932", + "puppeteer-core": "22.10.0" } }, "puppeteer-core": { - "version": "22.7.1", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.7.1.tgz", - "integrity": "sha512-jD7T7yN7PWGuJmNT0TAEboA26s0VVnvbgCxqgQIF+eNQW2u71ENaV2JwzSJiCHO+e72H4Ue6AgKD9USQ8xAcOQ==", + "version": "22.10.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.10.0.tgz", + "integrity": "sha512-I54J4Vy4I07UHsgB1QSmuFoF7KNQjJWcvFBPhtY+ezMdBfwgGDr8dzYrJa11aPgP9kxIUHjhktcMmmfJkOAtTw==", "requires": { "@puppeteer/browsers": "2.2.3", "chromium-bidi": "0.5.19", "debug": "4.3.4", - "devtools-protocol": "0.0.1273771", - "ws": "8.16.0" + "devtools-protocol": "0.0.1286932", + "ws": "8.17.0" } }, "query-string": { @@ -14167,9 +14167,9 @@ } }, "ws": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz", - "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==", + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", + "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", "requires": {} }, "xtend": { diff --git a/package.json b/package.json index 1f704ee7..85d8abc3 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "decamelize": "^5.0.0", "fast-stats": "0.0.6", "js-yaml": "^4.0.0", - "puppeteer": "^22.7.1" + "puppeteer": "^22.10.0" }, "devDependencies": { "@jest/globals": "^28.0.0", From 11e2048bc4683cacda154749b71c0c2b9b08a6b9 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 24 May 2024 12:55:53 +0100 Subject: [PATCH 2/7] Browser: make sure the meta.response is set --- lib/browser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/browser.js b/lib/browser.js index 313b88aa..9a11fff5 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -179,7 +179,7 @@ Browser.prototype.init = async (phantomasOptions) => { var meta = responses[data.requestId]; /* istanbul ignore if */ - if (typeof meta === "undefined") { + if (typeof meta?.response === "undefined") { // the browser sometimes looses trace of a request, let's ignore. networkDebug( "Can't find request id %d in previous requests", From 0d6b8687825bd66cb64559f5f39b36464df61ace Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 24 May 2024 20:54:42 +0100 Subject: [PATCH 3/7] phantomas: some more debugging around scope.injectJs --- lib/index.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/index.js b/lib/index.js index 11af35cf..5c674f79 100644 --- a/lib/index.js +++ b/lib/index.js @@ -89,7 +89,7 @@ function phantomas(url, opts) { return page.$$(selector); }, - // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageevaluateonnewdocumentpagefunction-args + // @see https://pptr.dev/api/puppeteer.page.evaluateonnewdocument injectJs: async (script) => { const debug = require("debug")("phantomas:injectJs"); @@ -98,7 +98,9 @@ function phantomas(url, opts) { suffix = "}"; const preloadFile = - prefix + require("fs").readFileSync(script, "utf8") + suffix; + prefix + + (await require("fs/promises").readFile(script, "utf8")) + + suffix; await page.evaluateOnNewDocument(preloadFile); @@ -117,8 +119,9 @@ function phantomas(url, opts) { // @see https://github.com/GoogleChrome/puppeteer/blob/v1.11.0/docs/api.md#pageexposefunctionname-puppeteerfunction await page.exposeFunction("__phantomas_emit", scope.emit); - // Inject helper code into the browser's scope - events.on("init", () => { + // Inject helper code into the browser's scope (but only once!) + events.once("init", () => { + debug("onInit: injecting the core/scope.js ..."); scope.injectJs(__dirname + "/../core/scope.js"); }); From 463a9f0058b1048e2e9594ae4de9cb5b1e3bd851 Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 24 May 2024 20:55:05 +0100 Subject: [PATCH 4/7] core/scope.js: keep the comment up to date --- core/scope.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/scope.js b/core/scope.js index 74e84161..f146c776 100644 --- a/core/scope.js +++ b/core/scope.js @@ -1,7 +1,7 @@ /** * phantomas browser "scope" with helper code * - * Code below is executed in page's "scope" (injected by lib/browser.js) + * Code below is executed in page's "scope" (injected by the scope.injectJs() helper) */ /* istanbul ignore next */ (function coreScope(scope) { From 793dba5b2cf6d1cda468478ae2e590c94653fcef Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 24 May 2024 21:15:08 +0100 Subject: [PATCH 5/7] lib/index.js: document the injectJs --- lib/index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/index.js b/lib/index.js index 5c674f79..de3dc855 100644 --- a/lib/index.js +++ b/lib/index.js @@ -90,6 +90,9 @@ function phantomas(url, opts) { }, // @see https://pptr.dev/api/puppeteer.page.evaluateonnewdocument + // Adds a function which would be invoked in when: + // - the child frame is attached or navigated + // - the page is navigated injectJs: async (script) => { const debug = require("debug")("phantomas:injectJs"); From c2578e3469b1827fa3eba9b955c5e179687fddaa Mon Sep 17 00:00:00 2001 From: macbre Date: Fri, 24 May 2024 21:16:04 +0100 Subject: [PATCH 6/7] modules/domComplexity/scope.js: query for iframes just once --- modules/domComplexity/scope.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/domComplexity/scope.js b/modules/domComplexity/scope.js index 8eba9e9a..ace830b2 100644 --- a/modules/domComplexity/scope.js +++ b/modules/domComplexity/scope.js @@ -92,13 +92,17 @@ })(); // count