From a70415cbf975cff110ac52e21740ba7a6f4f9033 Mon Sep 17 00:00:00 2001 From: anish-work Date: Wed, 13 Nov 2024 19:52:00 +0530 Subject: [PATCH] use more fake agents and add https --- src/proxyConfig.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/proxyConfig.js b/src/proxyConfig.js index 12b321c..1f58dfb 100644 --- a/src/proxyConfig.js +++ b/src/proxyConfig.js @@ -1,14 +1,21 @@ // proxyConfig.js const path = require("path"); const fs = require("fs"); -const axios = require("axios"); const https = require("https"); +const got = require("got"); // Fake user agents array - you can expand this list const FAKE_USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0", + // chrome + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36", + // edge + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19582", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19577", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17720", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.8810.3391 Safari/537.36 Edge/18.14383", ]; // Environment variables (you'll need to set these in your .env file) @@ -24,7 +31,7 @@ const config = { function getProxyUrl(scheme) { if (!config.SCRAPING_PROXY_HOST) return ""; - return `http://${config.SCRAPING_PROXY_USERNAME}:${config.SCRAPING_PROXY_PASSWORD}@${config.SCRAPING_PROXY_HOST}`; + return `${scheme}://${config.SCRAPING_PROXY_USERNAME}:${config.SCRAPING_PROXY_PASSWORD}@${config.SCRAPING_PROXY_HOST}`; } // Get proxy configuration @@ -45,7 +52,7 @@ async function getScrapingProxyCertPath() { if (!fs.existsSync(certPath)) { console.log(`Downloading proxy cert to ${certPath}`); - const response = await axios.get(config.SCRAPING_PROXY_CERT_URL, { + const response = await got(config.SCRAPING_PROXY_CERT_URL, { responseType: "arraybuffer", }); fs.writeFileSync(certPath, response.data);