use more fake agents and add https

GooeyAI · Nov 13, 2024 · a70415c · a70415c
1 parent 70af2d4
commit a70415c
Showing 1 changed file with 13 additions and 6 deletions.
diff --git a/src/proxyConfig.js b/src/proxyConfig.js
@@ -1,14 +1,21 @@
 // proxyConfig.js
 const path = require("path");
 const fs = require("fs");
-const axios = require("axios");
 const https = require("https");
+const got = require("got");
 
 // Fake user agents array - you can expand this list
 const FAKE_USER_AGENTS = [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
+ // chrome
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36",
+ "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36",
+ // edge
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19582",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19577",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17720",
+ "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.8810.3391 Safari/537.36 Edge/18.14383",
 ];
 
 // Environment variables (you'll need to set these in your .env file)
@@ -24,7 +31,7 @@ const config = {
 function getProxyUrl(scheme) {
  if (!config.SCRAPING_PROXY_HOST) return "";
 
- return `http://${config.SCRAPING_PROXY_USERNAME}:${config.SCRAPING_PROXY_PASSWORD}@${config.SCRAPING_PROXY_HOST}`;
+ return `${scheme}://${config.SCRAPING_PROXY_USERNAME}:${config.SCRAPING_PROXY_PASSWORD}@${config.SCRAPING_PROXY_HOST}`;
 }
 
 // Get proxy configuration
@@ -45,7 +52,7 @@ async function getScrapingProxyCertPath() {
 
  if (!fs.existsSync(certPath)) {
   console.log(`Downloading proxy cert to ${certPath}`);
-  const response = await axios.get(config.SCRAPING_PROXY_CERT_URL, {
+  const response = await got(config.SCRAPING_PROXY_CERT_URL, {
    responseType: "arraybuffer",
   });
   fs.writeFileSync(certPath, response.data);