use https-proxy-agent

GooeyAI · Nov 13, 2024 · a428504 · a428504
1 parent 9850e35
commit a428504
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 15 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -13,7 +13,7 @@
     "dotenv": "^8.6.0",
     "express": "^4.19.2",
     "got": "^11.8.0",
-    "hpagent": "^1.2.0",
+    "https-proxy-agent": "^7.0.5",
     "metascraper": "^5.14.18",
     "metascraper-description": "^5.14.18",
     "metascraper-image": "^5.14.18",

diff --git a/src/proxyConfig.js b/src/proxyConfig.js
@@ -1,6 +1,6 @@
 // proxyConfig.js
 const path = require("path");
-const { HttpsProxyAgent, HttpProxyAgent } = require("hpagent");
+const { HttpsProxyAgent } = require("https-proxy-agent");
 
 // Fake user agents array - you can expand this list
 const FAKE_USER_AGENTS = [
@@ -34,15 +34,15 @@ function getProxyUrl() {
 
 // Main function to get axios config for scraping
 async function getScrapingConfig() {
+ const httpsProxyAgent = new HttpsProxyAgent(getProxyUrl());
  return {
   headers: {
    "User-Agent":
     FAKE_USER_AGENTS[Math.floor(Math.random() * FAKE_USER_AGENTS.length)],
   },
   agent: {
-   http: new HttpProxyAgent({
-    http: getProxyUrl(),
-   }),
+   https: httpsProxyAgent,
+   http: httpsProxyAgent,
   },
  };
 }