fix: fetch Medium blogs directly from Medium feed and replace the usa…

…ge of `rss2json` API with `feed2json` package to fix errors Signed-off-by: Amr Elsayyad <[email protected]>
AmrElsayyad · Aug 23, 2024 · 37cb01e · 37cb01e
1 parent 5bc73ab
commit 37cb01e
Show file tree

Hide file tree

Showing 5 changed files with 2,440 additions and 1,399 deletions.
diff --git a/bun.lockb b/bun.lockb
diff --git a/fetch.js b/fetch.js
@@ -1,3 +1,4 @@
+const feed2json = require("feed2json");
 const fs = require("fs");
 const https = require("https");
 const process = require("process");
@@ -17,51 +18,6 @@ const ERR = {
     "The request to Medium didn't succeed. Check if Medium username in your .env file is correct."
 };
 
-function fetchMediumData(retries = 3) {
-  if (MEDIUM_USERNAME !== undefined) {
-    console.log(`Fetching Medium blogs data for ${MEDIUM_USERNAME}`);
-    const options = {
-      hostname: "api.rss2json.com",
-      path: `/v1/api.json?rss_url=https://medium.com/feed/@${MEDIUM_USERNAME}`,
-      port: 443,
-      method: "GET"
-    };
-
-    const req = https.request(options, res => {
-      let mediumData = "";
-
-      console.log(`statusCode: ${res.statusCode}`);
-      if (res.statusCode !== 200) {
-        res.on("data", d => {
-          console.error(`Error response: ${d}`);
-        });
-        if (retries > 0) {
-          console.log(`Retrying... (${retries} attempts left)`);
-          return fetchMediumData(retries - 1);
-        } else {
-          throw new Error(ERR.requestFailedMedium);
-        }
-      }
-
-      res.on("data", d => {
-        mediumData += d;
-      });
-      res.on("end", () => {
-        fs.writeFile("./public/blogs.json", mediumData, function (err) {
-          if (err) return console.log(err);
-          console.log("saved file to public/blogs.json");
-        });
-      });
-    });
-
-    req.on("error", error => {
-      throw error;
-    });
-
-    req.end();
-  }
-}
-
 if (USE_GITHUB_DATA === "true") {
   if (GITHUB_USERNAME === undefined) {
     throw new Error(ERR.noUserName);
@@ -140,4 +96,39 @@ if (USE_GITHUB_DATA === "true") {
   req.end();
 }
 
-fetchMediumData();
+if (MEDIUM_USERNAME !== undefined) {
+  console.log(`Fetching Medium blogs data for ${MEDIUM_USERNAME}`);
+  const url = `https://medium.com/feed/@${MEDIUM_USERNAME}`;
+  const options = {
+    hostname: "medium.com",
+    path: `/feed/@${MEDIUM_USERNAME}`,
+    port: 443,
+    method: "GET"
+  };
+
+  const req = https.request(options, res => {
+    console.log(`statusCode: ${res.statusCode}`);
+    if (res.statusCode !== 200) {
+      throw new Error(ERR.requestMediumFailed);
+    }
+
+    feed2json.fromStream(res, url, {}, (err, json) => {
+      if (err) {
+        console.error("Error converting feed to JSON:", err);
+        return;
+      }
+
+      const mediumData = JSON.stringify(json, null, 2);
+      fs.writeFile("./public/blogs.json", mediumData, function (err) {
+        if (err) return console.error(err);
+        console.log("Saved file to public/blogs.json");
+      });
+    });
+  });
+
+  req.on("error", error => {
+    throw error;
+  });
+
+  req.end();
+}