Make scraper use node-fetch in Node

README.md

@@ -42,6 +42,7 @@ For the Deno version the following packages are used:
 For the Node version the following packages are used:
 - [fast-xml-parser](https://www.npmjs.com/package/fast-xml-parser) (for parsing XML) ([MIT License](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/LICENSE))
 - [evt](https://www.npmjs.com/package/evt) (for event handling in Scraper) ([MIT License](https://github.com/garronej/evt/blob/main/LICENSE))
+- [node-fetch](https://www.npmjs.com/package/node-fetch) (for fetching data) ([MIT License](https://github.com/node-fetch/node-fetch/blob/main/LICENSE.md))
 
 ## API
 

scripts/update-node-deps.ts

@@ -49,25 +49,42 @@ async function update_npm_package(packageName: string, fileContent: string): Pro
 
 async function update_npm_packages(): Promise<void> {
   const pastebinPath = `${getDirPath()}/../src/node/Pastebin.ts`;
-
   const pastebinFile = await Deno.readTextFile(pastebinPath);
 
   try {
     // update node-fetch and fast-xml-parser
-    const updatedFile = await update_npm_package("node-fetch", pastebinFile);
-    const updatedFile2 = await update_npm_package("fast-xml-parser", updatedFile);
+    const updatedPastebinFile = await update_npm_package("node-fetch", pastebinFile).then((f) =>
+      update_npm_package("fast-xml-parser", f)
+    );
 
-    if (pastebinFile === updatedFile2) {
-      console.log("No changes to npm packages needed.");
-      return;
+    if (pastebinFile === updatedPastebinFile) {
+      console.log("No changes to npm packages needed in Pastebin.ts");
+    } else {
+      console.log("Updating npm packages in Scraper.ts");
+      await Deno.writeTextFile(pastebinPath, updatedPastebinFile);
     }
-
-    console.log("Updating npm packages in Pastebin.ts");
-    await Deno.writeTextFile(pastebinPath, updatedFile2);
   } catch (error) {
     console.error("Error updating version in Pastebin.ts");
     console.error(error);
   }
+
+  const scraperPath = `${getDirPath()}/../src/node/Scraper.ts`;
+  const scraperFile = await Deno.readTextFile(scraperPath);
+
+  try {
+    // update node-fetch
+    const updatedScraperFile = await update_npm_package("node-fetch", scraperFile);
+
+    if (scraperFile === updatedScraperFile) {
+      console.log("No changes to npm packages needed in Scraper.ts");
+    } else {
+      console.log("Updating npm packages in Scraper.ts");
+      await Deno.writeTextFile(scraperPath, updatedScraperFile);
+    }
+  } catch (error) {
+    console.error("Error updating version in Scraper.ts");
+    console.error(error);
+  }
 }
 
 async function update(_args: string[]): Promise<void> {

src/deno/Pastebin.ts

@@ -11,9 +11,12 @@ import {
 
 export class Pastebin extends AbstractPastebin {
   constructor(config?: IPastebinOptions | string | null) {
-    super(config);
-    // This is probably not the best way to do this, but it works
-    super.parseXML = this.parseXml;
+    super(config, {
+      parseXML: (xml: string): Record<string, string> => {
+        const data = parse(xml) as unknown as Record<string, string>;
+        return data;
+      },
+    });
   }
 
   /**
@@ -52,9 +55,4 @@ export class Pastebin extends AbstractPastebin {
 
     return this.createPaste(pasteOpts);
   }
-
-  parseXml = (xml: string): Record<string, string> => {
-    const data = parse(xml) as unknown as Record<string, string>;
-    return data;
-  };
 }

src/lib/Pastebin.ts

@@ -29,7 +29,8 @@ export abstract class AbstractPastebin {
   #config: IPastebinOptions;
 
   // We're able to overwrite fetch because it is an abstract class
-  fetch = globalThis.fetch;
+  #fetch = globalThis.fetch;
+  #parseXML: (_xml: string) => Record<string, string>;
 
   // RequestTimeout = 4000;
   requestTimeout = 4000;
@@ -40,7 +41,20 @@ export abstract class AbstractPastebin {
   // VERSION
   static version = "0.6.0";
 
-  constructor(config?: IPastebinOptions | string | null) {
+  constructor(config?: IPastebinOptions | string | null, options?: {
+    fetch?: typeof globalThis.fetch;
+    parseXML?: (_xml: string) => Record<string, string>;
+  }) {
+    if (options?.fetch) {
+      this.#fetch = options.fetch;
+    }
+    if (options?.parseXML) {
+      this.#parseXML = options.parseXML;
+    } else {
+      this.#parseXML = (xml: string): Record<string, string> => {
+        throw new Error("Not implemented!");
+      };
+    }
     if (isUndefined(config) || isNull(config)) {
       this.#config = {};
       return;
@@ -339,20 +353,17 @@ export abstract class AbstractPastebin {
   }
 
   // Parse
-  parseXML(_xml: string): Record<string, string> {
-    throw new Error("Not implemented!");
-  }
 
   #parseUser(xml: string): User {
-    const data = this.parseXML(xml) as { user?: User };
+    const data = this.#parseXML(xml) as { user?: User };
     if (isUndefined(data) || isNull(data) || isUndefined(data.user)) {
       throw new Error("No data returned to _parseUser!");
     }
     return data.user;
   }
 
   #parsePastes(xml: string): Paste[] {
-    const { root: data } = this.parseXML(`<root>${xml}</root>`) as unknown as {
+    const { root: data } = this.#parseXML(`<root>${xml}</root>`) as unknown as {
       root: { paste: Paste[] };
     };
     if (isUndefined(data) || isNull(data) || isUndefined(data.paste)) {
@@ -439,7 +450,7 @@ export abstract class AbstractPastebin {
     }
 
     try {
-      const res = await this.fetch(path, init);
+      const res = await this.#fetch(path, init);
       if (timeout !== null) {
         clearTimeout(timeout);
       }

src/lib/Scraper.ts

@@ -132,12 +132,16 @@ export class Scraper {
    * });
    * ```
    */
-  constructor(opts?: ScrapeOptions) {
+  constructor(opts?: ScrapeOptions, fetch?: typeof globalThis.fetch) {
     const options = {
       ...defaultOptions,
       ...opts,
     };
 
+    if (fetch) {
+      this.fetch = fetch;
+    }
+
     if (
       typeof options.intervalTime === "undefined" || !Number.isInteger(options.intervalTime) ||
       options.intervalTime < 1000

src/node/Pastebin.ts

@@ -12,15 +12,19 @@ import { Buffer } from "node:buffer";
 import fetch from "npm:[email protected]";
 import { XMLParser } from "npm:[email protected]";
 
+const parser = new XMLParser();
+
 export class Pastebin extends AbstractPastebin {
   constructor(config?: IPastebinOptions | string | null) {
-    super(config);
-    // This is probably not the best way to do this, but it works
-    super.parseXML = this.parseXml;
+    super(config, {
+      fetch: fetch as unknown as typeof globalThis.fetch,
+      parseXML: (xml: string): Record<string, string> => {
+        const data = parser.parse(xml);
+        return data;
+      },
+    });
   }
 
-  fetch = fetch as unknown as typeof globalThis.fetch;
-
   async createPasteFromFile(
     options: ICreatePasteFileOptions<Buffer> = { file: "" },
   ): Promise<unknown> {
@@ -50,10 +54,4 @@ export class Pastebin extends AbstractPastebin {
 
     return this.createPaste(pasteOpts);
   }
-
-  parseXml = (xml: string): Record<string, string> => {
-    const parser = new XMLParser();
-    const data = parser.parse(xml);
-    return data;
-  };
 }

src/node/Scraper.ts

@@ -0,0 +1,9 @@
+import { ScrapeOptions, Scraper as AbstractScraper } from "../lib/Scraper.ts";
+
+import fetch from "npm:[email protected]";
+
+export class Scraper extends AbstractScraper {
+  constructor(opts?: ScrapeOptions) {
+    super(opts, fetch as unknown as typeof globalThis.fetch);
+  }
+}

src/node/index.ts

@@ -1,5 +1,5 @@
 // Copyright 2023 J.W. Lagendijk. All rights reserved. MIT license.
 
 export * from "./Pastebin.ts";
-export * from "../lib/Scraper.ts";
+export * from "./Scraper.ts";
 export * from "../lib/interfaces.ts";