fix thisweekinreact extractor function

carlosqsilva · Oct 13, 2024 · 70ffc0e · 70ffc0e
1 parent 661f946
commit 70ffc0e
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 0 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -30,6 +30,7 @@ jobs:
         run: pnpm install
 
       - name: Update database
+        timeout-minutes: 2
         working-directory: ./scraper
         run: pnpm run update
 

diff --git a/scraper/database.db b/scraper/database.db
diff --git a/scraper/newsletter/thisweekinreact.ts b/scraper/newsletter/thisweekinreact.ts
@@ -9,6 +9,8 @@ import { extractContentDate } from "./javascripweekly/helper.ts";
 
 const baseUrl = "https://thisweekinreact.com";
 
+const issueUrlPattern = /newsletter\/\d+$/i;
+
 export async function extractThisWeekInReact(browser: Browser, db: Storage) {
   const page = browser.newPage();
 
@@ -22,6 +24,7 @@ export async function extractThisWeekInReact(browser: Browser, db: Storage) {
     const url = issue?.querySelector("a")?.href;
 
     if (!url) throw new Error("failed to extract url");
+    if (!issueUrlPattern.test(url)) continue;
     if (db.isSaved(url)) continue;
     if (url.endsWith("previous")) continue;