From f92e2a4cd2c50b36efa0f89524935f7296f22ac3 Mon Sep 17 00:00:00 2001 From: Gavin Nelson Date: Thu, 22 Dec 2022 18:54:40 +0000 Subject: [PATCH 1/2] add pagination to Readwise API call --- readwise.mjs | 96 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/readwise.mjs b/readwise.mjs index 800c4ad..3440b97 100644 --- a/readwise.mjs +++ b/readwise.mjs @@ -1,52 +1,66 @@ -import fetch from "isomorphic-fetch"; -import { config } from "./config.mjs"; +import fetch from "isomorphic-fetch" +import { config } from "./config.mjs" -const HIGHLIGHTS_URL = "https://readwise.io/api/v2/export/"; +const HIGHLIGHTS_URL = "https://readwise.io/api/v2/export/?" const getItemsFromReadwise = async (daysToFetch = 1) => { - const dateOffset = 24 * 60 * 60 * 1000 * daysToFetch; - const updatedAfterDate = new Date(); - updatedAfterDate.setTime(updatedAfterDate.getTime() - dateOffset); + let nextPageCursor = null - const response = await fetch( - `${HIGHLIGHTS_URL}?updatedAfter=${updatedAfterDate.toISOString()}`, - { + while (true) { + const queryParams = new URLSearchParams() + const dateOffset = 24 * 60 * 60 * 1000 * daysToFetch + const updatedAfterDate = new Date() + updatedAfterDate.setTime(updatedAfterDate.getTime() - dateOffset) + + if (nextPageCursor) { + queryParams.append("pageCursor", nextPageCursor) + } + if (updatedAfterDate) { + queryParams.append("updatedAfter", updatedAfterDate.toISOString()) + } + const response = await fetch(`${HIGHLIGHTS_URL + queryParams.toString()}`, { headers: { Authorization: `Token ${config.readwiseToken}`, }, - } - ); - const data = await response.json(); - data.results.forEach((book) => { - const isValidSourceURL = book.source_url?.startsWith("https://"); - const hasHighlights = book.highlights?.length > 0; - - console.log(`- ${book.title} #from-the-web`); - if (isValidSourceURL) { - console.log(` - URL:: ${book.source_url}`); - } - console.log(` - type:: ${book.category?.replace(/s$/, "")}`); - console.log(` - author:: ${book.author}`); - if (hasHighlights) { - console.log(` - Highlights`); - } + }) - book.highlights.forEach((highlight) => { - const lines = highlight.text.split("\n"); - lines.forEach((line) => { - const cleanedLine = line.replace(/•\s+/, "").trim(); - if (cleanedLine.length > 0) { - console.log(` - ${cleanedLine}`); - } - }); + const data = await response.json() + nextPageCursor = data.nextPageCursor - if (highlight.note) { - console.log(` - ${highlight.note}`); + data.results.forEach((book) => { + const isValidSourceURL = book.source_url?.startsWith("https://") + const hasHighlights = book.highlights?.length > 0 + + console.log(`- ${book.title} #from-the-web`) + if (isValidSourceURL) { + console.log(` - URL:: ${book.source_url}`) + } + console.log(` - type:: ${book.category?.replace(/s$/, "")}`) + console.log(` - author:: ${book.author}`) + if (hasHighlights) { + console.log(` - Highlights`) } - }); - }); -}; -const daysToFetch = process.argv[2]; -console.log(`%%tana%%`); -getItemsFromReadwise(daysToFetch); + book.highlights.forEach((highlight) => { + const lines = highlight.text.split("\n") + lines.forEach((line) => { + const cleanedLine = line.replace(/•\s+/, "").trim() + if (cleanedLine.length > 0) { + console.log(` - ${cleanedLine}`) + } + }) + + if (highlight.note) { + console.log(` - ${highlight.note}`) + } + }) + }) + if (nextPageCursor == null) { + break + } + } +} + +const daysToFetch = process.argv[2] +console.log(`%%tana%%`) +getItemsFromReadwise(daysToFetch) From 499aea43df3a6184779a7c46321df2d531f43f95 Mon Sep 17 00:00:00 2001 From: Gavin Nelson Date: Thu, 22 Dec 2022 19:04:58 +0000 Subject: [PATCH 2/2] fix prettier formatting --- readwise.mjs | 64 ++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/readwise.mjs b/readwise.mjs index 3440b97..53e2a41 100644 --- a/readwise.mjs +++ b/readwise.mjs @@ -1,66 +1,66 @@ -import fetch from "isomorphic-fetch" -import { config } from "./config.mjs" +import fetch from "isomorphic-fetch"; +import { config } from "./config.mjs"; -const HIGHLIGHTS_URL = "https://readwise.io/api/v2/export/?" +const HIGHLIGHTS_URL = "https://readwise.io/api/v2/export/?"; const getItemsFromReadwise = async (daysToFetch = 1) => { - let nextPageCursor = null + let nextPageCursor = null; while (true) { - const queryParams = new URLSearchParams() - const dateOffset = 24 * 60 * 60 * 1000 * daysToFetch - const updatedAfterDate = new Date() - updatedAfterDate.setTime(updatedAfterDate.getTime() - dateOffset) + const queryParams = new URLSearchParams(); + const dateOffset = 24 * 60 * 60 * 1000 * daysToFetch; + const updatedAfterDate = new Date(); + updatedAfterDate.setTime(updatedAfterDate.getTime() - dateOffset); if (nextPageCursor) { - queryParams.append("pageCursor", nextPageCursor) + queryParams.append("pageCursor", nextPageCursor); } if (updatedAfterDate) { - queryParams.append("updatedAfter", updatedAfterDate.toISOString()) + queryParams.append("updatedAfter", updatedAfterDate.toISOString()); } const response = await fetch(`${HIGHLIGHTS_URL + queryParams.toString()}`, { headers: { Authorization: `Token ${config.readwiseToken}`, }, - }) + }); - const data = await response.json() - nextPageCursor = data.nextPageCursor + const data = await response.json(); + nextPageCursor = data.nextPageCursor; data.results.forEach((book) => { - const isValidSourceURL = book.source_url?.startsWith("https://") - const hasHighlights = book.highlights?.length > 0 + const isValidSourceURL = book.source_url?.startsWith("https://"); + const hasHighlights = book.highlights?.length > 0; - console.log(`- ${book.title} #from-the-web`) + console.log(`- ${book.title} #from-the-web`); if (isValidSourceURL) { - console.log(` - URL:: ${book.source_url}`) + console.log(` - URL:: ${book.source_url}`); } - console.log(` - type:: ${book.category?.replace(/s$/, "")}`) - console.log(` - author:: ${book.author}`) + console.log(` - type:: ${book.category?.replace(/s$/, "")}`); + console.log(` - author:: ${book.author}`); if (hasHighlights) { - console.log(` - Highlights`) + console.log(` - Highlights`); } book.highlights.forEach((highlight) => { - const lines = highlight.text.split("\n") + const lines = highlight.text.split("\n"); lines.forEach((line) => { - const cleanedLine = line.replace(/•\s+/, "").trim() + const cleanedLine = line.replace(/•\s+/, "").trim(); if (cleanedLine.length > 0) { - console.log(` - ${cleanedLine}`) + console.log(` - ${cleanedLine}`); } - }) + }); if (highlight.note) { - console.log(` - ${highlight.note}`) + console.log(` - ${highlight.note}`); } - }) - }) + }); + }); if (nextPageCursor == null) { - break + break; } } -} +}; -const daysToFetch = process.argv[2] -console.log(`%%tana%%`) -getItemsFromReadwise(daysToFetch) +const daysToFetch = process.argv[2]; +console.log(`%%tana%%`); +getItemsFromReadwise(daysToFetch);