Add support for aggregators

README.md

@@ -4,17 +4,17 @@ An AI-powered news curator. It reads a list of articles, selects the best ones d
 
 ## Requirements
 
-- Node.js >= 18
-- an [OpenAI API](https://platform.openai.com/) key
+-   Node.js >= 18
+-   an [OpenAI API](https://platform.openai.com/) key
 
 ## CLI Usage
 
 ```sh
 # Install the package globally
 npm install -g curator-ai
 
-# Summarize a list of articles based on a file containing URLs
-OPENAI_API_KEY=XXX curate -f myFile.txt
+# Summarize a list of articles based on aggregator URLs
+OPENAI_API_KEY=XXX curate -a https://news.ycombinator.com/ https://lobste.rs/
 ```
 
 Example output:
@@ -27,14 +27,17 @@ More options:
 # Get usage information
 curate
 
-# Summarize a list of articles based on URLs passed as parameters
-OPENAI_API_KEY=XXX curate -l https://example.com/article1 https://example.com/article2
+# Summarize a list of articles based on URLs passed directly as parameters
+OPENAI_API_KEY=XXX curate -u https://example.com/article1 https://example.com/article2
+
+# Summarize a list of articles based on a file containing URLs
+OPENAI_API_KEY=XXX curate -uf myFile.txt
 
 # Return at most 5 articles
-OPENAI_API_KEY=XXX curate -f myFile.txt -m 5
+OPENAI_API_KEY=XXX curate -a https://news.ycombinator.com/ -m 5
 
 # Return the articles about AI and React
-OPENAI_API_KEY=XXX curate -f myFile.txt -i AI React
+OPENAI_API_KEY=XXX curate -a https://news.ycombinator.com/ -i AI React
 ```
 
 You can also put the API key in a `.env` file:
@@ -65,23 +68,23 @@ OPENAI_API_KEY=XXX
 Use the `curate` function:
 
 ```js
-const { curate } = require("curator-ai");
+const { curate } = require('curator-ai');
 
 const links = [
-  "https://stability.ai/news/stable-code-2024-llm-code-completion-release",
-  "https://www.fromjason.xyz/p/notebook/where-have-all-the-websites-gone/",
-  "https://www.joelonsoftware.com/2000/04/06/things-you-should-never-do-part-i/",
-  "https://biomejs.dev/blog/biome-v1-5/",
-  "https://birtles.blog/2024/01/06/weird-things-engineers-believe-about-development/",
-  "https://julesblom.com/writing/flushsync",
+    'https://stability.ai/news/stable-code-2024-llm-code-completion-release',
+    'https://www.fromjason.xyz/p/notebook/where-have-all-the-websites-gone/',
+    'https://www.joelonsoftware.com/2000/04/06/things-you-should-never-do-part-i/',
+    'https://biomejs.dev/blog/biome-v1-5/',
+    'https://birtles.blog/2024/01/06/weird-things-engineers-believe-about-development/',
+    'https://julesblom.com/writing/flushsync',
 ];
 
 curate({
-  links,
-  interests: ["react", "ai"],
-  max: 5,
-}).then((curatedLinks) => {
-  console.log(curatedLinks);
+    links,
+    interests: ['react', 'ai'],
+    max: 5,
+}).then(curatedLinks => {
+    console.log(curatedLinks);
 });
 
 // [
@@ -104,8 +107,8 @@ npm install
 # Get usage information
 npm start
 
-# Summarize a list of articles based on URLs passed as parameters (notice the --):
-npm start -- -l https://example.com/article1 https://example.com/article2
+# Summarize a list of articles based on aggregator URLs (notice the --):
+npm start -- -a https://news.ycombinator.com/
 ```
 
 Don't forget to pass a valid OpenAI key, either as an environment variable or in a `.env` file.

package.json

@@ -5,7 +5,7 @@
     "type": "git",
     "url": "https://github.com/marmelab/curator-ai"
   },
-  "version": "0.0.3",
+  "version": "0.0.4",
   "main": "./dist/index.js",
   "bin": {
     "curate": "./dist/cli.js"

src/cli.ts

@@ -3,41 +3,83 @@ import { program } from 'commander';
 import fs from 'node:fs';
 import cliProgress from 'cli-progress';
 
+import { getLinks } from './getLinks';
 import { curate } from './curate';
 import { consoleFormat } from './consoleFormat';
 
+const helpText = `Examples:
+  $ curate -a https://news.ycombinator.com/ -i science space research -m 3
+`;
 program
     .name('curate')
     .description('Read, select and summarize a list of articles')
-    .option('-l, --links [links...]', 'List of article links')
+    .option('-u, --urls [urls...]', 'Wep pages to curate')
     .option(
-        '-f, --file <filename>',
-        'File containing an array of URLs to curate, one per line'
+        '-uf, --url-file <filename>',
+        'Text file containing a list of URLs to curate, one per line'
+    )
+    .option('-a, --aggregators [urls...]', 'Aggregator web pages to curate')
+    .option(
+        '-af, --aggregator-file <filename>',
+        'Text file containing a list of aggregator URLs to curate, one per line'
     )
     .option('-i, --interests [interests...]', 'List of interests')
     .option('-m, --max <number>', 'Max number of articles to return', '5')
+    .addHelpText('after', helpText)
     .showHelpAfterError()
     .action(async options => {
-        let finalLinks: string[] = options.links || [];
-        if (options.file) {
-            const linkFile = fs.readFileSync(options.file, 'utf8');
-            finalLinks = linkFile.split('\n');
+        let urls: string[] = options.urls || [];
+        let aggregatorUrls: string[] = options.aggregators || [];
+        if (options.aggregatorFile) {
+            const aggregatorFile = fs.readFileSync(
+                options.aggregatorFile,
+                'utf8'
+            );
+            aggregatorUrls = [...aggregatorUrls, ...aggregatorFile.split('\n')];
         }
-        if (!finalLinks.length) {
+        if (!urls.length && !aggregatorUrls.length) {
             program.help();
         }
+
+        // get links from curators
+        if (aggregatorUrls.length) {
+            const progressBar = new cliProgress.SingleBar(
+                {},
+                cliProgress.Presets.shades_classic
+            );
+            progressBar.start(aggregatorUrls.length, 0);
+            for (let i = 0; i < aggregatorUrls.length; i++) {
+                const newUrls = await getLinks({
+                    url: aggregatorUrls[i],
+                });
+                urls = [...urls, ...newUrls];
+                progressBar.update(i + 1);
+            }
+            progressBar.stop();
+        }
+        if (options.urlFile) {
+            const linkFile = fs.readFileSync(options.urlFile, 'utf8');
+            urls = [...urls, ...linkFile.split('\n')];
+        }
+
+        // deduplicate urls
+        urls = [...new Set(urls)];
+
+        // curate
         const progressBar = new cliProgress.SingleBar(
             {},
             cliProgress.Presets.shades_classic
         );
-        progressBar.start(finalLinks.length, 0);
+        progressBar.start(urls.length, 0);
         const summaries = await curate({
-            links: finalLinks,
+            links: urls,
             interests: options.interests,
             max: options.max,
             onProgress: progress => progressBar.update(progress),
         });
         progressBar.stop();
+
+        // print summaries
         console.log();
         console.log(consoleFormat(summaries));
     });

src/index.ts

@@ -1 +1,2 @@
-export { curate } from "./curate";
+export { curate } from './curate';
+export { getLinks } from './getLinks';