added duplicate blog posts filter

gautamkrishnar · Jul 24, 2022 · 2d84497 · 2d84497
1 parent e220d95
commit 2d84497
Show file tree

Hide file tree

Showing 8 changed files with 165 additions and 32 deletions.
diff --git a/README.md b/README.md
diff --git a/action.yml b/action.yml
@@ -117,6 +117,10 @@ inputs:
     description: "Provide your own seeding string for the randomness"
     default: ""
     required: false
+  remove_duplicates:
+    description: "Allows you to remove duplicate blog posts from multiple sources"
+    default: ""
+    required: false
 outputs:
   results:
     description: "JSON stringified array of posts"

diff --git a/local-run.js b/local-run.js
@@ -34,6 +34,7 @@ fs.writeFile(path.join(__dirname, 'test', 'Readme.md'), template, () => {
   process.env.INPUT_DISABLE_ITEM_VALIDATION = 'false';
   process.env.INPUT_FILTER_DATES = '';
   process.env.INPUT_RAND_SEED = '';
+  process.env.INPUT_REMOVE_DUPLICATES = '';
   const testFile = process.env.DIST ? './dist/blog-post-workflow' : './src/blog-post-workflow';
   console.log('Testing: ', testFile);
   require(testFile);

diff --git a/src/blog-post-workflow.js b/src/blog-post-workflow.js
@@ -124,6 +124,9 @@ feedList.forEach((siteUrl) => {
           reject('Cannot read response->item');
         } else {
           const responsePosts = data.items;
+          // To handle duplicate filter
+          const appendedPostTitles = [];
+          const appendedPostDesc = [];
           const posts = responsePosts
             .filter(ignoreMediumComments)
             .filter(ignoreStackOverflowComments)
@@ -181,6 +184,17 @@ feedList.forEach((siteUrl) => {
                   process.exit(1);
                 }
               }
+              if (post && core.getInput('remove_duplicates') === 'true') {
+                if (
+                  appendedPostTitles.indexOf(post.title) !== -1 ||
+                  appendedPostDesc.indexOf(post.description) !== -1
+                ) {
+                  post = null;
+                } else {
+                  post.title && appendedPostTitles.push(post.title)
+                  post.description && appendedPostDesc.push(post.description);
+                }
+              }
 
               // Doing HTML encoding at last ref: #117
               const disableHtmlEncoding = core.getInput('disable_html_encoding') !== 'false';

diff --git a/test/Readme.removeDuplicates.md.snap b/test/Readme.removeDuplicates.md.snap
@@ -0,0 +1,9 @@
+# Readme test
+Post list example:
+<!-- BLOG-POST-LIST:START -->
+- [God Mode in browsers: document.designMode = &quot;on&quot;](https://dev.to/gautamkrishnar/god-mode-in-browsers-document-designmode-on-2pmo)
+- [Hi, I&#39;m Gautam krishna.R](https://dev.to/gautamkrishnar/hi-im-gautam-krishnar)
+<!-- BLOG-POST-LIST:END -->
+
+# Other contents
+Test content
diff --git a/test/sample.duplicate.xml b/test/sample.duplicate.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+    <channel>
+        <title>Gautam krishna.R</title>
+        <author>Gautam krishna.R</author>
+        <description>Developer, open source lover and after all a wonderful human being... Software Engineer at Red Hat | DuckDuckGo Community Leader | Polygot</description>
+        <link>https://dev.to/gautamkrishnar</link>
+        <language>en</language>
+        <item>
+        <title>Hi, I'm Gautam krishna.R</title>
+        <author>Gautam krishna.R</author>
+        <pubDate>Sun, 02 Apr 2017 19:04:43 +0000</pubDate>
+        <link>https://dev.to/gautamkrishnar/hi-im-gautam-krishnar</link>
+        <guid>https://dev.to/gautamkrishnar/hi-im-gautam-krishnar</guid>
+        <description>&lt;p&gt;I have been coding for 9 years, I am a Microsoft student partner and DuckDuckGo community leader who's actively involved in several open source projects.&lt;/p&gt;
+
+          &lt;p&gt;You can find me on GitHub at &lt;a href="https://github.com/gautamkrishnar"&gt;@gautamkrishnar&lt;/a&gt;&lt;/p&gt;
+
+          &lt;p&gt;You can also find me on Twitter as &lt;a href="https://twitter.com/gautamkrishnar"&gt;@gautamkrishnar&lt;/a&gt;&lt;/p&gt;
+
+          &lt;p&gt;I live in Kollam.&lt;/p&gt;
+
+          &lt;p&gt;I mostly program in these languages: Phython, NodeJS, PHP, C, C#, Java.&lt;/p&gt;
+
+          &lt;p&gt;I am currently learning more about ML.&lt;/p&gt;
+
+          &lt;p&gt;Nice to meet you.&lt;/p&gt;
+
+        </description>
+          <testingTag>hello</testingTag>
+          <testingTag2>apple</testingTag2>
+        <category>introductions</category>
+      </item>
+      <item>
+        <title>God Mode in browsers: document.designMode = "on"</title>
+        <author>Gautam krishna.R</author>
+        <pubDate>Thu, 16 Jul 2020 10:40:11 +0000</pubDate>
+        <testingTag>world</testingTag>
+        <link>https://dev.to/gautamkrishnar/god-mode-in-browsers-document-designmode-on-2pmo</link>
+        <guid>https://dev.to/gautamkrishnar/god-mode-in-browsers-document-designmode-on-2pmo</guid>
+        <description>&lt;p&gt;Just type &lt;code&gt;document.designMode = "on"&lt;/code&gt; on you favourite browser devtools and see the magic. &lt;/p&gt;
+
+          &lt;p&gt;It will make any website editable:&lt;/p&gt;
+
+          &lt;p&gt;&lt;a href="https://i.giphy.com/media/iDByhJJoZGSac2RH5z/giphy.gif" class="article-body-image-wrapper"&gt;&lt;img src="https://i.giphy.com/media/iDByhJJoZGSac2RH5z/giphy.gif" alt="preview"&gt;&lt;/a&gt;&lt;/p&gt;
+
+        </description>
+        <category>chrome</category>
+        <category>firefox</category>
+        <category>devtools</category>
+        <category>webdev</category>
+      </item>
+        <item>
+            <title>God Mode in browsers: document.designMode = "on"</title>
+            <author>Gautam krishna.R</author>
+            <pubDate>Thu, 16 Jul 2020 10:40:11 +0000</pubDate>
+          <testingTag>world</testingTag>
+            <link>https://dev.to/gautamkrishnar/god-mode-in-browsers-document-designmode-on-2pmo</link>
+            <guid>https://dev.to/gautamkrishnar/god-mode-in-browsers-document-designmode-on-2pmo</guid>
+            <description>&lt;p&gt;Just type &lt;code&gt;document.designMode = "on"&lt;/code&gt; on you favourite browser devtools and see the magic. &lt;/p&gt;
+
+                &lt;p&gt;It will make any website editable:&lt;/p&gt;
+
+                &lt;p&gt;&lt;a href="https://i.giphy.com/media/iDByhJJoZGSac2RH5z/giphy.gif" class="article-body-image-wrapper"&gt;&lt;img src="https://i.giphy.com/media/iDByhJJoZGSac2RH5z/giphy.gif" alt="preview"&gt;&lt;/a&gt;&lt;/p&gt;
+
+            </description>
+            <category>chrome</category>
+            <category>firefox</category>
+            <category>devtools</category>
+            <category>webdev</category>
+        </item>
+      <item>
+        <title>Comment by Gautam Krishna R on test</title>
+        <author>Gautam Krishna R</author>
+        <pubDate>Wed, 19 Apr 2017 12:24:15 +0000</pubDate>
+        <link>https://stackoverflow.com/questions/49310909/omniauth-facebook-login-does-not-work/49311637?cid=85720166#49311637</link>
+        <guid>https://stackoverflow.com/questions/49310909/omniauth-facebook-login-does-not-work/49311637?cid=85720166#49311637</guid>
+        <description>desc</description>
+      </item>
+        <item>
+          <title>Comment by Gautam Krishna R on test</title>
+          <author>Gautam Krishna R</author>
+          <pubDate>Wed, 19 Apr 2017 12:24:15 +0000</pubDate>
+          <link>https://stackoverflow.com/questions/49310909/omniauth-facebook-login-does-not-work/49311637?cid=85720166#49311637</link>
+          <guid>https://stackoverflow.com/questions/49310909/omniauth-facebook-login-does-not-work/49311637?cid=85720166#49311637</guid>
+          <description>desc</description>
+        </item>
+
+    </channel>
+</rss>
diff --git a/test/test-server.js b/test/test-server.js
@@ -11,6 +11,8 @@ const sendResponse = (res, statusCode, data) => {
 };
 
 const xmlData = fs.readFileSync(path.join(__dirname, 'sample.xml'), 'utf-8');
+const duplicateXmlData = fs.readFileSync(path.join(__dirname, 'sample.duplicate.xml'), 'utf-8');
+
 
 http.createServer(function (req, res) {
   if (req.url === '/failtest') {
@@ -42,6 +44,8 @@ http.createServer(function (req, res) {
     </rss>
     `;
     sendResponse(res, 200, emptyTagResponse);
+  } else if (req.url === '/duplicates') {
+    sendResponse(res, 200, duplicateXmlData);
   }
   else {
     sendResponse(res, 200, xmlData);

diff --git a/test/test.js b/test/test.js
@@ -29,7 +29,8 @@ const DEFAULT_TEST_ENV = {
   INPUT_CATEGORIES_TEMPLATE: 'default',
   INPUT_DISABLE_ITEM_VALIDATION: 'false',
   INPUT_FILTER_DATES: '',
-  INPUT_RAND_SEED: ''
+  INPUT_RAND_SEED: '',
+  INPUT_REMOVE_DUPLICATES: 'false'
 };
 
 // Folder with readme snapshots
@@ -256,4 +257,13 @@ describe('Blog post workflow tests', function () {
     };
     await runAndCompareSnap('Readme.filter_dates.currentYear.md', envObj);
   });
+  it('Generated readme with remove duplicates flag should match the snapshot', async function () {
+    const envObj = {
+      ...process.env,
+      ...DEFAULT_TEST_ENV,
+      INPUT_FEED_LIST: 'http://localhost:8080/duplicates',
+      INPUT_REMOVE_DUPLICATES: 'true'
+    };
+    await runAndCompareSnap('Readme.removeDuplicates.md', envObj);
+  });
 });