feat: add rss widget (#760)

Co-authored-by: SeDemal <[email protected]>

apps/nextjs/src/app/[locale]/layout.tsx

@@ -4,6 +4,7 @@ import { Inter } from "next/font/google";
 import "@homarr/ui/styles.css";
 import "@homarr/notifications/styles.css";
 import "@homarr/spotlight/styles.css";
+import "~/styles/scroll-area.scss";
 
 import { ColorSchemeScript, createTheme, MantineProvider } from "@mantine/core";
 

apps/nextjs/src/styles/scroll-area.scss

@@ -0,0 +1,4 @@
+.scroll-area-w100 .mantine-ScrollArea-viewport > div:nth-of-type(1) {
+  width: 100%;
+  display: inherit !important;
+}

packages/api/src/middlewares/item.ts

@@ -0,0 +1,29 @@
+import { TRPCError } from "@trpc/server";
+
+import { and, eq } from "@homarr/db";
+import { items } from "@homarr/db/schema/sqlite";
+import type { WidgetKind } from "@homarr/definitions";
+import { z } from "@homarr/validation";
+
+import { publicProcedure } from "../trpc";
+
+export const createOneItemMiddleware = (kind: WidgetKind) => {
+  return publicProcedure.input(z.object({ itemId: z.string() })).use(async ({ input, ctx, next }) => {
+    const item = await ctx.db.query.items.findFirst({
+      where: and(eq(items.id, input.itemId), eq(items.kind, kind)),
+    });
+
+    if (!item) {
+      throw new TRPCError({
+        code: "NOT_FOUND",
+        message: `Item with id ${input.itemId} not found`,
+      });
+    }
+
+    return next({
+      ctx: {
+        item,
+      },
+    });
+  });
+};

packages/api/src/router/widgets/index.ts

@@ -4,6 +4,7 @@ import { calendarRouter } from "./calendar";
 import { dnsHoleRouter } from "./dns-hole";
 import { mediaServerRouter } from "./media-server";
 import { notebookRouter } from "./notebook";
+import { rssFeedRouter } from "./rssFeed";
 import { smartHomeRouter } from "./smart-home";
 import { weatherRouter } from "./weather";
 
@@ -15,4 +16,5 @@ export const widgetRouter = createTRPCRouter({
   smartHome: smartHomeRouter,
   mediaServer: mediaServerRouter,
   calendar: calendarRouter,
+  rssFeed: rssFeedRouter,
 });

packages/api/src/router/widgets/rssFeed.ts

@@ -0,0 +1,12 @@
+import type { RssFeed } from "@homarr/cron-jobs";
+import { createItemChannel } from "@homarr/redis";
+
+import { createOneItemMiddleware } from "../../middlewares/item";
+import { createTRPCRouter, publicProcedure } from "../../trpc";
+
+export const rssFeedRouter = createTRPCRouter({
+  getFeeds: publicProcedure.unstable_concat(createOneItemMiddleware("rssFeed")).query(async ({ input }) => {
+    const channel = createItemChannel<RssFeed[]>(input.itemId);
+    return await channel.getAsync();
+  }),
+});

packages/cron-jobs-core/src/creator.ts

@@ -38,7 +38,8 @@ const createCallback = <TAllowedNames extends string, TName extends TAllowedName
         );
         await creatorOptions.onCallbackSuccess?.(name);
       } catch (error) {
-        creatorOptions.logger.logError(error);
+        // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        creatorOptions.logger.logError(`Failed to run job '${name}': ${error}`);
         await creatorOptions.onCallbackError?.(name, error);
       }
     };

packages/cron-jobs/package.json

@@ -21,18 +21,20 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@homarr/log": "workspace:^0.1.0",
+    "@extractus/feed-extractor": "^7.1.3",
+    "@homarr/analytics": "workspace:^0.1.0",
+    "@homarr/common": "workspace:^0.1.0",
     "@homarr/cron-job-status": "workspace:^0.1.0",
     "@homarr/cron-jobs-core": "workspace:^0.1.0",
-    "@homarr/analytics": "workspace:^0.1.0",
     "@homarr/db": "workspace:^0.1.0",
-    "@homarr/server-settings": "workspace:^0.1.0",
-    "@homarr/common": "workspace:^0.1.0",
     "@homarr/icons": "workspace:^0.1.0",
     "@homarr/integrations": "workspace:^0.1.0",
-    "@homarr/redis": "workspace:^0.1.0",
+    "@homarr/log": "workspace:^0.1.0",
     "@homarr/ping": "workspace:^0.1.0",
-    "@homarr/translation": "workspace:^0.1.0"
+    "@homarr/redis": "workspace:^0.1.0",
+    "@homarr/server-settings": "workspace:^0.1.0",
+    "@homarr/translation": "workspace:^0.1.0",
+    "@homarr/validation": "workspace:^0.1.0"
   },
   "devDependencies": {
     "@homarr/eslint-config": "workspace:^0.2.0",

packages/cron-jobs/src/index.ts

@@ -4,6 +4,8 @@ import { smartHomeEntityStateJob } from "./jobs/integrations/home-assistant";
 import { mediaOrganizerJob } from "./jobs/integrations/media-organizer";
 import { mediaServerJob } from "./jobs/integrations/media-server";
 import { pingJob } from "./jobs/ping";
+import type { RssFeed } from "./jobs/rss-feeds";
+import { rssFeedsJob } from "./jobs/rss-feeds";
 import { createCronJobGroup } from "./lib";
 
 export const jobGroup = createCronJobGroup({
@@ -13,6 +15,8 @@ export const jobGroup = createCronJobGroup({
   smartHomeEntityState: smartHomeEntityStateJob,
   mediaServer: mediaServerJob,
   mediaOrganizer: mediaOrganizerJob,
+  rssFeeds: rssFeedsJob,
 });
 
 export type JobGroupKeys = ReturnType<(typeof jobGroup)["getKeys"]>[number];
+export type { RssFeed };

packages/cron-jobs/src/jobs/rss-feeds.ts

@@ -0,0 +1,135 @@
+import type { FeedData, FeedEntry } from "@extractus/feed-extractor";
+import { extract } from "@extractus/feed-extractor";
+import SuperJSON from "superjson";
+
+import { EVERY_5_MINUTES } from "@homarr/cron-jobs-core/expressions";
+import { db, eq } from "@homarr/db";
+import { items } from "@homarr/db/schema/sqlite";
+import { logger } from "@homarr/log";
+import { createItemChannel } from "@homarr/redis";
+import { z } from "@homarr/validation";
+
+// This import is done that way to avoid circular dependencies.
+import type { WidgetComponentProps } from "../../../widgets";
+import { createCronJob } from "../lib";
+
+export const rssFeedsJob = createCronJob("rssFeeds", EVERY_5_MINUTES).withCallback(async () => {
+  const itemsForIntegration = await db.query.items.findMany({
+    where: eq(items.kind, "rssFeed"),
+  });
+
+  for (const item of itemsForIntegration) {
+    const options = SuperJSON.parse<WidgetComponentProps<"rssFeed">["options"]>(item.options);
+
+    const feeds = await Promise.all(
+      options.feedUrls.map(async (feedUrl) => ({
+        feedUrl,
+        feed: (await extract(feedUrl, {
+          getExtraEntryFields: (feedEntry) => {
+            const media = attemptGetImageFromEntry(feedUrl, feedEntry);
+            if (!media) {
+              return {};
+            }
+            return {
+              enclosure: media,
+            };
+          },
+        })) as ExtendedFeedData,
+      })),
+    );
+
+    const channel = createItemChannel<RssFeed[]>(item.id);
+    await channel.publishAndUpdateLastStateAsync(feeds);
+  }
+});
+
+const attemptGetImageFromEntry = (feedUrl: string, entry: object) => {
+  const media = getFirstMediaProperty(entry);
+  if (media !== null) {
+    return media;
+  }
+  return getImageFromStringAsFallback(feedUrl, JSON.stringify(entry));
+};
+
+const getImageFromStringAsFallback = (feedUrl: string, content: string) => {
+  const regex = /https?:\/\/\S+?\.(jpg|jpeg|png|gif|bmp|svg|webp|tiff)/i;
+  const result = regex.exec(content);
+
+  if (result == null) {
+    return null;
+  }
+
+  console.debug(
+    `Falling back to regex image search for '${feedUrl}'. Found ${result.length} matches in content: ${content}`,
+  );
+  return result[0];
+};
+
+const mediaProperties = [
+  {
+    path: ["enclosure", "@_url"],
+  },
+  {
+    path: ["media:content", "@_url"],
+  },
+];
+
+/**
+ * The RSS and Atom standards are poorly adhered to in most of the web.
+ * We want to show pretty background images on the posts and therefore need to extract
+ * the enclosure (aka. media images). This function uses the dynamic properties defined above
+ * to search through the possible paths and detect valid image URLs.
+ * @param feedObject The object to scan for.
+ * @returns the value of the first path that is found within the object
+ */
+const getFirstMediaProperty = (feedObject: object) => {
+  for (const mediaProperty of mediaProperties) {
+    let propertyIndex = 0;
+    let objectAtPath: object = feedObject;
+    while (propertyIndex < mediaProperty.path.length) {
+      const key = mediaProperty.path[propertyIndex];
+      if (key === undefined) {
+        break;
+      }
+      const propertyEntries = Object.entries(objectAtPath);
+      const propertyEntry = propertyEntries.find(([entryKey]) => entryKey === key);
+      if (!propertyEntry) {
+        break;
+      }
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+      const [_, propertyEntryValue] = propertyEntry;
+      objectAtPath = propertyEntryValue as object;
+      propertyIndex++;
+    }
+
+    const validationResult = z.string().url().safeParse(objectAtPath);
+    if (!validationResult.success) {
+      continue;
+    }
+
+    logger.debug(`Found an image in the feed entry: ${validationResult.data}`);
+    return validationResult.data;
+  }
+  return null;
+};
+
+/**
+ * We extend the feed with custom properties.
+ * This interface adds properties on top of the default ones.
+ */
+interface ExtendedFeedEntry extends FeedEntry {
+  enclosure?: string;
+}
+
+/**
+ * We extend the feed with custom properties.
+ * This interface omits the default entries with our custom definition.
+ */
+interface ExtendedFeedData extends Omit<FeedData, "entries"> {
+  entries?: ExtendedFeedEntry;
+}
+
+export interface RssFeed {
+  feedUrl: string;
+  feed: ExtendedFeedData;
+}

packages/definitions/src/widget.ts

@@ -10,5 +10,6 @@ export const widgetKinds = [
   "smartHome-executeAutomation",
   "mediaServer",
   "calendar",
+  "rssFeed",
 ] as const;
 export type WidgetKind = (typeof widgetKinds)[number];

packages/redis/src/index.ts

@@ -1,6 +1,6 @@
 import { createListChannel, createQueueChannel, createSubPubChannel } from "./lib/channel";
 
-export { createCacheChannel, createItemAndIntegrationChannel } from "./lib/channel";
+export { createCacheChannel, createItemAndIntegrationChannel, createItemChannel } from "./lib/channel";
 
 export const exampleChannel = createSubPubChannel<{ message: string }>("example");
 export const pingChannel = createSubPubChannel<{ url: string; statusCode: number } | { url: string; error: string }>(

packages/redis/src/lib/channel.ts

@@ -163,6 +163,14 @@ export const createCacheChannel = <TData>(name: string, cacheDurationMs: number
 
 export const createItemAndIntegrationChannel = <TData>(kind: WidgetKind, integrationId: string) => {
   const channelName = `item:${kind}:integration:${integrationId}`;
+  return createChannelWithLatestAndEvents<TData>(channelName);
+};
+
+export const createItemChannel = <TData>(itemId: string) => {
+  return createChannelWithLatestAndEvents<TData>(`item:${itemId}`);
+};
+
+const createChannelWithLatestAndEvents = <TData>(channelName: string) => {
   return {
     subscribe: (callback: (data: TData) => void) => {
       return ChannelSubscriptionTracker.subscribe(channelName, (message) => {

packages/translation/src/lang/en.ts

@@ -1033,6 +1033,21 @@ export default {
       description: "Show the current streams on your media servers",
       option: {},
     },
+    rssFeed: {
+      name: "RSS feeds",
+      description: "Monitor and display one or more generic RSS, ATOM or JSON feeds",
+      option: {
+        feedUrls: {
+          label: "Feed URLs",
+        },
+        textLinesClamp: {
+          label: "Description line clamp",
+        },
+        maximumAmountPosts: {
+          label: "Amount posts limit",
+        },
+      },
+    },
   },
   widgetPreview: {
     toggle: {
@@ -1494,6 +1509,9 @@ export default {
             mediaOrganizer: {
               label: "Media Organizers",
             },
+            rssFeeds: {
+              label: "RSS feeds",
+            },
           },
         },
       },

packages/widgets/package.json

@@ -22,6 +22,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@extractus/feed-extractor": "^7.1.3",
     "@homarr/api": "workspace:^0.1.0",
     "@homarr/common": "workspace:^0.1.0",
     "@homarr/definitions": "workspace:^0.1.0",

packages/widgets/src/_inputs/index.ts

@@ -1,6 +1,7 @@
 import type { WidgetOptionType } from "../options";
 import { WidgetAppInput } from "./widget-app-input";
 import { WidgetLocationInput } from "./widget-location-input";
+import { WidgetMultiTextInput } from "./widget-multi-text-input";
 import { WidgetMultiSelectInput } from "./widget-multiselect-input";
 import { WidgetNumberInput } from "./widget-number-input";
 import { WidgetSelectInput } from "./widget-select-input";
@@ -12,7 +13,7 @@ const mapping = {
   text: WidgetTextInput,
   location: WidgetLocationInput,
   multiSelect: WidgetMultiSelectInput,
-  multiText: () => null,
+  multiText: WidgetMultiTextInput,
   number: WidgetNumberInput,
   select: WidgetSelectInput,
   slider: WidgetSliderInput,