From 19cd41a8e52585963ffe210186daadb1513b4dca Mon Sep 17 00:00:00 2001 From: Manuel <30572287+manuel-rw@users.noreply.github.com> Date: Fri, 6 Sep 2024 22:51:18 +0200 Subject: [PATCH] feat: add crawling settings (#959) Co-authored-by: Meier Lukas --- apps/nextjs/src/app/[locale]/layout.tsx | 2 + .../src/app/[locale]/manage/about/page.tsx | 46 +++++++------ .../_components/analytics.settings.tsx | 45 ++---------- .../crawling-and-indexing.settings.tsx | 68 +++++++++++++++++++ .../settings/_components/setting-switch.tsx | 46 +++++++++++++ .../src/app/[locale]/manage/settings/page.tsx | 2 + .../api/about/contributors/crowdin/route.ts | 9 +++ .../api/about/contributors/github/route.ts | 9 +++ .../layout/search-engine-optimization.tsx | 33 +++++++++ packages/server-settings/src/index.ts | 8 ++- packages/translation/src/lang/en.ts | 21 ++++++ 11 files changed, 227 insertions(+), 62 deletions(-) create mode 100644 apps/nextjs/src/app/[locale]/manage/settings/_components/crawling-and-indexing.settings.tsx create mode 100644 apps/nextjs/src/app/[locale]/manage/settings/_components/setting-switch.tsx create mode 100644 apps/nextjs/src/app/api/about/contributors/crowdin/route.ts create mode 100644 apps/nextjs/src/app/api/about/contributors/github/route.ts create mode 100644 apps/nextjs/src/components/layout/search-engine-optimization.tsx diff --git a/apps/nextjs/src/app/[locale]/layout.tsx b/apps/nextjs/src/app/[locale]/layout.tsx index ae7c4600c..c1b37045b 100644 --- a/apps/nextjs/src/app/[locale]/layout.tsx +++ b/apps/nextjs/src/app/[locale]/layout.tsx @@ -12,6 +12,7 @@ import { ModalProvider } from "@homarr/modals"; import { Notifications } from "@homarr/notifications"; import { Analytics } from "~/components/layout/analytics"; +import { SearchEngineOptimization } from "~/components/layout/search-engine-optimization"; import { JotaiProvider } from "./_client-providers/jotai"; import { CustomMantineProvider } from "./_client-providers/mantine"; import { NextInternationalProvider } from "./_client-providers/next-international"; @@ -70,6 +71,7 @@ export default async function Layout(props: { children: React.ReactNode; params: + diff --git a/apps/nextjs/src/app/[locale]/manage/about/page.tsx b/apps/nextjs/src/app/[locale]/manage/about/page.tsx index 3724cb668..f500f604a 100644 --- a/apps/nextjs/src/app/[locale]/manage/about/page.tsx +++ b/apps/nextjs/src/app/[locale]/manage/about/page.tsx @@ -1,3 +1,4 @@ +import { headers } from "next/headers"; import Image from "next/image"; import { Accordion, @@ -17,16 +18,15 @@ import { Title, } from "@mantine/core"; import { IconLanguage, IconLibrary, IconUsers } from "@tabler/icons-react"; -import { setStaticParamsLocale } from "next-international/server"; -import { getScopedI18n, getStaticParams } from "@homarr/translation/server"; +import { getScopedI18n } from "@homarr/translation/server"; import { homarrLogoPath } from "~/components/layout/logo/homarr-logo"; import { DynamicBreadcrumb } from "~/components/navigation/dynamic-breadcrumb"; import { createMetaTitle } from "~/metadata"; import { getPackageAttributesAsync } from "~/versions/package-reader"; -import contributorsData from "../../../../../../../static-data/contributors.json"; -import translatorsData from "../../../../../../../static-data/translators.json"; +import type githubContributorsJson from "../../../../../../../static-data/contributors.json"; +import type crowdinContributorsJson from "../../../../../../../static-data/translators.json"; import classes from "./about.module.css"; export async function generateMetadata() { @@ -37,16 +37,26 @@ export async function generateMetadata() { }; } -interface PageProps { - params: { - locale: string; - }; -} +const getHost = () => { + if (process.env.HOSTNAME) { + return `${process.env.HOSTNAME}:3000`; + } -export default async function AboutPage({ params: { locale } }: PageProps) { - setStaticParamsLocale(locale); + return headers().get("host"); +}; + +export default async function AboutPage() { + const baseServerUrl = `http://${getHost()}`; const t = await getScopedI18n("management.page.about"); const attributes = await getPackageAttributesAsync(); + const githubContributors = (await fetch(`${baseServerUrl}/api/about/contributors/github`).then((res) => + res.json(), + )) as typeof githubContributorsJson; + + const crowdinContributors = (await fetch(`${baseServerUrl}/api/about/contributors/crowdin`).then((res) => + res.json(), + )) as typeof crowdinContributorsJson; + return (
@@ -70,14 +80,14 @@ export default async function AboutPage({ params: { locale } }: PageProps) { {t("accordion.contributors.title")} {t("accordion.contributors.subtitle", { - count: contributorsData.length, + count: githubContributors.length, })} - {contributorsData.map((contributor) => ( + {githubContributors.map((contributor) => ( {t("accordion.translators.title")} {t("accordion.translators.subtitle", { - count: translatorsData.length, + count: crowdinContributors.length, })} - {translatorsData.map((translator) => ( + {crowdinContributors.map((translator) => ( ); }; - -export function generateStaticParams() { - return getStaticParams(); -} - -export const dynamic = "force-static"; diff --git a/apps/nextjs/src/app/[locale]/manage/settings/_components/analytics.settings.tsx b/apps/nextjs/src/app/[locale]/manage/settings/_components/analytics.settings.tsx index fe10331c9..f56b4a1b9 100644 --- a/apps/nextjs/src/app/[locale]/manage/settings/_components/analytics.settings.tsx +++ b/apps/nextjs/src/app/[locale]/manage/settings/_components/analytics.settings.tsx @@ -1,16 +1,14 @@ "use client"; -import type { ReactNode } from "react"; import React from "react"; -import type { MantineSpacing } from "@mantine/core"; -import { Card, Group, LoadingOverlay, Stack, Switch, Text, Title, UnstyledButton } from "@mantine/core"; +import { Card, LoadingOverlay, Stack, Title } from "@mantine/core"; import { clientApi } from "@homarr/api/client"; -import type { UseFormReturnType } from "@homarr/form"; import { useForm } from "@homarr/form"; import type { defaultServerSettings } from "@homarr/server-settings"; import { useScopedI18n } from "@homarr/translation/client"; +import { SwitchSetting } from "~/app/[locale]/manage/settings/_components/setting-switch"; import { revalidatePathActionAsync } from "~/app/revalidatePathAction"; interface AnalyticsSettingsProps { @@ -62,6 +60,7 @@ export const AnalyticsSettings = ({ initialData }: AnalyticsSettingsProps) => { ms="xl" title={t("integrationData.title")} text={t("integrationData.text")} + disabled={!form.values.enableGeneral} /> { ms="xl" title={t("widgetData.title")} text={t("widgetData.text")} + disabled={!form.values.enableGeneral} /> { ms="xl" title={t("usersData.title")} text={t("usersData.text")} + disabled={!form.values.enableGeneral} /> ); }; - -const SwitchSetting = ({ - form, - ms, - title, - text, - formKey, -}: { - form: UseFormReturnType; - formKey: keyof typeof defaultServerSettings.analytics; - ms?: MantineSpacing; - title: string; - text: ReactNode; -}) => { - const disabled = formKey !== "enableGeneral" && !form.values.enableGeneral; - const handleClick = React.useCallback(() => { - if (disabled) { - return; - } - form.setFieldValue(formKey, !form.values[formKey]); - }, [form, formKey, disabled]); - - return ( - - - - {title} - - {text} - - - - - - ); -}; diff --git a/apps/nextjs/src/app/[locale]/manage/settings/_components/crawling-and-indexing.settings.tsx b/apps/nextjs/src/app/[locale]/manage/settings/_components/crawling-and-indexing.settings.tsx new file mode 100644 index 000000000..a74c4cf47 --- /dev/null +++ b/apps/nextjs/src/app/[locale]/manage/settings/_components/crawling-and-indexing.settings.tsx @@ -0,0 +1,68 @@ +"use client"; + +import React from "react"; +import { Card, LoadingOverlay, Stack, Text, Title } from "@mantine/core"; + +import { clientApi } from "@homarr/api/client"; +import { useForm } from "@homarr/form"; +import type { defaultServerSettings } from "@homarr/server-settings"; +import { useScopedI18n } from "@homarr/translation/client"; + +import { SwitchSetting } from "~/app/[locale]/manage/settings/_components/setting-switch"; +import { revalidatePathActionAsync } from "~/app/revalidatePathAction"; + +interface CrawlingAndIndexingSettingsProps { + initialData: typeof defaultServerSettings.crawlingAndIndexing; +} + +export const CrawlingAndIndexingSettings = ({ initialData }: CrawlingAndIndexingSettingsProps) => { + const t = useScopedI18n("management.page.settings.section.crawlingAndIndexing"); + const form = useForm({ + initialValues: initialData, + onValuesChange: (updatedValues, _) => { + if (!form.isValid()) { + return; + } + + void mutateAsync({ + settingsKey: "crawlingAndIndexing", + value: updatedValues, + }); + }, + }); + + const { mutateAsync, isPending } = clientApi.serverSettings.saveSettings.useMutation({ + onSettled: async () => { + await revalidatePathActionAsync("/manage/settings"); + }, + }); + + return ( + <> + {t("title")} + + + + {t("warning")} + + + + + + + + + + + ); +}; diff --git a/apps/nextjs/src/app/[locale]/manage/settings/_components/setting-switch.tsx b/apps/nextjs/src/app/[locale]/manage/settings/_components/setting-switch.tsx new file mode 100644 index 000000000..804858768 --- /dev/null +++ b/apps/nextjs/src/app/[locale]/manage/settings/_components/setting-switch.tsx @@ -0,0 +1,46 @@ +import type { ReactNode } from "react"; +import React from "react"; +import type { MantineSpacing } from "@mantine/core"; +import { Group, Stack, Switch, Text, UnstyledButton } from "@mantine/core"; + +import type { UseFormReturnType } from "@homarr/form"; + +export const SwitchSetting = >({ + form, + ms, + title, + text, + formKey, + disabled, +}: { + form: Omit TFormValue>, "setFieldValue"> & { + setFieldValue: (key: keyof TFormValue, value: (previous: boolean) => boolean) => void; + }; + formKey: keyof TFormValue; + ms?: MantineSpacing; + title: string; + text: ReactNode; + disabled?: boolean; +}) => { + const handleClick = React.useCallback(() => { + if (disabled) { + return; + } + + form.setFieldValue(formKey, (previous) => !previous); + }, [form, formKey, disabled]); + + return ( + + + + {title} + + {text} + + + + + + ); +}; diff --git a/apps/nextjs/src/app/[locale]/manage/settings/page.tsx b/apps/nextjs/src/app/[locale]/manage/settings/page.tsx index c90261c37..26ef0b3c1 100644 --- a/apps/nextjs/src/app/[locale]/manage/settings/page.tsx +++ b/apps/nextjs/src/app/[locale]/manage/settings/page.tsx @@ -3,6 +3,7 @@ import { Stack, Title } from "@mantine/core"; import { api } from "@homarr/api/server"; import { getScopedI18n } from "@homarr/translation/server"; +import { CrawlingAndIndexingSettings } from "~/app/[locale]/manage/settings/_components/crawling-and-indexing.settings"; import { DynamicBreadcrumb } from "~/components/navigation/dynamic-breadcrumb"; import { AnalyticsSettings } from "./_components/analytics.settings"; @@ -24,6 +25,7 @@ export default async function SettingsPage() { {t("title")} + ); diff --git a/apps/nextjs/src/app/api/about/contributors/crowdin/route.ts b/apps/nextjs/src/app/api/about/contributors/crowdin/route.ts new file mode 100644 index 000000000..a149c082d --- /dev/null +++ b/apps/nextjs/src/app/api/about/contributors/crowdin/route.ts @@ -0,0 +1,9 @@ +import { NextResponse } from "next/server"; + +import crowdinContributors from "../../../../../../../../static-data/translators.json"; + +export const GET = () => { + return NextResponse.json(crowdinContributors); +}; + +export const dynamic = "force-static"; diff --git a/apps/nextjs/src/app/api/about/contributors/github/route.ts b/apps/nextjs/src/app/api/about/contributors/github/route.ts new file mode 100644 index 000000000..a6fc81173 --- /dev/null +++ b/apps/nextjs/src/app/api/about/contributors/github/route.ts @@ -0,0 +1,9 @@ +import { NextResponse } from "next/server"; + +import githubContributors from "../../../../../../../../static-data/contributors.json"; + +export const GET = () => { + return NextResponse.json(githubContributors); +}; + +export const dynamic = "force-static"; diff --git a/apps/nextjs/src/components/layout/search-engine-optimization.tsx b/apps/nextjs/src/components/layout/search-engine-optimization.tsx new file mode 100644 index 000000000..2fcc82d66 --- /dev/null +++ b/apps/nextjs/src/components/layout/search-engine-optimization.tsx @@ -0,0 +1,33 @@ +import SuperJSON from "superjson"; + +import { db, eq } from "@homarr/db"; +import { serverSettings } from "@homarr/db/schema/sqlite"; +import type { defaultServerSettings } from "@homarr/server-settings"; + +export const SearchEngineOptimization = async () => { + const crawlingAndIndexingSetting = await db.query.serverSettings.findFirst({ + where: eq(serverSettings.settingKey, "crawlingAndIndexing"), + }); + + if (!crawlingAndIndexingSetting) { + return null; + } + + const value = SuperJSON.parse<(typeof defaultServerSettings)["crawlingAndIndexing"]>( + crawlingAndIndexingSetting.value, + ); + + const robotsAttributes = [...(value.noIndex ? ["noindex"] : []), ...(value.noIndex ? ["nofollow"] : [])]; + + const googleAttributes = [ + ...(value.noSiteLinksSearchBox ? ["nositelinkssearchbox"] : []), + ...(value.noTranslate ? ["notranslate"] : []), + ]; + + return ( + <> + + + + ); +}; diff --git a/packages/server-settings/src/index.ts b/packages/server-settings/src/index.ts index 672481db4..ee5a6ade8 100644 --- a/packages/server-settings/src/index.ts +++ b/packages/server-settings/src/index.ts @@ -1,4 +1,4 @@ -export const defaultServerSettingsKeys = ["analytics"] as const; +export const defaultServerSettingsKeys = ["analytics", "crawlingAndIndexing"] as const; export type ServerSettingsRecord = { [key in (typeof defaultServerSettingsKeys)[number]]: Record; @@ -11,6 +11,12 @@ export const defaultServerSettings = { enableIntegrationData: false, enableUserData: false, }, + crawlingAndIndexing: { + noIndex: true, + noFollow: true, + noTranslate: true, + noSiteLinksSearchBox: false, + }, } satisfies ServerSettingsRecord; export type ServerSettings = typeof defaultServerSettings; diff --git a/packages/translation/src/lang/en.ts b/packages/translation/src/lang/en.ts index e251e1337..aadea37dc 100644 --- a/packages/translation/src/lang/en.ts +++ b/packages/translation/src/lang/en.ts @@ -1622,6 +1622,27 @@ export default { text: "Send the amount of users and whether you've activated SSO", }, }, + crawlingAndIndexing: { + title: "Crawling and Indexing", + warning: + "Enabling or disabling any settings here will severely impact how search engines will index & crawl your page. Any setting is a request and it is up to the crawler to apply these settings. Any modification may take up to multiple days or weeks to apply. Some settings may be search engine specific.", + noIndex: { + title: "No index", + text: "Do not index the website on search engines and don't show it in any search results", + }, + noFollow: { + title: "No follow", + text: "Do not follow any links while indexing. Disabling this will lead to crawlers attempting to follow all links on Homarr.", + }, + noTranslate: { + title: "No translate", + text: "When the site language is likely not that the user is likely to want to read, Google will show a translation link in the search results", + }, + noSiteLinksSearchBox: { + title: "No site links search box", + text: "Google will build a search box with the crawled links along with other direct links. Enabling this will ask Google to disable that box.", + }, + }, }, }, tool: {