diff --git a/.github/workflows/update-indexes.yml b/.github/workflows/update-indexes.yml new file mode 100644 index 0000000..57a97bc --- /dev/null +++ b/.github/workflows/update-indexes.yml @@ -0,0 +1,18 @@ +name: Update Meilisearch Indexes for All Users + +# This workflow updates the settings for all indexes for all users. + +on: workflow_dispatch + +jobs: + update-indexes: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Update Meilisearch Indexes + run: ./scripts/update-indexes.sh + env: + MEILISEARCH_HOST: ${{ secrets.MEILISEARCH_HOST }} + MEILISEARCH_MASTER_KEY: ${{ secrets.MEILISEARCH_MASTER_KEY }} diff --git a/apps/api/package.json b/apps/api/package.json index ea5e0f6..7790b88 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -37,6 +37,7 @@ "meilisearch": "^0.40.0", "multer": "1.4.5-lts.1", "oslo": "^1.1.2", + "ts-fsrs": "^3.5.7", "tsx": "^4.7.1", "zod": "^3.22.4" }, diff --git a/apps/api/src/mail.ts b/apps/api/src/clients/mail.ts similarity index 100% rename from apps/api/src/mail.ts rename to apps/api/src/clients/mail.ts diff --git a/apps/api/src/clients/meilisearch.ts b/apps/api/src/clients/meilisearch.ts new file mode 100644 index 0000000..bec18e1 --- /dev/null +++ b/apps/api/src/clients/meilisearch.ts @@ -0,0 +1,27 @@ +import { MeiliSearch } from "meilisearch"; + +export const meilisearchClient = new MeiliSearch({ + host: process.env.MEILISEARCH_HOST!, + apiKey: process.env.MEILISEARCH_API_KEY!, +}); + +/** + * Creates a new index for a user and configures it + * with the necessary universal settings. + * + * @param userId The user's id. + */ +export const createUserIndex = async (userId: string) => { + const { taskUid: createTaskUid } = + await meilisearchClient.createIndex(userId); + + await meilisearchClient.waitForTask(createTaskUid); + + const userIndex = meilisearchClient.index(userId); + + const { taskUid: updateTaskUid } = await userIndex.updateSettings({ + filterableAttributes: ["flashcard.due_timestamp"], + }); + + await meilisearchClient.waitForTask(updateTaskUid); +}; diff --git a/apps/api/src/redis.ts b/apps/api/src/clients/redis.ts similarity index 100% rename from apps/api/src/redis.ts rename to apps/api/src/clients/redis.ts diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 2db5331..79d3421 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -9,6 +9,7 @@ import cors from "cors"; import { userRouter } from "./routers/user"; import { authRouter } from "./routers/auth/github"; import { dictionaryRouter } from "./routers/dictionary"; +import { flashcardRouter } from "./routers/flashcard"; import { trpcAuthRouter } from "./routers/auth"; import { csrf } from "./middleware"; import { getAllowedDomains } from "./utils"; @@ -31,6 +32,7 @@ const port = process.env.PORT; const appRouter = router({ user: userRouter, auth: trpcAuthRouter, + flashcard: flashcardRouter, }); const app = express(); diff --git a/apps/api/src/meilisearch.ts b/apps/api/src/meilisearch.ts deleted file mode 100644 index f254dbc..0000000 --- a/apps/api/src/meilisearch.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { MeiliSearch } from "meilisearch"; - -export const meilisearchClient = new MeiliSearch({ - host: process.env.MEILISEARCH_HOST!, - apiKey: process.env.MEILISEARCH_API_KEY!, -}); diff --git a/apps/api/src/routers/auth/github.ts b/apps/api/src/routers/auth/github.ts index 95ef587..b319b6f 100644 --- a/apps/api/src/routers/auth/github.ts +++ b/apps/api/src/routers/auth/github.ts @@ -6,7 +6,7 @@ import { db } from "../../db"; import { users } from "../../db/schema/users"; import { eq, or } from "drizzle-orm"; import { generateIdFromEntropySize } from "lucia"; -import { meilisearchClient } from "../../meilisearch"; +import { createUserIndex } from "../../clients/meilisearch"; interface GitHubUser { id: number; @@ -99,9 +99,7 @@ authRouter.get("/login/github/callback", async (req, res) => { email: githubUser.email, }); - const { taskUid } = await meilisearchClient.createIndex(userId); - - await meilisearchClient.waitForTask(taskUid); + await createUserIndex(userId); const session = await lucia.createSession(userId, {} as any); diff --git a/apps/api/src/routers/auth/index.ts b/apps/api/src/routers/auth/index.ts index 39d964f..94096ce 100644 --- a/apps/api/src/routers/auth/index.ts +++ b/apps/api/src/routers/auth/index.ts @@ -12,11 +12,11 @@ import { } from "../../trpc"; import { z } from "zod"; import { TRPCError } from "@trpc/server"; -import { sendMail } from "../../mail"; -import { redisClient } from "../../redis"; +import { sendMail } from "../../clients/mail"; +import { redisClient } from "../../clients/redis"; import { base64 } from "oslo/encoding"; import { OTP_VALID_PERIOD, generateOTP, verifyOTP } from "../../otp/totp"; -import { meilisearchClient } from "../../meilisearch"; +import { createUserIndex } from "../../clients/meilisearch"; /** * A buffer added to the redis ttl for otp @@ -253,9 +253,7 @@ export const trpcAuthRouter = trpcRouter({ email, }); - const { taskUid } = await meilisearchClient.createIndex(userId); - - await meilisearchClient.waitForTask(taskUid); + await createUserIndex(userId); const session = await lucia.createSession(userId, {} as any); diff --git a/apps/api/src/routers/dictionary.ts b/apps/api/src/routers/dictionary.ts index 8bfd5fd..1340f1b 100644 --- a/apps/api/src/routers/dictionary.ts +++ b/apps/api/src/routers/dictionary.ts @@ -8,7 +8,7 @@ import Ajv from "ajv"; import multer from "multer"; import schema from "../schema.json"; -import { meilisearchClient } from "../meilisearch"; +import { meilisearchClient } from "../clients/meilisearch"; import { auth } from "../middleware"; import { ErrorCode, MeilisearchError } from "../error"; diff --git a/apps/api/src/routers/flashcard.ts b/apps/api/src/routers/flashcard.ts new file mode 100644 index 0000000..575e26a --- /dev/null +++ b/apps/api/src/routers/flashcard.ts @@ -0,0 +1,109 @@ +import { router, protectedProcedure } from "../trpc"; +import { meilisearchClient } from "../clients/meilisearch"; +import { Card, createEmptyCard } from "ts-fsrs"; +import { z } from "zod"; + +type Flashcard = Card & { + id: string; + due: string; + last_review: string; + due_timestamp: number; + last_review_timestamp: number; +}; + +const FlashcardSchema = z.object({ + id: z.string(), + elapsed_days: z.number(), + lapses: z.number(), + reps: z.number(), + scheduled_days: z.number(), + stability: z.number(), + state: z.number(), + difficulty: z.number(), + due: z.string(), + due_timestamp: z.number(), + last_review: z.string().nullable(), + last_review_timestamp: z.number().nullable(), +}); + +export const flashcardRouter = router({ + today: protectedProcedure + .output( + z.object({ + flashcards: z.array(FlashcardSchema), + }), + ) + .query(async ({ ctx }) => { + const { user } = ctx; + + /** + * Current timestamp in seconds + */ + // const now = Math.floor(new Date().getTime() / 1000); + + const { results } = await meilisearchClient.index(user.id).getDocuments({ + filter: [ + `flashcard.due_timestamp EXISTS`, + // `flashcard.due_timestamp NOT EXISTS OR flashcard.due_timestamp <= ${now}`, + ], + }); + + const flashcards: Flashcard[] = results.map(({ id, flashcard }) => { + const card = flashcard ?? getEmptyFlashcard(id); + + return { + id, + ...card, + }; + }); + + return { + flashcards, + }; + }), + + update: protectedProcedure + .input(FlashcardSchema) + .mutation(async ({ ctx, input }) => { + const { user } = ctx; + + const { id, ...flashcard } = input; + + const { taskUid } = await meilisearchClient + .index(user.id) + .updateDocuments([ + { + id, + flashcard, + }, + ]); + + await meilisearchClient.index(user.id).waitForTask(taskUid); + + return { + id, + ...flashcard, + }; + }), +}); + +const getEmptyFlashcard = (id: string): Flashcard => { + return createEmptyCard(new Date(), (c) => { + const due = new Date(c.due); + const last_review = c.last_review ? new Date(c.last_review) : null; + + const due_timestamp = Math.floor(due.getTime() / 1000); + const last_review_timestamp = last_review + ? Math.floor(last_review.getTime() / 1000) + : null; + + return { + ...c, + id, + due: due.toISOString(), + last_review: last_review?.toISOString() ?? null, + due_timestamp, + last_review_timestamp, + } as Flashcard; + }); +}; diff --git a/apps/api/src/schema.json b/apps/api/src/schema.json index 8af5bed..4674c62 100644 --- a/apps/api/src/schema.json +++ b/apps/api/src/schema.json @@ -152,6 +152,80 @@ } } } + }, + "flashcard": { + "type": "object", + "description": "Properties used for scheduling the flashcards associated with the word using FSRS. When adding new words, just set these to the default values. For updating existing words, leave them as-is to keep flashcard progress or set to default values to reset it.", + "properties": { + "difficulty": { + "type": "number", + "minimum": 0, + "default": 0 + }, + "due": { + "type": "string", + "format": "date-time", + "description": "The date when the card needs to be reviewed. Set to the current time for new cards." + }, + "due_timestamp": { + "type": "integer", + "minimum": 0, + "description": "A UNIX timestamp in seconds that represents the same date as `due`." + }, + "elapsed_days": { + "type": "integer", + "minimum": 0, + "default": 0 + }, + "lapses": { + "type": "integer", + "minimum": 0, + "default": 0 + }, + "last_review": { + "type": "string", + "format": "date-time", + "default": null + }, + "last_review_timestamp": { + "type": "integer", + "minimum": 0, + "default": null, + "description": "A UNIX timestamp in seconds that represents the same date as `last_review`, if present." + }, + "reps": { + "type": "integer", + "minimum": 0, + "default": 0 + }, + "scheduled_days": { + "type": "integer", + "minimum": 0, + "default": 0 + }, + "stability": { + "type": "number", + "minimum": 0, + "default": 0 + }, + "state": { + "type": "integer", + "enum": [0, 1, 2, 3], + "description": "0 is New, 1 is Learning, 2 is Review, and 3 is Relearning.", + "default": 0 + } + }, + "required": [ + "difficulty", + "due", + "due_timestamp", + "elapsed_days", + "lapses", + "reps", + "scheduled_days", + "stability", + "state" + ] } }, "required": ["id", "word", "translation"] diff --git a/apps/api/src/trpc.ts b/apps/api/src/trpc.ts index b07a8f1..afe2147 100644 --- a/apps/api/src/trpc.ts +++ b/apps/api/src/trpc.ts @@ -2,7 +2,7 @@ import { TRPCError, initTRPC } from "@trpc/server"; import * as trpcExpress from "@trpc/server/adapters/express"; import { validateRequest } from "./auth"; import { Ratelimit } from "@upstash/ratelimit"; -import { redisClient } from "./redis"; +import { redisClient } from "./clients/redis"; // Created for each request export const createContext = async ({ diff --git a/apps/web/package.json b/apps/web/package.json index c797a0b..4af6ee6 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -48,6 +48,7 @@ "react-instantsearch": "^7.7.0", "tailwind-merge": "^2.2.1", "tailwindcss-animate": "^1.0.7", + "ts-fsrs": "^3.5.7", "zod": "^3.22.4" }, "devDependencies": { diff --git a/apps/web/src/routes/_search-layout/index.lazy.tsx b/apps/web/src/routes/_search-layout/index.lazy.tsx index aabe6b9..9e9ac96 100644 --- a/apps/web/src/routes/_search-layout/index.lazy.tsx +++ b/apps/web/src/routes/_search-layout/index.lazy.tsx @@ -1,4 +1,13 @@ import { createLazyFileRoute } from "@tanstack/react-router"; +import { + createEmptyCard, + formatDate, + fsrs, + generatorParameters, + Rating, + Grades, + State, +} from "ts-fsrs"; import { Card, CardContent, @@ -13,15 +22,42 @@ import { useWindowScroll, useWindowSize } from "@uidotdev/usehooks"; import { Trans } from "@lingui/macro"; import { cn } from "@/lib/utils"; import { Page } from "@/components/Page"; +import { useEffect } from "react"; +import { trpc } from "@/lib/trpc"; const Index = () => { const [{ y }, scrollTo] = useWindowScroll(); const { height } = useWindowSize(); + const { mutate } = trpc.flashcard.update.useMutation(); + const { data } = trpc.flashcard.today.useQuery(); // Check that the window dimensions are available const hasLoadedHeight = height !== null && height > 0 && y !== null; const hasScrolledPastInitialView = hasLoadedHeight ? y > height : false; + useEffect(() => { + if (!data) return; + + const f = fsrs(); + const cards = data?.flashcards ?? []; + const now = new Date(); + const scheduling_cards = f.repeat(cards[0], now); + + const ratedCard = scheduling_cards[Rating.Easy].card; + const card = { + ...ratedCard, + id: cards[0].id, + due: ratedCard.due.toISOString(), + last_review: ratedCard?.last_review?.toISOString() ?? null, + due_timestamp: Math.floor(ratedCard.due.getTime() / 1000), + last_review_timestamp: ratedCard?.last_review + ? Math.floor(ratedCard.last_review.getTime() / 1000) + : null, + }; + + // mutate(card); + }, [data]); + return ( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index af1fa17..6243bf5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -80,6 +80,9 @@ importers: oslo: specifier: ^1.1.2 version: 1.1.2 + ts-fsrs: + specifier: ^3.5.7 + version: 3.5.7 tsx: specifier: ^4.7.1 version: 4.7.1 @@ -237,6 +240,9 @@ importers: tailwindcss-animate: specifier: ^1.0.7 version: 1.0.7(tailwindcss@3.4.1) + ts-fsrs: + specifier: ^3.5.7 + version: 3.5.7 zod: specifier: ^3.22.4 version: 3.22.4 @@ -8414,6 +8420,10 @@ packages: resolution: {integrity: sha512-Orrsjf9trHHxFRuo9/rzm0KIWmgzE8RMlZMzuhZOJ01Rnz3D0YBAe+V6473t6/H6c7irs6Lt48brULAiRWb3Vw==} dev: false + /seedrandom@3.0.5: + resolution: {integrity: sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==} + dev: false + /semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} hasBin: true @@ -8862,6 +8872,13 @@ packages: typescript: 5.3.3 dev: true + /ts-fsrs@3.5.7: + resolution: {integrity: sha512-xA18Igi1adkuCNgNjKkorjYgZq842+HdgRjatQPDERcrAWMlzM+3BwIbrF/Q0ijYEyw3NcTp3yJIMSO7agBLKQ==} + engines: {node: '>=16.0.0'} + dependencies: + seedrandom: 3.0.5 + dev: false + /ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} diff --git a/scripts/update-indexes.sh b/scripts/update-indexes.sh new file mode 100755 index 0000000..3f5e267 --- /dev/null +++ b/scripts/update-indexes.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e + +if [ -z "$MEILISEARCH_HOST" ] || [ -z "$MEILISEARCH_MASTER_KEY" ]; then + echo "Please set the MEILISEARCH_HOST and MEILISEARCH_MASTER_KEY environment variables." + exit 1 +fi + +FILTERABLE_ATTRIBUTES='["flashcard.due_timestamp"]' + +echo "Updating indexes for all users..." + +# Fetch all indexes +INDEXES=$(curl -s -X GET "$MEILISEARCH_HOST/indexes?limit=1000" -H "Authorization: Bearer $MEILISEARCH_MASTER_KEY" | jq -r '.results[].uid') + +echo "Found indexes: " +echo [$INDEXES] + +for index in $INDEXES; do + echo "Updating filterable attributes for index: $index" + # Update filterable attributes + response=$(curl \ + -X PUT "$MEILISEARCH_HOST/indexes/$index/settings/filterable-attributes" \ + -H "Authorization: Bearer $MEILISEARCH_MASTER_KEY" \ + -H "Content-Type: application/json" \ + -s \ + --data-binary "$FILTERABLE_ATTRIBUTES") + + if [[ $response =~ "code" && $response =~ "type" ]]; then + echo $response | jq + echo "Error updating filterable attributes for index: $index" + exit 1 + fi +done + +echo "Successfully updated indexes for all users."