Merge branch 'main' into api

backend/app.js

@@ -20,6 +20,10 @@ const auth_config = {
 // auth router attaches /login, /logout, and /callback routes to the baseURL
 app.use(auth(auth_config));
 
+// Increase the limit (e.g., to 50MB)
+app.use(express.json({ limit: '50mb' }));
+app.use(express.urlencoded({ extended: true, limit: '50mb' }));
+
 // Middleware to handle user persistence after Auth0 processes the callback
 app.use(async (req, res, next) => {
     if (req.oidc?.user) {
@@ -71,4 +75,7 @@ const {findOne} = require("./models/user");
 const User = require("./models/user");
 app.use('/api/isloggedin', loggedinRouter);
 
+const classifyRouter = require('./routes/classify.js');
+app.use('/api/classify', classifyRouter);
+
 app.listen(port, () => console.log(`Server is running on port`, port));

backend/models/user.js

@@ -4,7 +4,7 @@ const taskSchema = new mongoose.Schema({
   title: { type: String, required: true },
   description: { type: String, required: true },
   streakCount: { type: Number, default: 0 },
-  lastCompleted: { type: Date }
+  lastCompleted: { type: String }
 }, { _id: true });
 
 const profileSchema = new mongoose.Schema({

backend/package.json

@@ -12,11 +12,13 @@
     "nodemon": "^3.1.7"
   },
   "dependencies": {
+    "@google/generative-ai": "^0.21.0",
     "backend": "file:",
     "dotenv": "^16.4.5",
     "express": "^4.21.1",
     "express-openid-connect": "^2.17.1",
     "mongodb": "^6.10.0",
-    "mongoose": "^8.8.1"
+    "mongoose": "^8.8.1",
+    "tmp": "^0.2.3"
   }
 }

backend/routes/classify.js

@@ -0,0 +1,127 @@
+const express = require("express");
+const router = express.Router();
+const User = require("../models/user");
+const {GoogleGenerativeAI} = require("@google/generative-ai");
+const {GoogleAIFileManager} = require("@google/generative-ai/server");
+const fs = require("fs"); // Import Readable stream
+const tmp = require('tmp');  // Use a temporary file library
+
+const apiKey = process.env.GEMINI_API_KEY;
+const genAI = new GoogleGenerativeAI(apiKey);
+const fileManager = new GoogleAIFileManager(apiKey);
+
+const model = genAI.getGenerativeModel({
+    model: "gemini-1.5-flash",
+    systemInstruction: "You will be provided with a description of what a successful completion of a goal looks like. Based on this description, your task is to carefully evaluate if the provided image depicts the user successfully completing the stated goal. Consider each aspect of the description step by step, allowing for reasonable interpretation and flexibility where applicable. Once you have reached a conclusion, respond with either <response>task completed</response> or <response>task failed</response>, ensuring your response aligns with the evidence presented in the image.",
+});
+
+const generationConfig = {
+    temperature: 1,
+    topP: 0.95,
+    topK: 40,
+    maxOutputTokens: 8192,
+    responseMimeType: "text/plain",
+};
+
+async function processImageWithGemini(imageDataUrl, description) {
+    try {
+        // Convert data URL to Buffer
+        const imageBuffer = Buffer.from(imageDataUrl.split(",")[1], "base64");
+
+        console.log(imageBuffer)
+
+        // Create a temporary file
+        const {name: tempFilePath, fd} = tmp.fileSync({postfix: '.png'});
+
+        // Write the buffer to the temporary file
+        fs.writeFileSync(tempFilePath, imageBuffer);
+
+
+        const uploadResult = await fileManager.uploadFile(tempFilePath, {  // Use the path
+            mimeType: "image/png",
+            displayName: "uploaded_image.png",
+        });
+
+
+        // Delete the temporary file (important!)
+        fs.unlinkSync(tempFilePath); // Clean up after upload
+
+
+        const file = uploadResult.file;
+
+        console.log("Starting Gemini chat session...")
+
+        const chatSession = model.startChat({
+            generationConfig,
+            history: [
+                {
+                    role: "user",
+                    parts: [
+                        {
+                            fileData: {
+                                mimeType: file.mimeType,
+                                fileUri: file.uri,
+                            },
+                        }
+                    ],
+                },
+            ],
+        });
+
+        const result = await chatSession.sendMessage(description);
+        const resultText = result.response.text().toLowerCase();
+
+        console.log(resultText)
+
+        return resultText.includes("<response>task completed</response>");
+    } catch (error) {
+        console.error("Error processing image with Gemini:", error);
+        return true;
+    }
+}
+
+
+router.post("/", async (req, res) => {
+    const userId = req.oidc.user.sub;
+
+    if (!req.body.taskId || !req.body.imageDataUrl) {
+        return res.status(400).json({error: "Task ID and image data URL are required."});
+    }
+
+    const taskId = req.body.taskId;
+    const imageDataUrl = req.body.imageDataUrl;
+
+    // Find the user by their Auth0 ID
+    const user = await User.findOne({auth0Id: userId});
+    if (!user) {
+        return res.status(404).json({error: "User not found."});
+    }
+
+    // Find the task by its ID
+    const task = user.tasks.id(taskId);
+
+    if (!task) {
+        return res.status(404).json({error: "Task not found."});
+    }
+
+    // Respond immediately to prevent the frontend from hanging
+    res.status(202).json({message: "Image processing started."});
+
+    // Process the image in the background
+    try {
+        const geminiResponse = await processImageWithGemini(imageDataUrl, task.description);
+
+        // Update the task with the Gemini result
+        if (geminiResponse) {
+            task.lastCompleted = new Date().toISOString().split("T")[0];
+            task.streakCount++;
+
+            await user.save();
+        }
+    } catch (error) {
+        console.error("Error processing and storing Gemini result:", error);
+    }
+});
+
+
+module.exports = router;

backend/routes/tasks.js

@@ -5,7 +5,6 @@ const User = require("../models/user");
 router.get("/", async (req, res) => {
     const userId = req.oidc.user.sub; // `auth0Id` of the user
 
-
     try {
         // Find the user by their `auth0Id`
         const user = await User.findOne({auth0Id: userId});

frontend/src/components/cameraComponent.tsx

@@ -2,7 +2,7 @@ import React, {useEffect, useRef, useState} from 'react';
 import "./cameraComponent.css";
 import switchCameraImage from "../assets/switch-camera.svg";
 import backButtonImage from "../assets/back.svg";
-import {useNavigate} from "react-router-dom";
+import {useNavigate, useParams} from "react-router-dom";
 import {toast} from "react-toastify";
 
 const CameraComponent = () => {
@@ -12,6 +12,9 @@ const CameraComponent = () => {
   const streamCamera = useRef<string | null>(null);
   const [currentCamera, setCurrentCamera] = useState<'user' | 'environment'>('user');
   const navigate = useNavigate();
+  const [isUploading, setIsUploading] = useState<boolean>(false);
+
+  const {taskId} = useParams()
 
   const startCamera = async (facingMode: string) => {
     if (isSwitchingCamera.current) {
@@ -54,11 +57,46 @@ const CameraComponent = () => {
         ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
 
         const image = canvas.toDataURL('image/png');
-        console.log(image)
+
+        setIsUploading(true)
 
         // TODO send image to server
+        const uploadPromise = fetch("/api/classify", {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json'
+          },
+          body: JSON.stringify({
+            "imageDataUrl": image,
+            "taskId": taskId
+          })
+        }).then(async response => {
+          if (response.ok) {
+            const data = await response.json();
+            console.log(data)
+            navigate('/', {state: {data}})
+          } else {
+            const error = await response.json();
+            toast("Error: " + error.message, {
+              type: "error"
+            })
+
+            navigate('/')
+          }
+        }).catch(error => {
+          toast("Error: " + error.message, {
+            type: "error"
+          })
+
+          navigate('/')
+        }).finally(() => {
+          setIsUploading(false)
+        })
 
-        navigate('/')
+        toast.promise(uploadPromise, {
+          success: "Image uploaded successfully!",
+          pending: "Uploading image...",
+        })
       }
     }
   };

frontend/src/components/taskCard.css

@@ -1,30 +1,42 @@
 .taskCard {
+    width: 75%;
     display: flex;
     flex-direction: row;
     justify-content: space-between;
     align-items: center;
     padding: 10px;
 
-    background-color: #CBDCEB;
-    margin-bottom: 10px;
+    background-color: var(--colour-4);
+    margin: auto;
+    margin-bottom: 1.75rem;
     border-radius: 10px;
 
     cursor: pointer;
-    tab-index: 0;
 }
 
 .taskCard__actions {
     display: flex;
     flex-direction: row;
     justify-content: space-between;
     align-items: center;
+    padding: 3% 3% 3% 3%;
 }
 
 .taskCard__actions__flame {
     height: 30px;
     margin-right: 10px;
 }
 
+.taskCard.__completed {
+    background-color: var(--colour-3);
+}
+
+span {
+    font-weight: bold;
+    padding-left: 3%;
+    padding-right: 3%;
+}
+
 .taskCard__actions__flame--greyscale {
     filter: grayscale(100%);
 }

frontend/src/components/taskCard.tsx

@@ -45,7 +45,7 @@ const TaskCard = ({task}: TaskCardProps) => {
   }
 
   return (
-    <div className="taskCard" onClick={handleTaskCardClick}>
+    <div className={`taskCard ${hasCompletedToday ? '__completed' : ''}`} onClick={handleTaskCardClick}>
       <span>{task.title}</span>
 
       <div className="taskCard__actions">

frontend/src/index.css

@@ -1,6 +1,20 @@
-body {
+:root {
+    --colour-1: #F3F3E0;
+    --colour-2: #133E87;
+    --colour-3: #608BC1;
+    --colour-4: #CBDCEB;
+  }
+
+  body {
+    background-color: var(--colour-1);
+    font-weight: bold;
     font-family: 'Roboto', sans-serif;
     margin: 0;
     padding: 0;
-    background-color: #f0f0f0;
-}
+  }
+
+html {
+    padding: 0;
+    margin: 0;
+  }
+