diff --git a/backend/app.js b/backend/app.js index b534867..f8041b0 100644 --- a/backend/app.js +++ b/backend/app.js @@ -20,6 +20,10 @@ const auth_config = { // auth router attaches /login, /logout, and /callback routes to the baseURL app.use(auth(auth_config)); +// Increase the limit (e.g., to 50MB) +app.use(express.json({ limit: '50mb' })); +app.use(express.urlencoded({ extended: true, limit: '50mb' })); + // Middleware to handle user persistence after Auth0 processes the callback app.use(async (req, res, next) => { if (req.oidc?.user) { @@ -71,4 +75,7 @@ const {findOne} = require("./models/user"); const User = require("./models/user"); app.use('/api/isloggedin', loggedinRouter); +const classifyRouter = require('./routes/classify.js'); +app.use('/api/classify', classifyRouter); + app.listen(port, () => console.log(`Server is running on port`, port)); diff --git a/backend/models/user.js b/backend/models/user.js index 49c6514..fae0fc7 100644 --- a/backend/models/user.js +++ b/backend/models/user.js @@ -4,7 +4,7 @@ const taskSchema = new mongoose.Schema({ title: { type: String, required: true }, description: { type: String, required: true }, streakCount: { type: Number, default: 0 }, - lastCompleted: { type: Date } + lastCompleted: { type: String } }, { _id: true }); const profileSchema = new mongoose.Schema({ diff --git a/backend/package-lock.json b/backend/package-lock.json index 5f861e7..6050dfc 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -9,17 +9,27 @@ "version": "0.0.0", "license": "ISC", "dependencies": { + "@google/generative-ai": "^0.21.0", "backend": "file:", "dotenv": "^16.4.5", "express": "^4.21.1", "express-openid-connect": "^2.17.1", "mongodb": "^6.10.0", - "mongoose": "^8.8.1" + "mongoose": "^8.8.1", + "tmp": "^0.2.3" }, "devDependencies": { "nodemon": "^3.1.7" } }, + "node_modules/@google/generative-ai": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.21.0.tgz", + "integrity": "sha512-7XhUbtnlkSEZK15kN3t+tzIMxsbKm/dSkKBFalj+20NvPKe1kBY7mR2P7vuijEn+f06z5+A8bVGKO0v39cr6Wg==", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@hapi/hoek": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", @@ -1875,6 +1885,14 @@ "node": ">=4" } }, + "node_modules/tmp": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", + "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", + "engines": { + "node": ">=14.14" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", diff --git a/backend/package.json b/backend/package.json index 4fa76d8..6c823e4 100644 --- a/backend/package.json +++ b/backend/package.json @@ -12,11 +12,13 @@ "nodemon": "^3.1.7" }, "dependencies": { + "@google/generative-ai": "^0.21.0", "backend": "file:", "dotenv": "^16.4.5", "express": "^4.21.1", "express-openid-connect": "^2.17.1", "mongodb": "^6.10.0", - "mongoose": "^8.8.1" + "mongoose": "^8.8.1", + "tmp": "^0.2.3" } } diff --git a/backend/routes/classify.js b/backend/routes/classify.js new file mode 100644 index 0000000..ec51b41 --- /dev/null +++ b/backend/routes/classify.js @@ -0,0 +1,127 @@ +const express = require("express"); +const router = express.Router(); +const User = require("../models/user"); +const {GoogleGenerativeAI} = require("@google/generative-ai"); +const {GoogleAIFileManager} = require("@google/generative-ai/server"); +const fs = require("fs"); // Import Readable stream +const tmp = require('tmp'); // Use a temporary file library + +const apiKey = process.env.GEMINI_API_KEY; +const genAI = new GoogleGenerativeAI(apiKey); +const fileManager = new GoogleAIFileManager(apiKey); + +const model = genAI.getGenerativeModel({ + model: "gemini-1.5-flash", + systemInstruction: "You will be provided with a description of what a successful completion of a goal looks like. Based on this description, your task is to carefully evaluate if the provided image depicts the user successfully completing the stated goal. Consider each aspect of the description step by step, allowing for reasonable interpretation and flexibility where applicable. Once you have reached a conclusion, respond with either task completed or task failed, ensuring your response aligns with the evidence presented in the image.", +}); + +const generationConfig = { + temperature: 1, + topP: 0.95, + topK: 40, + maxOutputTokens: 8192, + responseMimeType: "text/plain", +}; + +async function processImageWithGemini(imageDataUrl, description) { + try { + // Convert data URL to Buffer + const imageBuffer = Buffer.from(imageDataUrl.split(",")[1], "base64"); + + console.log(imageBuffer) + + // Create a temporary file + const {name: tempFilePath, fd} = tmp.fileSync({postfix: '.png'}); + + // Write the buffer to the temporary file + fs.writeFileSync(tempFilePath, imageBuffer); + + + const uploadResult = await fileManager.uploadFile(tempFilePath, { // Use the path + mimeType: "image/png", + displayName: "uploaded_image.png", + }); + + + // Delete the temporary file (important!) + fs.unlinkSync(tempFilePath); // Clean up after upload + + + const file = uploadResult.file; + + console.log("Starting Gemini chat session...") + + const chatSession = model.startChat({ + generationConfig, + history: [ + { + role: "user", + parts: [ + { + fileData: { + mimeType: file.mimeType, + fileUri: file.uri, + }, + } + ], + }, + ], + }); + + const result = await chatSession.sendMessage(description); + const resultText = result.response.text().toLowerCase(); + + console.log(resultText) + + return resultText.includes("task completed"); + } catch (error) { + console.error("Error processing image with Gemini:", error); + return true; + } +} + + +router.post("/", async (req, res) => { + const userId = req.oidc.user.sub; + + if (!req.body.taskId || !req.body.imageDataUrl) { + return res.status(400).json({error: "Task ID and image data URL are required."}); + } + + const taskId = req.body.taskId; + const imageDataUrl = req.body.imageDataUrl; + + // Find the user by their Auth0 ID + const user = await User.findOne({auth0Id: userId}); + if (!user) { + return res.status(404).json({error: "User not found."}); + } + + // Find the task by its ID + const task = user.tasks.id(taskId); + + if (!task) { + return res.status(404).json({error: "Task not found."}); + } + + // Respond immediately to prevent the frontend from hanging + res.status(202).json({message: "Image processing started."}); + + // Process the image in the background + try { + const geminiResponse = await processImageWithGemini(imageDataUrl, task.description); + + // Update the task with the Gemini result + if (geminiResponse) { + task.lastCompleted = new Date().toISOString().split("T")[0]; + task.streakCount++; + + await user.save(); + } + } catch (error) { + console.error("Error processing and storing Gemini result:", error); + } +}); + + +module.exports = router; diff --git a/backend/routes/tasks.js b/backend/routes/tasks.js index 6ec3ab9..c22500e 100644 --- a/backend/routes/tasks.js +++ b/backend/routes/tasks.js @@ -5,7 +5,6 @@ const User = require("../models/user"); router.get("/", async (req, res) => { const userId = req.oidc.user.sub; // `auth0Id` of the user - try { // Find the user by their `auth0Id` const user = await User.findOne({auth0Id: userId}); diff --git a/frontend/src/assets/logo.png b/frontend/src/assets/logo.png new file mode 100644 index 0000000..56eb58e Binary files /dev/null and b/frontend/src/assets/logo.png differ diff --git a/frontend/src/components/cameraComponent.tsx b/frontend/src/components/cameraComponent.tsx index b845e27..dd64c0b 100644 --- a/frontend/src/components/cameraComponent.tsx +++ b/frontend/src/components/cameraComponent.tsx @@ -2,7 +2,7 @@ import React, {useEffect, useRef, useState} from 'react'; import "./cameraComponent.css"; import switchCameraImage from "../assets/switch-camera.svg"; import backButtonImage from "../assets/back.svg"; -import {useNavigate} from "react-router-dom"; +import {useNavigate, useParams} from "react-router-dom"; import {toast} from "react-toastify"; const CameraComponent = () => { @@ -12,6 +12,9 @@ const CameraComponent = () => { const streamCamera = useRef(null); const [currentCamera, setCurrentCamera] = useState<'user' | 'environment'>('user'); const navigate = useNavigate(); + const [isUploading, setIsUploading] = useState(false); + + const {taskId} = useParams() const startCamera = async (facingMode: string) => { if (isSwitchingCamera.current) { @@ -54,11 +57,46 @@ const CameraComponent = () => { ctx.drawImage(video, 0, 0, canvas.width, canvas.height); const image = canvas.toDataURL('image/png'); - console.log(image) + + setIsUploading(true) // TODO send image to server + const uploadPromise = fetch("/api/classify", { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + "imageDataUrl": image, + "taskId": taskId + }) + }).then(async response => { + if (response.ok) { + const data = await response.json(); + console.log(data) + navigate('/', {state: {data}}) + } else { + const error = await response.json(); + toast("Error: " + error.message, { + type: "error" + }) + + navigate('/') + } + }).catch(error => { + toast("Error: " + error.message, { + type: "error" + }) + + navigate('/') + }).finally(() => { + setIsUploading(false) + }) - navigate('/') + toast.promise(uploadPromise, { + success: "Image uploaded successfully!", + pending: "Uploading image...", + }) } } }; diff --git a/frontend/src/components/taskCard.css b/frontend/src/components/taskCard.css index a8fc7da..1ebfbf6 100644 --- a/frontend/src/components/taskCard.css +++ b/frontend/src/components/taskCard.css @@ -1,16 +1,17 @@ .taskCard { + width: 75%; display: flex; flex-direction: row; justify-content: space-between; align-items: center; padding: 10px; - background-color: #CBDCEB; - margin-bottom: 10px; + background-color: var(--colour-4); + margin: auto; + margin-bottom: 1.75rem; border-radius: 10px; cursor: pointer; - tab-index: 0; } .taskCard__actions { @@ -18,6 +19,7 @@ flex-direction: row; justify-content: space-between; align-items: center; + padding: 3% 3% 3% 3%; } .taskCard__actions__flame { @@ -25,6 +27,16 @@ margin-right: 10px; } +.taskCard.__completed { + background-color: var(--colour-3); +} + +span { + font-weight: bold; + padding-left: 3%; + padding-right: 3%; +} + .taskCard__actions__flame--greyscale { filter: grayscale(100%); } diff --git a/frontend/src/components/taskCard.tsx b/frontend/src/components/taskCard.tsx index 82e2da9..107b284 100644 --- a/frontend/src/components/taskCard.tsx +++ b/frontend/src/components/taskCard.tsx @@ -45,7 +45,7 @@ const TaskCard = ({task}: TaskCardProps) => { } return ( -
+
{task.title}
diff --git a/frontend/src/index.css b/frontend/src/index.css index 42bef39..94dda18 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1,6 +1,20 @@ -body { +:root { + --colour-1: #F3F3E0; + --colour-2: #133E87; + --colour-3: #608BC1; + --colour-4: #CBDCEB; + } + + body { + background-color: var(--colour-1); + font-weight: bold; font-family: 'Roboto', sans-serif; margin: 0; padding: 0; - background-color: #f0f0f0; -} + } + +html { + padding: 0; + margin: 0; + } + diff --git a/frontend/src/routes/createTask.css b/frontend/src/routes/createTask.css index 6eb4eff..969af56 100644 --- a/frontend/src/routes/createTask.css +++ b/frontend/src/routes/createTask.css @@ -1,10 +1,15 @@ .createGoal { - margin: 10px; + margin: 3%; + background-color: var(); } .createTask__back_button { + margin: 1rem; + margin-left: 10%; background: none; border: none; + z-index: 1; + position: relative; } .createTask__back_button__image { @@ -12,7 +17,23 @@ } .createTask__title { - margin: 10px 0 20px 0; + margin: 2% 0 2% 0; + text-align: center; + z-index: 0; +} + +.createTask__main_content { + z-index: 0; + position: absolute; + top: 27.5%; + left: 50%; + transform: translate(-50%, -50%); + min-width: 100%; + } + +.createGoal form { + width: 75%; + margin: auto; } .createTask__form_section { @@ -21,15 +42,36 @@ margin-bottom: 20px; } -.createTask__submit { - width: 100%; +.createGoal label { + width: 75%; + margin: auto; + padding-bottom: 1rem; +} + +.createGoal #title{ + width: 75%; + margin: auto; + padding: 0.5rem; + border-radius: 0.4rem; +} - background-color: #CBDCEB; - padding: 10px; +.createTask__submit { + background-color: var(--colour-3); + display: block; + width: 65%; + color: white; + font-weight: bold; + padding: 1rem; + margin: auto; border: none; border-radius: 10px; } .createTask__form_section textarea { resize: vertical; + min-height: 5rem; + width: 75%; + margin: auto; + padding: 0.5rem; + border-radius: 0.3rem; } diff --git a/frontend/src/routes/createTask.tsx b/frontend/src/routes/createTask.tsx index 5ed94db..7d2f154 100644 --- a/frontend/src/routes/createTask.tsx +++ b/frontend/src/routes/createTask.tsx @@ -41,8 +41,8 @@ const CreateTask = () => { toast("Goal created successfully! 🎉", {type: "success"}); navigate('/'); }).catch((error) => { - console.error('Error:', error); - }) + console.error('Error:', error); + }) }; return ( @@ -50,34 +50,36 @@ const CreateTask = () => { -

Create Goal

+
+

Create Goal

-
-
- - setTitle(e.target.value)} - placeholder="Go to the gym" - required - /> -
-
- -