Merge remote-tracking branch 'origin/jh-dev' into pjt-dev

.github/workflows/deploy.yml

@@ -0,0 +1,56 @@
+name: Build and Deploy
+
+on:
+  push:
+    branches:
+      - jh-dev
+
+jobs:
+  build-and-push:
+    if: "!contains(github.event.head_commit.message, '[skip ci]')"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Read current version
+        id: read_version
+        run: |
+          VERSION=$(cat version.txt)
+          echo "Current version: $VERSION"
+          echo "VERSION=$VERSION" >> $GITHUB_ENV
+
+      - name: Increment version
+        id: increment_version
+        run: |
+          NEW_VERSION=$(awk -F. '{print $1"."($2+1)}' version.txt)
+          echo "New version: $NEW_VERSION"
+          echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV
+          echo $NEW_VERSION > version.txt
+
+      - name: Build and push frontend image
+        run: |
+          docker build -t mdv-frontend:${{ env.NEW_VERSION }} -f Dockerfile .
+          docker tag mdv-frontend:${{ env.NEW_VERSION }} ${{ secrets.DOCKER_HUB_USERNAME }}/mdv-frontend:${{ env.NEW_VERSION }}
+          docker tag mdv-frontend:${{ env.NEW_VERSION }} ${{ secrets.DOCKER_HUB_USERNAME }}/mdv-frontend:latest
+          docker push ${{ secrets.DOCKER_HUB_USERNAME }}/mdv-frontend:${{ env.NEW_VERSION }}
+          docker push ${{ secrets.DOCKER_HUB_USERNAME }}/mdv-frontend:latest
+
+      - name: Commit and push new version
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git add version.txt
+          git commit -m "Increment version to ${{ env.NEW_VERSION }} [skip ci]"
+          git push
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -20,6 +20,7 @@ dist/_avocato
 *.spec
 python/build
 python/data
+python/mdvtools/dbutils/__pycache__
 python/mdvtools/tests/temp/*
 python/mdvtools/test_projects/cache/*
 python/mdvtools/static/*

Dockerfile

@@ -1,42 +1,31 @@
-# Stage 1: Build the frontend with npm
-FROM node:14 AS frontend-builder
+# Build the frontend with npm
+FROM nikolaik/python-nodejs:python3.12-nodejs22 as frontend-builder
 
 # Set the working directory inside the container
 WORKDIR /app
 
-# Copy package.json and package-lock.json to the working directory
-COPY package.json .
-COPY package-lock.json .
+# Copy the entire project to the working directory
+COPY . .
 
 # Install npm dependencies
 RUN npm install
 
-# Copy the entire project to the working directory
-COPY . .
-
 # Run the npm build script for Flask and Vite
 RUN npm run build-flask-vite
 
-# Stage 2: Build the Python backend
-FROM python:3.10.9 AS python-builder
+# bootstrap project folder - this won't be necessary in future
+RUN mkdir -p /app/mdv/pbmc3k /app/mdv/pbmc3k_project2
 
-# Set the working directory inside the container
-WORKDIR /app
-
-# Copy everything from inside the frontend-builder /app to /app
-COPY --from=frontend-builder /app/python /app/python
+# Install HDF5 library, for some reason poetry can't install it in this context as of now
+# see https://github.com/h5py/h5py/issues/2146 for similar-ish issue
+RUN apt-get update && apt-get install -y libhdf5-dev
 
-# Set the working directory to the Python directory
+# Install Python dependencies using Poetry
 WORKDIR /app/python
-
-# Install Python dependencies
-RUN pip install -e /app/python
-
-# Set the working directory back to /app
-WORKDIR /app
+RUN poetry install --with dev,backend 
 
 # Expose the port that Flask will run on
-EXPOSE 5052
+EXPOSE 5055 
 
 # Run your Python script
-CMD ["python", "-m", "mdvtools.test_projects.scanpy_pbmc3k"]
+CMD ["poetry", "run", "python", "-m", "mdvtools.dbutils.mdv_server_app"]

Makefile

@@ -0,0 +1,19 @@
+check-docker:
+	@echo "Checking Docker daemon..."
+	@docker info >/dev/null 2>&1 && echo "Docker daemon is running" || (echo "Docker daemon is not running"; exit 1)
+
+docker-frontend-only: check-docker
+	docker-compose -f docker-compose.yml up -d --no-deps frontend
+
+docker-backend: check-docker
+	docker-compose -f docker-compose.yml up -d
+
+docker-down: check-docker
+	docker-compose -f docker-compose.yml down
+
+docker-build: check-docker
+	docker-compose -f docker-compose.yml build
+
+docker-restart-frontend: docker-down docker-frontend-only
+
+docker-restart-backend: docker-down docker-backend

docker-compose.yml

@@ -0,0 +1,28 @@
+version: '3.8'
+
+services:
+
+  frontend:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    # todo: configure dev vs dockerhub etc
+    # image: jayeshire/mdv-frontend:latest
+    ports:
+      - "5055:5055"
+    depends_on:
+      - database
+
+  database:
+    image: postgres:latest
+    environment:
+      POSTGRES_USER: admin
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: mydatabase
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+
+volumes:
+  postgres-data:

package.json

@@ -21,6 +21,7 @@
     "poetry-setup": "curl -sSL https://install.python-poetry.org | python3 -",
     "python-setup": "python -m venv venv && source venv/bin/activate && cd python && poetry install --with dev && npm run build-flask-vite",
     "mdv_desktop": "source venv/bin/activate && python -m mdvtools.mdv_desktop",
+    "docker_dev": "docker-compose up -d --force-recreate",
     "fix_npm_win": "npm config set script-shell \"cmd\""
   },
   "repository": {

python/mdvtools/dbutils/app.py

@@ -2,7 +2,7 @@
 import json
 from flask import Flask
 
-app = Flask(__name__)
+app = Flask(__name__, template_folder='../templates', static_folder='../static')
 
 try:
     config_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json')
@@ -11,8 +11,9 @@
         config = json.load(config_file)
         app.config['SQLALCHEMY_DATABASE_URI'] = config.get('database_uri', '')
         app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = config.get('track_modifications', False)
-        # app.config['UPLOAD_FOLDER'] = config.get('upload_folder', '')
-        print("Configuration loaded successfully!")
+        app.config['upload_folder'] = config.get('upload_folder', '')
+        app.config['projects_base_dir'] = config.get('projects_base_dir', '')
+        print("In app.py : Configuration loaded successfully!")
 except FileNotFoundError:
     print("Error: app.py script -> Configuration file not found.")
 except Exception as e:

python/mdvtools/dbutils/config.json

@@ -1,7 +1,6 @@
 {
-    "database_uri": "postgresql://postgres@localhost/mydatabase",
-    "track_modifications": false,
-    "upload_folder": "/path/to/upload/folder",
-    "projects_base_dir": "/Users/jayesh/mdv"
-  }
-
+  "database_uri": "postgresql://admin:password@database/mydatabase",
+  "track_modifications": false,
+  "upload_folder": "/python",
+  "projects_base_dir": "/app/mdv"
+}

python/mdvtools/dbutils/dbmodels.py

@@ -1,23 +1,80 @@
-from .app import app
+from dataclasses import dataclass, field
+from typing import Optional
+from mdvtools.dbutils.app import app
 from datetime import datetime
 from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy.orm import relationship
 
 db = SQLAlchemy(app)
 
+@dataclass
 class Project(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
+    id: int = field(init=False)
+    name: str
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
     name = db.Column(db.String(255), nullable=False, unique=True)
-    files = db.relationship('File', backref='project', lazy=True)
-    
+
+@dataclass
 class File(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
+    id: int = field(init=False)
+    name: str
+    file_path: Optional[str] = None
+    upload_timestamp: datetime = field(default_factory=datetime.now)
+    update_timestamp: datetime = field(default_factory=datetime.now, compare=False)
+    project_id: int = field(init=False)
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
     name = db.Column(db.String(255), nullable=False)
-    file_path = db.Column(db.String(255), nullable=False)
+    file_path = db.Column(db.String(255), nullable=True)
     upload_timestamp = db.Column(db.DateTime, nullable=False, default=datetime.now)
     update_timestamp = db.Column(db.DateTime, nullable=False, default=datetime.now, onupdate=datetime.now)
     project_id = db.Column(db.Integer, db.ForeignKey('project.id'), nullable=False)
 
+@dataclass
 class User(db.Model):
-    id = db.Column(db.Integer, primary_key=True)
+    id: int = field(init=False)
+    username: str
+    email: str
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
     username = db.Column(db.String(80), unique=True, nullable=False)
     email = db.Column(db.String(120), unique=True, nullable=False)
+    projects = relationship('UserProject', backref='user', lazy=True)
+
+@dataclass
+class UserProject(db.Model):
+    id: int = field(init=False)
+    user_id: int
+    project_id: int
+    can_read: bool = field(default=False)
+    can_write: bool = field(default=False)
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'), nullable=False)
+    project_id = db.Column(db.Integer, db.ForeignKey('project.id'), nullable=False)
+    can_read = db.Column(db.Boolean, nullable=False, default=False)
+    can_write = db.Column(db.Boolean, nullable=False, default=False)
+
+# Function to create default entries
+def create_default_projects():
+    default_projects = ['pbmc3k', 'pbmc3k_project2']
+    with app.app_context():
+        # Create tables if they don't exist
+        db.create_all()
+
+        # Iterate through default projects
+        for project_name in default_projects:
+            # Check if the project already exists
+            existing_project = Project.query.filter_by(name=project_name).first()
+
+            # If the project doesn't exist, add it
+            if not existing_project:
+                project = Project(name=project_name)
+                db.session.add(project)
+
+        # Commit the changes
+        db.session.commit()
+
+# Call the function to create default entries when the application starts
+create_default_projects()

python/mdvtools/dbutils/mdv_server_app.py

@@ -3,7 +3,7 @@
 from mdvtools.server import add_safe_headers
 from mdvtools.dbutils.app import app
 from mdvtools.dbutils.dbmodels import db
-from mdvtools.dbutils.routes import register_routes
+from mdvtools.dbutils.routes import register_routes, register_global_routes
 from mdvtools.dbutils.project_loader import create_all_projects
 
 
@@ -13,7 +13,7 @@ def serve_projects(projects):
 
         for p in projects:
             try:
-                print(p.name)
+                print(p.name, p.state)
                 p.serve(open_browser=False, app=app)
             except Exception as e:
                 print(f'Error serving {p.name}: {e}')
@@ -49,12 +49,14 @@ def serve_projects(projects):
     with app.app_context():
         try:
             db.create_all()
+            register_global_routes()
             projects = create_all_projects(base_dir)
+
             serve_projects(projects)
         except Exception as e:
             print(f'Error initializing app: {e}')
 
     try:
-        app.run(debug=True, port=5051)
+         app.run(host='0.0.0.0', debug=True, port=5055)
     except Exception as e:
         print(f'Error running app: {e}')