feat: add classes for task, task manager, and celery tasks (#2)

.pre-commit-config.yaml

@@ -8,17 +8,19 @@ repos:
 
   - repo: local
     hooks:
-      # NOTE: This is a total unnecessary check, just used as part of the
-      #       template. Remove this after creating the template.
-      - id: check-python
-        name: check-python
-        entry: python --version
+      - id: ruff format
+        name: ruff format
+        entry: ruff format
         language: system
-        pass_filenames: no
+        pass_filenames: true
+        require_serial: yes
+        files: "./"
+        types:
+          - python
 
-      - id: ruff
-        name: ruff
-        entry: ruff --fix
+      - id: ruff check
+        name: ruff check
+        entry: ruff check
         language: system
         pass_filenames: true
         require_serial: yes
@@ -28,12 +30,9 @@ repos:
 
       - id: mypy
         name: mypy
-        entry: mypy
+        entry: mypy .
         language: system
-        files: "src/retsu"
-        pass_filenames: true
-        types:
-          - python
+        pass_filenames: false
 
       - id: bandit
         name: bandit

.sugar.yaml

@@ -0,0 +1,16 @@
+version: 1.0
+compose-app: docker compose
+# env-file: .env
+defaults:
+  group: dev
+  project-name: retsu
+groups:
+  dev:
+    compose-path:
+      - containers/compose.yaml
+    # env-file: .env
+    services:
+      default: valkey #,celery
+      available:
+        - name: valkey
+        # - name: celery

LICENSE

@@ -1,4 +1,4 @@
-BSD License
+BSD 3-Clause License
 
 Copyright (c) 2024, Ivan Ogasawara
 All rights reserved.

containers/celery/Dockerfile

@@ -0,0 +1,89 @@
+# ref: https://github.com/mamba-org/micromamba-docker/blob/main/Dockerfile
+
+FROM condaforge/mambaforge:24.3.0-0
+
+LABEL maintainer="Ivan Ogasawara <[email protected]>"
+LABEL org.opencontainers.image.title="Retsu"
+LABEL org.opencontainers.image.authors="Retsu Team"
+LABEL org.opencontainers.image.source="https://github.com/osl-incubator/retsu"
+LABEL org.opencontainers.image.version="latest"
+LABEL org.opencontainers.image.description="Retsu"
+LABEL org.thegraphnetwork.config.version="latest"
+
+# it is the default, but using it here to have it explicitly
+USER root
+
+SHELL ["/bin/bash", "-c"]
+# Use bash in Dockerfile RUN commands and make sure bashrc is sourced when
+# executing commands with /bin/bash -c
+# Needed to have the micromamba activate command configured etc.
+
+ENV ENV_NAME=retsu
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+ARG UID=1000
+ARG GID=1000
+
+RUN apt-get update -y \
+  && apt-get install -y \
+    apt-utils \
+    build-essential \
+    curl \
+    tini \
+    sudo \
+    tzdata \
+    gcc-multilib \
+    g++-multilib \
+    openssl \
+  && rm -rf /var/lib/apt/lists/* \
+    /var/cache/apt/archives \
+    /tmp/*
+
+RUN addgroup --gid ${GID} retsu \
+  && useradd --uid ${UID} --gid ${GID} -ms /bin/bash retsu \
+  && mkdir -p /opt/services/retsu /opt/data/retsu \
+  && chmod -R a+rwx /opt/conda /opt/services \
+  && export ENV_NAME="$ENV_NAME" \
+  && chown -R retsu:retsu /opt/services /opt/data \
+  && echo "retsu ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/retsu \
+  && chmod 0440 /etc/sudoers.d/retsu
+
+USER retsu
+
+WORKDIR /opt/services/retsu
+
+COPY --chown=retsu:retsu ./conda/ /tmp/conda
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+
+RUN mamba env create -n $ENV_NAME --file /tmp/conda/dev.yaml \
+  && conda clean --all \
+  && find /opt/conda/ -type f,l -name '*.pyc' -delete \
+  && find /opt/conda/ -type f,l -name '*.js.map' -delete \
+  && rm -rf /opt/conda/pkgs /tmp/*
+
+ENV CONDA_PREFIX /opt/conda/envs/$ENV_NAME
+ENV PATH ${CONDA_PREFIX}/bin:$PATH
+
+# install dependencies
+COPY --chown=retsu:retsu pyproject.toml poetry.lock /tmp/
+
+ARG ENV=prod
+
+RUN cd /tmp && poetry install --no-root
+
+COPY --chown=retsu:retsu containers/celery/scripts/entrypoint.sh /opt/entrypoint.sh
+COPY --chown=retsu:retsu . /opt/services/retsu/
+
+RUN chmod +x /opt/entrypoint.sh \
+  && echo "source /opt/entrypoint.sh" > ~/.bashrc
+
+WORKDIR /opt/services/retsu/
+
+ENV PYTHONPATH='/opt/services/retsu/'
+
+RUN poetry install
+
+ENTRYPOINT ["tini", "--", "/opt/entrypoint.sh"]
+CMD ["python", "example/app.py"]

containers/celery/scripts/entrypoint.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# prepare the conda environment
+is_conda_in_path=$(echo $PATH|grep -m 1 --count /opt/conda/)
+
+if [ $is_conda_in_path == 0 ]; then
+  export PATH="/opt/conda/condabin:/opt/conda/bin:$PATH"
+  echo "[II] included conda to the PATH"
+fi
+
+echo "[II] activate environment"
+source activate retsu
+
+# created by docker build
+poetry install
+
+set +ex
+
+if [ $# -ne 0 ]
+  then
+    echo "Running: ${@}"
+    ${@}
+fi

containers/compose.yaml

@@ -0,0 +1,19 @@
+version: '3.9'
+
+services:
+  valkey:
+    image: valkey/valkey:7.2.5-alpine
+    hostname: valkey
+    container_name: valkey
+    ports:
+      - 6379:6379
+
+  # celery:
+  #   hostname: celery
+  #   container_name: celery
+  #   build:
+  #     context: ..
+  #     dockerfile: containers/celery/Dockerfile
+  #   command: celery --app=config worker --loglevel=INFO  --concurrency=4
+  #   depends_on:
+  #     - valkey

example/.gitignore

@@ -0,0 +1 @@
+results/*

example/__init__.py

@@ -0,0 +1 @@
+"""Example of usage of retsu."""

example/app.py

@@ -0,0 +1,103 @@
+"""Example of usage retsu with a flask app."""
+
+import os
+import signal
+
+from typing import Optional
+
+from flask import Flask
+from tasks import MyTaskManager
+
+app = Flask(__name__)
+
+task_manager = MyTaskManager()
+task_manager.start()
+
+
+def signal_handler(signum: int, frame: Optional[int]) -> None:
+    """Define signal handler."""
+    print(f"Received signal {signum}, shutting down...")
+    try:
+        task_manager.stop()
+    except Exception:
+        ...
+    # Perform any other cleanup here if necessary
+    os._exit(0)
+
+
+# Register the signal handler
+signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
+
+
+@app.route("/")
+def api() -> str:
+    """Define the root endpoint."""
+    menu = """
+    Select an endpoint for your request:
+
+    * serial
+    * parallel
+    * status
+    * result
+    """
+
+    return menu
+
+
+@app.route("/serial/<int:a>/<int:b>")
+def serial(a: int, b: int) -> str:
+    """Define the serial endpoint."""
+    task1 = task_manager.get_task("serial")
+    key = task1.request(a=a, b=b)
+    return f"your task ({key}) is running now, please wait until it is done."
+
+
+@app.route("/parallel/<int:a>/<int:b>")
+def parallel(a: int, b: int) -> str:
+    """Define the parallel endpoint."""
+    task2 = task_manager.get_task("parallel")
+    key = task2.request(a=a, b=b)
+    return f"your task ({key}) is running now, please wait until it is done."
+
+
+@app.route("/serial/status/<string:task_id>")
+def serial_status(task_id: str) -> str:
+    """Define serial/status endpoint."""
+    task1 = task_manager.get_task("serial")
+    _status = task1.status(task_id)
+    return {"status": _status, "task_id": task_id}
+
+
+@app.route("/parallel/status/<string:task_id>")
+def parallel_status(task_id: str) -> str:
+    """Define parallel/status endpoint."""
+    task2 = task_manager.get_task("parallel")
+    _status = task2.status(task_id)
+    return {"status": _status, "task_id": task_id}
+
+
+@app.route("/serial/result/<string:task_id>")
+def serial_result(task_id: str) -> str:
+    """Define serial/result endpoint."""
+    task1 = task_manager.get_task("serial")
+    return task1.get_result(task_id)
+
+
+@app.route("/parallel/result/<string:task_id>")
+def parallel_result(task_id: str) -> str:
+    """Define parallel/result endpoint."""
+    task2 = task_manager.get_task("parallel")
+    return task2.get_result(task_id)
+
+
+if __name__ == "__main__":
+    try:
+        app.run(
+            debug=True,
+            passthrough_errors=True,
+            use_debugger=True,
+            use_reloader=False,
+        )
+    except KeyboardInterrupt:
+        signal_handler(signal.SIGINT, None)

example/settings.py

@@ -0,0 +1,8 @@
+"""Settings used by the example."""
+
+import os
+
+from pathlib import Path
+
+RESULTS_PATH = Path(__file__).parent / "results"
+os.makedirs(RESULTS_PATH, exist_ok=True)