Improve NEBULA installation

.gitignore

@@ -118,9 +118,9 @@ venv.bak/
 # Rope project settings
 .ropeproject
 
-# poetry
-.poetry/
-poetry.lock
+# uv
+.venv/
+uv.lock
 
 # mkdocs documentation
 /site

Dockerfile

@@ -20,12 +20,6 @@ RUN apt-get install -y curl net-tools iproute2 iputils-ping
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 2
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 
-# Install pip
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-RUN python3.11 get-pip.py
-
-RUN python3.11 -m pip install --upgrade pip setuptools virtualenv
-
 # Install gcc and git
 RUN apt-get update && apt-get install -y build-essential gcc g++ clang git make cmake
 
@@ -42,12 +36,18 @@ RUN apt-get update
 
 RUN apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 
-RUN curl -sSL https://install.python-poetry.org | python3 -
+ADD https://astral.sh/uv/install.sh /uv-installer.sh
 
-ENV PATH="${PATH}:/root/.local/bin"
+RUN sh /uv-installer.sh && rm /uv-installer.sh
 
-ENV POETRY_VIRTUALENVS_CREATE=false
+ENV PATH="/root/.local/bin/:$PATH"
 
 COPY pyproject.toml .
 
-RUN poetry install --only core --no-root
+RUN uv python install 3.11.7
+
+RUN uv python pin 3.11.7
+
+RUN uv sync --group core
+
+ENV PATH=".venv/bin:$PATH"

Makefile

@@ -1,142 +1,146 @@
-POETRY_HOME := $(CURDIR)/.poetry
-POETRY := $(POETRY_HOME)/bin/poetry
-
-MIN_PYTHON_VERSION := 3.10
-
-PYTHON_VERSIONS := 3.11 3.10
-
-PYTHON := $(shell \
-    for ver in $(PYTHON_VERSIONS); do \
-        if command -v python$$ver >/dev/null 2>&1; then echo python$$ver; exit 0; fi; \
-    done \
-)
-
-ifndef PYTHON
-$(error "Python version $(MIN_PYTHON_VERSION) or higher is required but not found.")
-endif
-
-.PHONY: pre-install
-pre-install:
-	@echo "🐍 Using Python interpreter: $(PYTHON)"
-	@echo "🐍 Checking if Python is installed"
-	@command -v $(PYTHON) >/dev/null 2>&1 || { echo >&2 "$(PYTHON) is not installed. Aborting."; exit 1; }
-	@echo "🐍 Checking Python version"
-	@$(PYTHON) --version | grep -E "Python 3\.(1[0-9]|[2-9][0-9])" >/dev/null 2>&1 || { echo >&2 "Python $(MIN_PYTHON_VERSION) or higher is required. Aborting."; exit 1; }
-	@echo "📦 Checking if Poetry is installed"
-	@if ! command -v poetry >/dev/null 2>&1 || [ ! -d "$(POETRY_HOME)" ]; then \
-	    echo "Poetry is not installed or POETRY_HOME does not exist. Installing Poetry."; \
-	    curl -sSL https://install.python-poetry.org | POETRY_HOME=$(POETRY_HOME) $(PYTHON) -; \
-	fi
-	@echo "📦 Configuring Poetry"
-	@if [ -z "$$CONDA_PREFIX" ] && [ -z "$$VIRTUAL_ENV" ]; then \
-	    echo "Configuring Poetry to create a virtual environment."; \
-	    $(POETRY) config virtualenvs.in-project true; \
+UV := uv
+PYTHON_VERSION := 3.11
+UV_INSTALL_SCRIPT := https://astral.sh/uv/install.sh
+
+command_exists = $(shell command -v $(1) >/dev/null 2>&1 && echo true || echo false)
+
+define install_uv
+	@echo "📦 uv is not installed. Installing uv..."
+	@curl -LsSf $(UV_INSTALL_SCRIPT) | sh
+endef
+
+.PHONY: check-uv
+check-uv:		## Check and install uv if necessary
+	@if command -v $(UV) >/dev/null 2>&1; then \
+		echo "📦 uv is already installed."; \
 	else \
-	    echo "Configuring Poetry to use the existing environment."; \
-	    $(POETRY) config virtualenvs.create false; \
+		echo "📦 uv is not installed. Installing uv..."; \
+		curl -LsSf $(UV_INSTALL_SCRIPT) | sh; \
 	fi
-	@echo "📦 Setting Poetry to use $(PYTHON)"
-	@$(POETRY) env use $(PYTHON) || { echo "Failed to set Python version for Poetry. Aborting."; exit 1; }
+
+.PHONY: install-python
+install-python: check-uv	## Install Python with uv
+	@echo "🐍 Installing Python $(PYTHON_VERSION) with uv"
+	@$(UV) python install $(PYTHON_VERSION)
+	@echo "🔧 Configuring Python $(PYTHON_VERSION) as the default Python version"
+	@$(UV) python pin $(PYTHON_VERSION)
 
 .PHONY: install
-install: pre-install ## Install the poetry environment and install the pre-commit hooks
-	@echo "📦 Installing dependencies with Poetry"
-	@PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring $(POETRY) install --with core
+install: install-python		## Install core dependencies
+	@echo "📦 Installing core dependencies with uv"
+	@$(UV) sync --group core
 	@echo "🔧 Installing pre-commit hooks"
-	@$(POETRY) run pre-commit install
+	@$(UV) run pre-commit install
+	@echo ""
+	@echo "🐳 Building nebula-frontend docker image. Do you want to continue? (y/n)"
+	@read ans && [ $${ans:-N} = y ] || { echo "Build cancelled."; exit 1; }
+	@docker build -t nebula-frontend -f nebula/frontend/Dockerfile .
+	@echo ""
+	@echo "🐳 Building nebula-core docker image. Do you want to continue? (y/n)"
+	@read ans && [ $${ans:-N} = y ] || { echo "Build cancelled."; exit 1; }
+	@docker build -t nebula-core .
+	@echo ""
 	@$(MAKE) shell
 
 .PHONY: full-install
-full-install: pre-install ## Install the poetry environment and install the pre-commit hooks
-	@echo "📦 Installing dependencies with Poetry"
-	@PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring $(POETRY) install --with core,docs,dev
+full-install: install-python	## Install all dependencies (core, docs)
+	@echo "📦 Installing all dependencies with uv"
+	@$(UV) sync --group core --group docs
 	@echo "🔧 Installing pre-commit hooks"
-	@$(POETRY) run pre-commit install
+	@$(UV) run pre-commit install
 	@$(MAKE) shell
 
 .PHONY: shell
-shell: ## Start a shell in the poetry environment
-	@if [ -z "$$CONDA_PREFIX" ] && [ -z "$$VIRTUAL_ENV" ]; then \
-	    echo "🐚 Activating virtual environment"; \
-	    $(POETRY) shell; \
+shell:				## Start a shell in the uv environment
+	@echo "🐚 Starting a shell in the uv environment"
+	@if [ -n "$$VIRTUAL_ENV" ]; then \
+		echo "🐚 Already in a virtual environment: $$VIRTUAL_ENV"; \
+	elif [ ! -d ".venv" ]; then \
+		echo "❌ .venv directory not found. Running 'make install' to create it..."; \
+		$(MAKE) install; \
 	else \
-	    echo "🐚 Conda or virtual environment detected, skipping Poetry shell activation"; \
+		echo "🐚 Run the following command to activate the virtual environment:"; \
+		echo ""; \
+		echo '[Linux/MacOS]	\033[1;32msource .venv/bin/activate\033[0m'; \
+		echo '[Windows]	\033[1;32m.venv\\bin\\activate\033[0m'; \
+		echo ""; \
+		echo "🚀 NEBULA is ready to use!"; \
+		echo "🚀 Created by \033[1;34mEnrique Tomás Martínez Beltrán\033[0m <\033[1;[email protected]\033[0m>"; \
 	fi
 
-.PHONY: sync
-sync: ## Sync the lock file
-	@echo "📦 Syncing the lock file"
-	@PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring $(POETRY) lock
+.PHONY: lock
+lock:				## Update the lock file
+	@echo "🔒 This will update the lock file. Do you want to continue? (y/n)"
+	@read ans && [ $${ans:-N} = y ] || { echo "Lock cancelled."; exit 1; }
+	@echo "🔒 Locking dependencies..."
+	@$(UV) lock
 
 .PHONY: update-libs
-update-libs: ## Update libraries to the latest version
-	@echo "🔧 This will override the version of current libraries. Do you want to continue? (y/n)"
+update-libs:			## Update libraries to the latest version
+	@echo "🔧 This will override the versions of current libraries. Do you want to continue? (y/n)"
 	@read ans && [ $${ans:-N} = y ] || { echo "Update cancelled."; exit 1; }
 	@echo "📦 Updating libraries..."
-	@PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring $(POETRY) update
+	@$(UV) update
 
 .PHONY: check
-check: ## Run code quality tools.
+check:				## Run code quality tools
 	@echo "🛠️ Running code quality checks"
-	@echo "🔍 Checking Poetry lock file consistency"
-	@$(POETRY) check --lock
+	@echo "🔍 Checking uv lock file consistency"
+	@$(UV) sync
 	@echo "🚨 Linting code with pre-commit"
-	@$(POETRY) run pre-commit run -a
+	@$(UV) run pre-commit run -a
 
 .PHONY: check-plus
-check-plus: check ## Run additional code quality tools.
-	@echo "🔍 Checking code formatting with black
-	@$(POETRY) run black --check ."
+check-plus: check		## Run additional code quality tools
+	@echo "🔍 Checking code formatting with black"
+	@$(UV) run black --check .
 	@echo "⚙️ Static type checking with mypy"
-	@$(POETRY) run mypy
+	@$(UV) run mypy
 	@echo "🔎 Checking for obsolete dependencies"
-	@$(POETRY) run deptry .
+	@$(UV) run deptry .
 
 .PHONY: build
-build: clean-build ## Build wheel file using poetry
+build: clean-build		## Build the wheel file
 	@echo "🚀 Creating wheel file"
-	@$(POETRY) build
+	@$(UV) build
 
 .PHONY: clean-build
-clean-build: ## clean build artifacts
+clean-build:			## Clean build artifacts
 	@rm -rf dist
 
 .PHONY: publish
-publish: ## publish a release to pypi.
-	@echo "🚀 Publishing: Dry run."
-	@$(POETRY) config pypi-token.pypi $(PYPI_TOKEN)
-	@$(POETRY) publish --dry-run
-	@echo "🚀 Publishing."
-	@$(POETRY) publish
+publish:			## Publish a release to PyPI
+	@echo "🚀 Publishing...""
+	@$(UV) publish --token $(PYPI_TOKEN)
 
 .PHONY: build-and-publish
-build-and-publish: build publish ## Build and publish.
+build-and-publish: build publish	## Build and publish the package
 
 .PHONY: doc-test
-doc-test: ## Test if documentation can be built without warnings or errors
-	@$(POETRY) run mkdocs build -f docs/mkdocs.yml -d _build -s
+doc-test:			## Test if documentation can be built without errors
+	@$(UV) run mkdocs build -f docs/mkdocs.yml -d _build -s
 
 .PHONY: doc-build
-doc-build: ## Build the documentation
-	@$(POETRY) run mkdocs build -f docs/mkdocs.yml -d _build
+doc-build:			## Build the documentation
+	@$(UV) run mkdocs build -f docs/mkdocs.yml -d _build
 
 .PHONY: doc-serve
-doc-serve: ## Build and serve the documentation
-	@$(POETRY) run mkdocs serve -f docs/mkdocs.yml
+doc-serve:			## Serve the documentation locally
+	@$(UV) run mkdocs serve -f docs/mkdocs.yml
 
 .PHONY: format
-format: ## Format code with black and isort
+format:				## Format code with black and isort
 	@echo "🎨 Formatting code"
-	@$(POETRY) run black .
-	@$(POETRY) run isort .
+	@$(UV) run black .
+	@$(UV) run isort .
 
 .PHONY: clean
-clean: clean-build ## Clean up build artifacts and cache files
+clean: clean-build		## Clean up build artifacts and caches
 	@echo "🧹 Cleaning up build artifacts and caches"
 	@rm -rf __pycache__ */__pycache__ .mypy_cache
 
 .PHONY: help
-help:
+help:				## Display available commands
 	@echo "Available commands:"
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "💡 \033[36m%-20s\033[0m %s\n", $$1, $$2}'
 

README.md

@@ -147,7 +147,7 @@ Distributed under the GNU GPLv3 License. See `LICENSE` for more information.
 We would like to thank the following projects for their contributions which have helped shape NEBULA:
 
 - [PyTorch Lightning](https://github.com/Lightning-AI/pytorch-lightning) for the training loop and model management
-- [Tensorboard](https://github.com/tensorflow/tensorboard) and [Aim](https://github.com/aimhubio/aim) for the visualization tools and monitoring capabilities
+- [Tensorboard](https://github.com/tensorflow/tensorboard) for the visualization tools and monitoring capabilities
 - Different datasets ([nebula/core/datasets](https://github.com/CyberDataLab/nebula/tree/main/nebula/core/datasets)) and models ([nebula/core/models](https://github.com/CyberDataLab/nebula/tree/main/nebula/core/models)) for testing and validation purposes
 - [FastAPI](https://github.com/tiangolo/fastapi) for the RESTful API
 - [Web3](https://github.com/ethereum/web3.py) for the blockchain integration

nebula/controller.py

@@ -525,7 +525,7 @@ def run_frontend(self):
                     - NEBULA_PRODUCTION={production}
                     - NEBULA_GPU_AVAILABLE={gpu_available}
                     - NEBULA_ADVANCED_ANALYTICS={advanced_analytics}
-                    - SERVER_LOG=/nebula/app/logs/server.log
+                    - NEBULA_SERVER_LOG=/nebula/app/logs/server.log
                     - NEBULA_LOGS_DIR=/nebula/app/logs/
                     - NEBULA_CONFIG_DIR=/nebula/app/config/
                     - NEBULA_CERTS_DIR=/nebula/app/certs/

nebula/core/engine.py

@@ -17,7 +17,6 @@
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 logging.getLogger("fsspec").setLevel(logging.WARNING)
 logging.getLogger("matplotlib").setLevel(logging.ERROR)
-logging.getLogger("aim").setLevel(logging.ERROR)
 logging.getLogger("plotly").setLevel(logging.ERROR)
 
 import pdb

nebula/core/training/lightning.py

@@ -16,15 +16,10 @@
 from lightning.pytorch.loggers import CSVLogger
 from torch.nn import functional as F
 
+from nebula.config.config import TRAINING_LOGGER
 from nebula.core.utils.deterministic import enable_deterministic
 from nebula.core.utils.nebulalogger_tensorboard import NebulaTensorBoardLogger
 
-try:
-    from nebula.core.utils.nebulalogger import NebulaLogger
-except:
-    pass
-from nebula.config.config import TRAINING_LOGGER
-
 logging_training = logging.getLogger(TRAINING_LOGGER)
 
 
@@ -170,23 +165,6 @@ def create_logger(self):
             )
             # Restore logger configuration
             nebulalogger.set_logger_config(logger_config)
-        elif self.config.participant["tracking_args"]["local_tracking"] == "advanced":
-            nebulalogger = NebulaLogger(
-                config=self.config,
-                engine=self,
-                scenario_start_time=self.config.participant["scenario_args"]["start_time"],
-                repo=f"{self.config.participant['tracking_args']['log_dir']}",
-                experiment=self.experiment_name,
-                run_name=f"participant_{self.idx}",
-                train_metric_prefix="train_",
-                test_metric_prefix="test_",
-                val_metric_prefix="val_",
-                log_system_params=False,
-            )
-            # nebulalogger_aim = NebulaLogger(config=self.config, engine=self, scenario_start_time=self.config.participant["scenario_args"]["start_time"], repo=f"aim://nebula-frontend:8085",
-            #                                     experiment=self.experiment_name, run_name=f"participant_{self.idx}",
-            #                                     train_metric_prefix='train_', test_metric_prefix='test_', val_metric_prefix='val_', log_system_params=False)
-            self.config.participant["tracking_args"]["run_hash"] = nebulalogger.experiment.hash
         else:
             nebulalogger = None