Merge pull request #226 from NASA-IMPACT/fix/install-dependencies

Fix installing the deployment dependencies

Makefile

@@ -1,30 +1,30 @@
-
-
 .PHONY:
 	clean
 	all
 	test
+	list
 
+list:
+	$(MAKE) -C sm2a list
 
 all:
 	$(MAKE) -C sm2a all
 
-
 sm2a-local-run:
 	$(MAKE) -C sm2a sm2a-local-run
 
 sm2a-local-init:
 	$(MAKE) -C sm2a sm2a-local-init
 
-
 sm2a-local-stop:
 	$(MAKE) -C sm2a sm2a-local-stop
 
-
-
 sm2a-deploy:
 	$(MAKE) -C sm2a sm2a-deploy
 
+sm2a-local-build:
+	$(MAKE) -C sm2a sm2a-local-build
+
 clean:
 	$(MAKE) -C sm2a clean
 

README.md

@@ -38,8 +38,12 @@ See [getting-started-install](https://docs.aws.amazon.com/cli/latest/userguide/g
 
 
 ### Setup a local SM2A development environment
+1. Build services
+```shell
+make sm2a-local-build
+```
 
-1. Initialize the metadata db
+2. Initialize the metadata db
 
 ```shell
 make sm2a-local-init
@@ -50,14 +54,14 @@ which will require you to reinitialize SM2A with `make sm2a-local-init`
 
 This will create an airflow username: `airflow` with password `airflow`
 
-2. Start all services
+3. Start all services
 
 ```shell
 make sm2a-local-run
 ```
 This will start SM2A services and will be running on http://localhost:8080
 
-3. Stop all services
+4. Stop all services
 
 ```shell
 make sm2a-local-stop

sm2a/Makefile

@@ -1,14 +1,12 @@
 SECRET_NAME=""
 ENV_FILE=".env"
 
-
 important_message = \
 	@echo "\033[0;31m$(1) \033[0m"
 
 info_message = \
 	@echo "\033[0;32m$(1) \033[0m"
 
-
 count_down = \
 	@echo "Spinning up the system please wait..."; \
 	secs=40 ;\
@@ -18,16 +16,13 @@ count_down = \
 		: $$((secs--)); \
 	done;
 
-
 .PHONY:
 	clean
 	all
-
-
+	list
 
 all: sm2a-local-init sm2a-local-run
 
-
 sm2a-local-run: sm2a-local-stop
 	@echo "Running SM2A"
 	docker compose up -d
@@ -39,16 +34,22 @@ sm2a-local-run: sm2a-local-stop
 
 sm2a-local-init:
 	cp sm2a-local-config/env_example sm2a-local-config/.env
+	cp -r ../dags .
 	docker compose run --rm airflow-cli db init
 	docker compose run --rm airflow-cli users create --email [email protected] --firstname airflow --lastname airflow --password airflow --username airflow --role Admin
+	rm -rf dags
 
 sm2a-local-stop:
 	docker compose down
 
+sm2a-local-build:
+	cp -r ../dags .
+	docker compose build
+	rm -rf dags
 
 sm2a-deploy:
 	@echo "Installing the deployment dependency"
-	pip install -r deploy_requirements.txt
+	pip install -r ../deploy_requirements.txt
 	@echo "Deploying SM2A"
 	cp -r ../dags .
 	python scripts/generate_env_file.py --secret-id ${SECRET_NAME} --env-file ${ENV_FILE}
@@ -60,3 +61,6 @@ clean: sm2a-local-stop
 	docker container prune -f
 	docker image prune -f
 	docker volume prune -f
+
+list:
+	@grep '^[^#[:space:]].*:' Makefile

sm2a/README.md

@@ -80,17 +80,34 @@ This project uses Terraform modules to deploy Apache Airflow and related AWS res
 - [gitflow.yml](./.github/workflows/gitflow.yml) provides a structured way to manage the development, testing, and deployment of terraform modules. For more info refer to [gitflow](https://github.com/NASA-IMPACT/csda-data-pipelines/blob/dev/GITFLOW.md)
 
 
+### Setup a local SM2A development environment
+1. Build services
+```shell
+make sm2a-local-build
+```
 
-### Deployment via local machine
-You can deploy SM2A from your local machine by running:
-```bash
-$python scripts/generate_env_file.py --secret-id $AWS_SECRET_NAME --env-file .env
+2. Initialize the metadata db
+
+```shell
+make sm2a-local-init
 ```
-Assuming you have access to [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/) where the deployment variables are stored.
+🚨 NOTE: This command is typically required only once at the beginning. 
+After running it, you generally do not need to run it again unless you run `make clean`,
+which will require you to reinitialize SM2A with `make sm2a-local-init`
 
-```bash
-./scripts/deploy.sh .env <<< init
-./scripts/deploy.sh .env <<< deploy
+This will create an airflow username: `airflow` with password `airflow`
+
+3. Start all services
+
+```shell
+make sm2a-local-run
+```
+This will start SM2A services and will be running on http://localhost:8080
+
+4. Stop all services
+
+```shell
+make sm2a-local-stop
 ```
 
 ### Login to UI