Simplify docker deployment

Signed-off-by: rodneyosodo <[email protected]>

Makefile

@@ -21,20 +21,13 @@ DOCKER_PROJECT ?= $(shell echo $(subst $(space),,$(USER_REPO)) | tr -c -s '[:aln
 DOCKER_COMPOSE_COMMANDS_SUPPORTED := up down config
 DEFAULT_DOCKER_COMPOSE_COMMAND  := up
 GRPC_MTLS_CERT_FILES_EXISTS = 0
-DOCKER_PROFILE ?= $(MG_MQTT_BROKER_TYPE)_$(MG_MESSAGE_BROKER_TYPE)
 MOCKERY_VERSION=v2.38.0
 ifneq ($(MG_MESSAGE_BROKER_TYPE),)
     MG_MESSAGE_BROKER_TYPE := $(MG_MESSAGE_BROKER_TYPE)
 else
     MG_MESSAGE_BROKER_TYPE=nats
 endif
 
-ifneq ($(MG_MQTT_BROKER_TYPE),)
-    MG_MQTT_BROKER_TYPE := $(MG_MQTT_BROKER_TYPE)
-else
-    MG_MQTT_BROKER_TYPE=nats
-endif
-
 ifneq ($(MG_ES_TYPE),)
     MG_ES_TYPE := $(MG_ES_TYPE)
 else
@@ -111,7 +104,7 @@ clean:
 
 cleandocker:
 	# Stops containers and removes containers, networks, volumes, and images created by up
-	docker compose -f docker/docker-compose.yml --profile $(DOCKER_PROFILE) -p $(DOCKER_PROJECT) down --rmi all -v --remove-orphans
+	docker compose -f docker/docker-compose.yml -p $(DOCKER_PROJECT) down --rmi all -v --remove-orphans
 
 ifdef pv
 	# Remove unused volumes
@@ -208,53 +201,10 @@ endif
 endif
 endif
 
-define edit_docker_config
-	sed -i "s/MG_MQTT_BROKER_TYPE=.*/MG_MQTT_BROKER_TYPE=$(1)/" docker/.env
-	sed -i "s/MG_MQTT_BROKER_HEALTH_CHECK=.*/MG_MQTT_BROKER_HEALTH_CHECK=$$\{MG_$(shell echo ${MG_MQTT_BROKER_TYPE} | tr 'a-z' 'A-Z')_HEALTH_CHECK}/" docker/.env
-	sed -i "s/MG_MQTT_ADAPTER_WS_TARGET_PATH=.*/MG_MQTT_ADAPTER_WS_TARGET_PATH=$$\{MG_$(shell echo ${MG_MQTT_BROKER_TYPE} | tr 'a-z' 'A-Z')_WS_TARGET_PATH}/" docker/.env
-	sed -i "s/MG_MESSAGE_BROKER_TYPE=.*/MG_MESSAGE_BROKER_TYPE=$(2)/" docker/.env
-	sed -i "s,file: .*.yml,file: brokers/$(2).yml," docker/brokers/docker-compose.yml
-	sed -i "s,MG_MESSAGE_BROKER_URL=.*,MG_MESSAGE_BROKER_URL=$$\{MG_$(shell echo ${MG_MESSAGE_BROKER_TYPE} | tr 'a-z' 'A-Z')_URL\}," docker/.env
-	sed -i "s,MG_MQTT_ADAPTER_MQTT_QOS=.*,MG_MQTT_ADAPTER_MQTT_QOS=$$\{MG_$(shell echo ${MG_MQTT_BROKER_TYPE} | tr 'a-z' 'A-Z')_MQTT_QOS\}," docker/.env
-endef
-
-change_config:
-ifeq ($(DOCKER_PROFILE),nats_nats)
-	sed -i "s/- broker/- nats/g" docker/docker-compose.yml
-	sed -i "s/- rabbitmq/- nats/g" docker/docker-compose.yml
-	sed -i "s,MG_NATS_URL=.*,MG_NATS_URL=nats://nats:$$\{MG_NATS_PORT}," docker/.env
-	$(call edit_docker_config,nats,nats)
-else ifeq ($(DOCKER_PROFILE),nats_rabbitmq)
-	sed -i "s/nats/broker/g" docker/docker-compose.yml
-	sed -i "s,MG_NATS_URL=.*,MG_NATS_URL=nats://nats:$$\{MG_NATS_PORT}," docker/.env
-	sed -i "s/rabbitmq/broker/g" docker/docker-compose.yml
-	$(call edit_docker_config,nats,rabbitmq)
-else ifeq ($(DOCKER_PROFILE),vernemq_nats)
-	sed -i "s/nats/broker/g" docker/docker-compose.yml
-	sed -i "s/rabbitmq/broker/g" docker/docker-compose.yml
-	sed -i "s,MG_NATS_URL=.*,MG_NATS_URL=nats://broker:$$\{MG_NATS_PORT}," docker/.env
-	$(call edit_docker_config,vernemq,nats)
-else ifeq ($(DOCKER_PROFILE),vernemq_rabbitmq)
-	sed -i "s/nats/broker/g" docker/docker-compose.yml
-	sed -i "s/rabbitmq/broker/g" docker/docker-compose.yml
-	$(call edit_docker_config,vernemq,rabbitmq)
-else
-	$(error Invalid DOCKER_PROFILE $(DOCKER_PROFILE))
-endif
-
-run: check_certs change_config
-ifeq ($(MG_ES_TYPE), redis)
-	sed -i "s/MG_ES_TYPE=.*/MG_ES_TYPE=redis/" docker/.env
-	sed -i "s/MG_ES_URL=.*/MG_ES_URL=$$\{MG_REDIS_URL}/" docker/.env
-	docker compose -f docker/docker-compose.yml --profile $(DOCKER_PROFILE) --profile redis -p $(DOCKER_PROJECT) $(DOCKER_COMPOSE_COMMAND) $(args)
-else
-	sed -i "s,MG_ES_TYPE=.*,MG_ES_TYPE=$$\{MG_MESSAGE_BROKER_TYPE}," docker/.env
-	sed -i "s,MG_ES_URL=.*,MG_ES_URL=$$\{MG_$(shell echo ${MG_MESSAGE_BROKER_TYPE} | tr 'a-z' 'A-Z')_URL\}," docker/.env
-	docker compose -f docker/docker-compose.yml --env-file docker/.env --profile $(DOCKER_PROFILE) -p $(DOCKER_PROJECT) $(DOCKER_COMPOSE_COMMAND) $(args)
-endif
+run: check_certs
+	docker compose -f docker/docker-compose.yml --env-file docker/.env -p $(DOCKER_PROJECT) $(DOCKER_COMPOSE_COMMAND) $(args)
 
 run_addons: check_certs
-	$(call change_config)
 	$(foreach SVC,$(RUN_ADDON_ARGS),$(if $(filter $(SVC),$(ADDON_SERVICES) $(EXTERNAL_SERVICES)),,$(error Invalid Service $(SVC))))
 	@for SVC in $(RUN_ADDON_ARGS); do \
 		MG_ADDONS_CERTS_PATH_PREFIX="../."  docker compose -f docker/addons/$$SVC/docker-compose.yml -p $(DOCKER_PROJECT) --env-file ./docker/.env $(DOCKER_COMPOSE_COMMAND) $(args) & \

README.md

@@ -51,7 +51,7 @@ Developing Magistrala will also require:
 Once the prerequisites are installed, execute the following commands from the project's root:
 
 ```bash
-docker compose -f docker/docker-compose.yml --env-file docker/.env --profile nats_nats -p git_github_com_absmach_magistrala_git_  up
+docker compose -f docker/docker-compose.yml --env-file docker/.env -p git_github_com_absmach_magistrala_git_  up
 ```
 
 This will bring up the Magistrala docker services and interconnect them. This command can also be executed using the project's included Makefile:

docker/README.md

@@ -21,3 +21,104 @@ docker-compose -f docker/addons/<path>/docker-compose.yml  up
 ```
 
 To pull docker images from a specific release you need to change the value of `MG_RELEASE_TAG` in `.env` before running these commands.
+
+## Broker Configuration
+
+Magistrala supports configurable MQTT broker and Message broker, which also acts as events store. Magistrala uses two types of brokers:
+
+1. MQTT_BROKER: Handles MQTT communication between MQTT adapters and message broker. This can either be 'vernemq' or 'nats'.
+2. MESSAGE_BROKER: Manages communication between adapters and Magistrala writer services. This can either be 'nats' or 'rabbitmq' or 'redis'. This is used to store messages for distributed processing.
+
+Events store: This is the same as MESSAGE_BROKER. This can either be 'nats' or 'rabbitmq' or 'redis'. This is used by Magistrala services to store events for distributed processing. If redis is used as an events store, then rabbitmq or nats is used as a message broker.
+
+Since nats is used as both MQTT_BROKER and MESSAGE_BROKER, it is not possible to run nats as an MQTT_BROKER and nats as a MESSAGE_BROKER at the same time, this is the current depolyment strategy for Magistrala in `docker/docker-compose.yml`.
+
+. Therefore, the following combinations are possible:
+
+- MQTT_BROKER: vernemq, MESSAGE_BROKER: nats, EVENTS_STORE: nats
+- MQTT_BROKER: vernemq, MESSAGE_BROKER: nats, EVENTS_STORE: redis
+- MQTT_BROKER: vernemq, MESSAGE_BROKER: rabbitmq, EVENTS_STORE: rabbitmq
+- MQTT_BROKER: vernemq, MESSAGE_BROKER: rabbitmq, EVENTS_STORE: redis
+- MQTT_BROKER: nats, MESSAGE_BROKER: rabbitmq, EVENTS_STORE: rabbitmq
+- MQTT_BROKER: nats, MESSAGE_BROKER: rabbitmq, EVENTS_STORE: redis
+- MQTT_BROKER: nats, MESSAGE_BROKER: nats, EVENTS_STORE: nats
+- MQTT_BROKER: nats, MESSAGE_BROKER: nats, EVENTS_STORE: redis
+
+For Message brokers other than nats, you would need to change the `docker/.env`. For example, to use rabbitmq as a message broker:
+
+```env
+MG_MESSAGE_BROKER_TYPE=rabbitmq
+MG_MESSAGE_BROKER_URL=${MG_RABBITMQ_URL}
+```
+
+For redis as an events store, you would need to run rabbitmq or nats as a message broker. For example, to use redis as an events store with rabbitmq as a message broker:
+
+```env
+MG_MESSAGE_BROKER_TYPE=rabbitmq
+MG_MESSAGE_BROKER_URL=${MG_RABBITMQ_URL}
+MG_ES_TYPE=redis
+MG_ES_URL=${MG_REDIS_URL}
+```
+
+For MQTT brokers other than nats, you would need to change the `docker/.env`. For example, to use vernemq as a MQTT broker:
+
+```env
+MG_MQTT_BROKER_TYPE=vernemq
+MG_MQTT_BROKER_HEALTH_CHECK=${MG_VERNEMQ_HEALTH_CHECK}
+MG_MQTT_ADAPTER_MQTT_QOS=${MG_VERNEMQ_MQTT_QOS}
+MG_MQTT_ADAPTER_MQTT_TARGET_HOST=${MG_MQTT_BROKER_TYPE}
+MG_MQTT_ADAPTER_MQTT_TARGET_PORT=1883
+MG_MQTT_ADAPTER_MQTT_TARGET_HEALTH_CHECK=${MG_MQTT_BROKER_HEALTH_CHECK}
+MG_MQTT_ADAPTER_WS_TARGET_HOST=${MG_MQTT_BROKER_TYPE}
+MG_MQTT_ADAPTER_WS_TARGET_PORT=8080
+MG_MQTT_ADAPTER_WS_TARGET_PATH=${MG_VERNEMQ_WS_TARGET_PATH}
+```
+
+### VerneMQ configuration
+
+```yaml
+services:
+  vernemq:
+    image: magistrala/vernemq:${MG_RELEASE_TAG}
+    container_name: magistrala-vernemq
+    restart: on-failure
+    environment:
+      DOCKER_VERNEMQ_ALLOW_ANONYMOUS: ${MG_DOCKER_VERNEMQ_ALLOW_ANONYMOUS}
+      DOCKER_VERNEMQ_LOG__CONSOLE__LEVEL: ${MG_DOCKER_VERNEMQ_LOG__CONSOLE__LEVEL}
+    networks:
+      - magistrala-base-net
+    volumes:
+      - magistrala-broker-volume:/var/lib/vernemq
+```
+
+### RabbitMQ configuration
+
+```yaml
+services:
+  rabbitmq:
+    image: rabbitmq:3.9.20-management-alpine
+    environment:
+      RABBITMQ_ERLANG_COOKIE: ${MG_RABBITMQ_COOKIE}
+      RABBITMQ_DEFAULT_USER: ${MG_RABBITMQ_USER}
+      RABBITMQ_DEFAULT_PASS: ${MG_RABBITMQ_PASS}
+      RABBITMQ_DEFAULT_VHOST: ${MG_RABBITMQ_VHOST}
+    ports:
+      - ${MG_RABBITMQ_PORT}:${MG_RABBITMQ_PORT}
+      - ${MG_RABBITMQ_HTTP_PORT}:${MG_RABBITMQ_HTTP_PORT}
+    networks:
+      - magistrala-base-net
+```
+
+### Redis configuration
+
+```yaml
+services:
+  redis:
+    image: redis:7.2.0-alpine
+    container_name: magistrala-es-redis
+    restart: on-failure
+    networks:
+      - magistrala-base-net
+    volumes:
+      - magistrala-broker-volume:/data
+```