NOISSUE - Simplify docker deployment

Makefile

@@ -21,20 +21,13 @@ DOCKER_PROJECT ?= $(shell echo $(subst $(space),,$(USER_REPO)) | tr -c -s '[:aln
 DOCKER_COMPOSE_COMMANDS_SUPPORTED := up down config
 DEFAULT_DOCKER_COMPOSE_COMMAND  := up
 GRPC_MTLS_CERT_FILES_EXISTS = 0
-DOCKER_PROFILE ?= $(MG_MQTT_BROKER_TYPE)_$(MG_MESSAGE_BROKER_TYPE)
 MOCKERY_VERSION=v2.38.0
 ifneq ($(MG_MESSAGE_BROKER_TYPE),)
     MG_MESSAGE_BROKER_TYPE := $(MG_MESSAGE_BROKER_TYPE)
 else
     MG_MESSAGE_BROKER_TYPE=nats
 endif
 
-ifneq ($(MG_MQTT_BROKER_TYPE),)
-    MG_MQTT_BROKER_TYPE := $(MG_MQTT_BROKER_TYPE)
-else
-    MG_MQTT_BROKER_TYPE=nats
-endif
-
 ifneq ($(MG_ES_TYPE),)
     MG_ES_TYPE := $(MG_ES_TYPE)
 else
@@ -111,7 +104,7 @@ clean:
 
 cleandocker:
 	# Stops containers and removes containers, networks, volumes, and images created by up
-	docker compose -f docker/docker-compose.yml --profile $(DOCKER_PROFILE) -p $(DOCKER_PROJECT) down --rmi all -v --remove-orphans
+	docker compose -f docker/docker-compose.yml -p $(DOCKER_PROJECT) down --rmi all -v --remove-orphans
 
 ifdef pv
 	# Remove unused volumes
@@ -208,53 +201,10 @@ endif
 endif
 endif
 
-define edit_docker_config
-	sed -i "s/MG_MQTT_BROKER_TYPE=.*/MG_MQTT_BROKER_TYPE=$(1)/" docker/.env
-	sed -i "s/MG_MQTT_BROKER_HEALTH_CHECK=.*/MG_MQTT_BROKER_HEALTH_CHECK=$$\{MG_$(shell echo ${MG_MQTT_BROKER_TYPE} | tr 'a-z' 'A-Z')_HEALTH_CHECK}/" docker/.env
-	sed -i "s/MG_MQTT_ADAPTER_WS_TARGET_PATH=.*/MG_MQTT_ADAPTER_WS_TARGET_PATH=$$\{MG_$(shell echo ${MG_MQTT_BROKER_TYPE} | tr 'a-z' 'A-Z')_WS_TARGET_PATH}/" docker/.env
-	sed -i "s/MG_MESSAGE_BROKER_TYPE=.*/MG_MESSAGE_BROKER_TYPE=$(2)/" docker/.env
-	sed -i "s,file: .*.yml,file: brokers/$(2).yml," docker/brokers/docker-compose.yml
-	sed -i "s,MG_MESSAGE_BROKER_URL=.*,MG_MESSAGE_BROKER_URL=$$\{MG_$(shell echo ${MG_MESSAGE_BROKER_TYPE} | tr 'a-z' 'A-Z')_URL\}," docker/.env
-	sed -i "s,MG_MQTT_ADAPTER_MQTT_QOS=.*,MG_MQTT_ADAPTER_MQTT_QOS=$$\{MG_$(shell echo ${MG_MQTT_BROKER_TYPE} | tr 'a-z' 'A-Z')_MQTT_QOS\}," docker/.env
-endef
-
-change_config:
-ifeq ($(DOCKER_PROFILE),nats_nats)
-	sed -i "s/- broker/- nats/g" docker/docker-compose.yml
-	sed -i "s/- rabbitmq/- nats/g" docker/docker-compose.yml
-	sed -i "s,MG_NATS_URL=.*,MG_NATS_URL=nats://nats:$$\{MG_NATS_PORT}," docker/.env
-	$(call edit_docker_config,nats,nats)
-else ifeq ($(DOCKER_PROFILE),nats_rabbitmq)
-	sed -i "s/nats/broker/g" docker/docker-compose.yml
-	sed -i "s,MG_NATS_URL=.*,MG_NATS_URL=nats://nats:$$\{MG_NATS_PORT}," docker/.env
-	sed -i "s/rabbitmq/broker/g" docker/docker-compose.yml
-	$(call edit_docker_config,nats,rabbitmq)
-else ifeq ($(DOCKER_PROFILE),vernemq_nats)
-	sed -i "s/nats/broker/g" docker/docker-compose.yml
-	sed -i "s/rabbitmq/broker/g" docker/docker-compose.yml
-	sed -i "s,MG_NATS_URL=.*,MG_NATS_URL=nats://broker:$$\{MG_NATS_PORT}," docker/.env
-	$(call edit_docker_config,vernemq,nats)
-else ifeq ($(DOCKER_PROFILE),vernemq_rabbitmq)
-	sed -i "s/nats/broker/g" docker/docker-compose.yml
-	sed -i "s/rabbitmq/broker/g" docker/docker-compose.yml
-	$(call edit_docker_config,vernemq,rabbitmq)
-else
-	$(error Invalid DOCKER_PROFILE $(DOCKER_PROFILE))
-endif
-
-run: check_certs change_config
-ifeq ($(MG_ES_TYPE), redis)
-	sed -i "s/MG_ES_TYPE=.*/MG_ES_TYPE=redis/" docker/.env
-	sed -i "s/MG_ES_URL=.*/MG_ES_URL=$$\{MG_REDIS_URL}/" docker/.env
-	docker compose -f docker/docker-compose.yml --profile $(DOCKER_PROFILE) --profile redis -p $(DOCKER_PROJECT) $(DOCKER_COMPOSE_COMMAND) $(args)
-else
-	sed -i "s,MG_ES_TYPE=.*,MG_ES_TYPE=$$\{MG_MESSAGE_BROKER_TYPE}," docker/.env
-	sed -i "s,MG_ES_URL=.*,MG_ES_URL=$$\{MG_$(shell echo ${MG_MESSAGE_BROKER_TYPE} | tr 'a-z' 'A-Z')_URL\}," docker/.env
-	docker compose -f docker/docker-compose.yml --env-file docker/.env --profile $(DOCKER_PROFILE) -p $(DOCKER_PROJECT) $(DOCKER_COMPOSE_COMMAND) $(args)
-endif
+run: check_certs
+	docker compose -f docker/docker-compose.yml --env-file docker/.env -p $(DOCKER_PROJECT) $(DOCKER_COMPOSE_COMMAND) $(args)
 
 run_addons: check_certs
-	$(call change_config)
 	$(foreach SVC,$(RUN_ADDON_ARGS),$(if $(filter $(SVC),$(ADDON_SERVICES) $(EXTERNAL_SERVICES)),,$(error Invalid Service $(SVC))))
 	@for SVC in $(RUN_ADDON_ARGS); do \
 		MG_ADDONS_CERTS_PATH_PREFIX="../."  docker compose -f docker/addons/$$SVC/docker-compose.yml -p $(DOCKER_PROJECT) --env-file ./docker/.env $(DOCKER_COMPOSE_COMMAND) $(args) & \

README.md

@@ -51,7 +51,7 @@ Developing Magistrala will also require:
 Once the prerequisites are installed, execute the following commands from the project's root:
 
 ```bash
-docker compose -f docker/docker-compose.yml --env-file docker/.env --profile nats_nats -p git_github_com_absmach_magistrala_git_  up
+docker compose -f docker/docker-compose.yml --env-file docker/.env -p git_github_com_absmach_magistrala_git_  up
 ```
 
 This will bring up the Magistrala docker services and interconnect them. This command can also be executed using the project's included Makefile:

docker/.env

@@ -12,7 +12,7 @@ MG_NGINX_MQTTS_PORT=8883
 MG_NATS_PORT=4222
 MG_NATS_HTTP_PORT=8222
 MG_NATS_JETSTREAM_KEY=u7wFoAPgXpDueXOFldBnXDh4xjnSOyEJ2Cb8Z5SZvGLzIZ3U4exWhhoIBZHzuNvh
-MG_NATS_URL=nats://broker:${MG_NATS_PORT}
+MG_NATS_URL=nats://nats:${MG_NATS_PORT}
 # Configs for nats as MQTT broker
 MG_NATS_HEALTH_CHECK=http://nats:${MG_NATS_HTTP_PORT}/healthz
 MG_NATS_WS_TARGET_PATH=
@@ -25,7 +25,7 @@ MG_RABBITMQ_USER=magistrala
 MG_RABBITMQ_PASS=magistrala
 MG_RABBITMQ_COOKIE=magistrala
 MG_RABBITMQ_VHOST=/
-MG_RABBITMQ_URL=amqp://${MG_RABBITMQ_USER}:${MG_RABBITMQ_PASS}@broker:${MG_RABBITMQ_PORT}${MG_RABBITMQ_VHOST}
+MG_RABBITMQ_URL=amqp://${MG_RABBITMQ_USER}:${MG_RABBITMQ_PASS}@rabbitmq:${MG_RABBITMQ_PORT}${MG_RABBITMQ_VHOST}
 
 ## Message Broker
 MG_MESSAGE_BROKER_TYPE=nats
@@ -55,7 +55,7 @@ MG_REDIS_URL=redis://es-redis:${MG_REDIS_TCP_PORT}/0
 
 ## Event Store
 MG_ES_TYPE=${MG_MESSAGE_BROKER_TYPE}
-MG_ES_URL=${MG_NATS_URL}
+MG_ES_URL=${MG_MESSAGE_BROKER_URL}
 
 ## Jaeger
 MG_JAEGER_COLLECTOR_OTLP_ENABLED=true

docker/README.md

@@ -10,7 +10,7 @@ Follow the [official documentation](https://docs.docker.com/compose/install/).
 
 ## Usage
 
-Run following commands from project root directory.
+Run the following commands from the project root directory.
 
 ```bash
 docker-compose -f docker/docker-compose.yml up
@@ -21,3 +21,99 @@ docker-compose -f docker/addons/<path>/docker-compose.yml  up
 ```
 
 To pull docker images from a specific release you need to change the value of `MG_RELEASE_TAG` in `.env` before running these commands.
+
+## Broker Configuration
+
+Magistrala supports configurable MQTT broker and Message broker, which also acts as an events store. Magistrala uses two types of brokers:
+
+1. MQTT_BROKER: Handles MQTT communication between MQTT adapters and message broker. This can either be 'VerneMQ' or 'NATS'.
+2. MESSAGE_BROKER: Manages message exchange between Magistrala core, optional, and external services. This can either be 'NATS' or 'RabbitMQ'. This is used to store messages for distributed processing.
+
+Events store: This is used by Magistrala services to store events for distributed processing. Magistrala uses a single service to be the message broker and events store. This can either be 'NATS' or 'RabbitMQ'. Redis can also be used as an events store, but it requires a message broker to be deployed along with it for message exchange.
+
+This is the same as MESSAGE_BROKER. This can either be 'NATS' or 'RabbitMQ' or 'Redis'.  If Redis is used as an events store, then RabbitMQ or NATS is used as a message broker.
+
+The current deployment strategy for Magistrala in `docker/docker-compose.yml` is to use VerneMQ as a MQTT_BROKER and NATS as a MESSAGE_BROKER and EVENTS_STORE.
+
+Therefore, the following combinations are possible:
+
+- MQTT_BROKER: VerneMQ, MESSAGE_BROKER: NATS, EVENTS_STORE: NATS
+- MQTT_BROKER: VerneMQ, MESSAGE_BROKER: NATS, EVENTS_STORE: Redis
+- MQTT_BROKER: VerneMQ, MESSAGE_BROKER: RabbitMQ, EVENTS_STORE: RabbitMQ
+- MQTT_BROKER: VerneMQ, MESSAGE_BROKER: RabbitMQ, EVENTS_STORE: Redis
+- MQTT_BROKER: NATS, MESSAGE_BROKER: RabbitMQ, EVENTS_STORE: RabbitMQ
+- MQTT_BROKER: NATS, MESSAGE_BROKER: RabbitMQ, EVENTS_STORE: Redis
+- MQTT_BROKER: NATS, MESSAGE_BROKER: NATS, EVENTS_STORE: NATS
+- MQTT_BROKER: NATS, MESSAGE_BROKER: NATS, EVENTS_STORE: Redis
+
+For Message brokers other than NATS, you would need to build the docker images with RabbitMQ as the build tag and change the `docker/.env`. For example, to use RabbitMQ as a message broker:
+
+```bash
+MG_MESSAGE_BROKER_TYPE=rabbitmq make dockers
+```
+
+```env
+MG_MESSAGE_BROKER_TYPE=rabbitmq
+MG_MESSAGE_BROKER_URL=${MG_RABBITMQ_URL}
+```
+
+For Redis as an events store, you would need to run RabbitMQ or NATS as a message broker. For example, to use Redis as an events store with rabbitmq as a message broker:
+
+```bash
+MG_ES_TYPE=redis MG_MESSAGE_BROKER_TYPE=rabbitmq make dockers
+```
+
+```env
+MG_MESSAGE_BROKER_TYPE=rabbitmq
+MG_MESSAGE_BROKER_URL=${MG_RABBITMQ_URL}
+MG_ES_TYPE=redis
+MG_ES_URL=${MG_REDIS_URL}
+```
+
+For MQTT broker other than VerneMQ, you would need to change the `docker/.env`. For example, to use NATS as a MQTT broker:
+
+```env
+MG_MQTT_BROKER_TYPE=nats
+MG_MQTT_BROKER_HEALTH_CHECK=${MG_NATS_HEALTH_CHECK}
+MG_MQTT_ADAPTER_MQTT_QOS=${MG_NATS_MQTT_QOS}
+MG_MQTT_ADAPTER_MQTT_TARGET_HOST=${MG_MQTT_BROKER_TYPE}
+MG_MQTT_ADAPTER_MQTT_TARGET_PORT=1883
+MG_MQTT_ADAPTER_MQTT_TARGET_HEALTH_CHECK=${MG_MQTT_BROKER_HEALTH_CHECK}
+MG_MQTT_ADAPTER_WS_TARGET_HOST=${MG_MQTT_BROKER_TYPE}
+MG_MQTT_ADAPTER_WS_TARGET_PORT=8080
+MG_MQTT_ADAPTER_WS_TARGET_PATH=${MG_NATS_WS_TARGET_PATH}
+```
+
+### RabbitMQ configuration
+
+```yaml
+services:
+  rabbitmq:
+    image: rabbitmq:3.12.12-management-alpine
+    container_name: magistrala-rabbitmq
+    restart: on-failure
+    environment:
+      RABBITMQ_ERLANG_COOKIE: ${MG_RABBITMQ_COOKIE}
+      RABBITMQ_DEFAULT_USER: ${MG_RABBITMQ_USER}
+      RABBITMQ_DEFAULT_PASS: ${MG_RABBITMQ_PASS}
+      RABBITMQ_DEFAULT_VHOST: ${MG_RABBITMQ_VHOST}
+    ports:
+      - ${MG_RABBITMQ_PORT}:${MG_RABBITMQ_PORT}
+      - ${MG_RABBITMQ_HTTP_PORT}:${MG_RABBITMQ_HTTP_PORT}
+    networks:
+      - magistrala-base-net
+```
+
+### Redis configuration
+
+```yaml
+services:
+  redis:
+    image: redis:7.2.4-alpine
+    container_name: magistrala-es-redis
+    restart: on-failure
+    networks:
+      - magistrala-base-net
+    volumes:
+      - magistrala-broker-volume:/data
+```