Update dev ssl

.env.all

@@ -0,0 +1,17 @@
+PROGS_CFG=/app/docker.json
+
+# Change mount tyoe to "-nfs" to enable nfs, otherwise leave it empty
+MOUNT_TYPE=-nfs
+
+# Enable disable services
+ENABLE_PROGS=true
+ENABLE_B2B=true
+ENABLE_MKTG=true
+ENABLE_EDX=true
+
+ENABLE_STATE_MANAGER=false
+ENABLE_JUDGE=false
+ENABLE_ANALYTICS=false
+ENABLE_AUTH=false
+ENABLE_NOTIFIER=false
+

.env.b2b

@@ -0,0 +1,16 @@
+PROGS_CFG=/app/docker-b2b.json
+
+# Change mount tyoe to "-nfs" to enable nfs, otherwise leave it empty
+MOUNT_TYPE=-nfs
+
+# Enable disable services
+ENABLE_PROGS=false
+ENABLE_B2B=true
+ENABLE_MKTG=false
+ENABLE_EDX=false
+
+ENABLE_STATE_MANAGER=false
+ENABLE_JUDGE=false
+ENABLE_ANALYTICS=false
+ENABLE_AUTH=false
+ENABLE_NOTIFIER=false

.env.core

@@ -0,0 +1,17 @@
+PROGS_CFG=/app/docker.json
+
+# Change mount tyoe to "-nfs" to enable nfs, otherwise leave it empty
+MOUNT_TYPE=-nfs
+
+# Enable disable services
+ENABLE_PROGS=true
+ENABLE_B2B=false
+ENABLE_MKTG=true
+ENABLE_EDX=true
+
+ENABLE_STATE_MANAGER=true
+ENABLE_JUDGE=false
+ENABLE_ANALYTICS=false
+ENABLE_AUTH=false
+ENABLE_NOTIFIER=false
+

Makefile

@@ -8,16 +8,58 @@
 .PHONY: requirements
 
 DEVSTACK_WORKSPACE ?= $(shell pwd)/..
+include .env
 
 OS := $(shell uname)
 
-COMPOSE_PROJECT_NAME=devstack
+COMPOSE_PROJECT_NAME=${EDRAAK_COMPOSE_PROJECT_NAME:-devstack}
+DOCKER_COMPOSE_FILES=-f docker-compose.yml
+
+ifeq (${ENABLE_PROGS}, true)
+	DOCKER_COMPOSE_FILES+=-f docker-compose-progs.yml
+	DOCKER_COMPOSE_FILES+=-f docker-compose-progs${MOUNT_TYPE}.yml
+endif
+
+ifeq (${ENABLE_B2B}, true)
+	DOCKER_COMPOSE_FILES+=-f docker-compose-b2b.yml
+	DOCKER_COMPOSE_FILES+=-f docker-compose-b2b${MOUNT_TYPE}.yml
+endif
+
+ifeq (${ENABLE_MKTG}, true)
+	DOCKER_COMPOSE_FILES+=-f docker-compose-mktg.yml
+	DOCKER_COMPOSE_FILES+=-f docker-compose-mktg${MOUNT_TYPE}.yml
+endif
+
+ifeq (${ENABLE_EDX}, true)
+	DOCKER_COMPOSE_FILES+=-f docker-compose-edx.yml
+	DOCKER_COMPOSE_FILES+=-f docker-compose-edx${MOUNT_TYPE}.yml
+endif
+
+ifeq (${ENABLE_STATE_MANAGER}, true)
+	DOCKER_COMPOSE_FILES+=-f docker-compose-state-manager.yml
+	DOCKER_COMPOSE_FILES+=-f docker-compose-state-manager${MOUNT_TYPE}.yml
+endif
+
+ifeq (${ENABLE_JUDGE}, true)
+	DOCKER_COMPOSE_FILES+=-f docker-compose-judge.yml
+	DOCKER_COMPOSE_FILES+=-f docker-compose-judge${MOUNT_TYPE}.yml
+endif
+
+ifeq (${ENABLE_AUTH}, true)
+	DOCKER_COMPOSE_FILES+=-f docker-compose-auth-provider.yml
+	DOCKER_COMPOSE_FILES+=-f docker-compose-auth-provider${MOUNT_TYPE}.yml
+endif
+
+export DOCKER_COMPOSE_FILES
 
 export DEVSTACK_WORKSPACE
 export COMPOSE_PROJECT_NAME
 
 include *.mk
 
+compose-files:
+	@echo ${DOCKER_COMPOSE_FILES};
+
 # Generates a help message. Borrowed from https://github.com/pydanny/cookiecutter-djangopackage.
 help: ## Display this help message
 	@echo "Please use \`make <target>' where <target> is one of"
@@ -34,18 +76,17 @@ upgrade: ## Upgrade requirements with pip-tools
 		requirements/pip-tools.txt \
 		requirements/base.txt \
 
+dev.checkout: ## Check out "masters" otherwise
+	./repo.sh checkout
+
 dev.clone: ## Clone service repos to the parent directory
 	./repo.sh clone
 
 dev.provision.run: ## Provision all services with local mounted directories
-	DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-host.yml" ./provision.sh
+	./provision.sh
 
 dev.provision: | check-memory dev.clone dev.provision.run stop ## Provision dev environment with all services stopped
 
-dev.provision.xqueue: | check-memory dev.provision.xqueue.run stop stop.xqueue
-
-dev.provision.xqueue.run:
-	DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-xqueue.yml" ./provision-xqueue.sh
 
 dev.reset: | down dev.repo.reset pull dev.up static update-db ## Attempts to reset the local devstack to a the master working state
 
@@ -55,72 +96,55 @@ dev.status: ## Prints the status of all git repositories
 dev.repo.reset: ## Attempts to reset the local repo checkouts to the master working state
 	./repo.sh reset
 
-dev.up: | check-memory ## Bring up all services with host volumes
-	docker-compose -f docker-compose.yml -f docker-compose-host.yml up -d
-	@# Comment out this next line if you want to save some time and don't care about catalog programs
-	./programs/provision.sh cache >/dev/null
-
-dev.up.watchers: | check-memory ## Bring up asset watcher containers
-	docker-compose -f docker-compose-watchers.yml up -d
-
-dev.up.xqueue: | check-memory ## Bring up xqueue, assumes you already have lms running
-	docker-compose -f docker-compose.yml -f docker-compose-xqueue.yml -f docker-compose-host.yml up -d
-
-dev.up.all: | dev.up dev.up.watchers ## Bring up all services with host volumes, including watchers
+dev.editable-envs:  ## Copy env files outside the docker containers so it's editable by the developer
+	mkdir -p $(DEVSTACK_WORKSPACE)/src/edxapp-envs/
+	@docker exec -it edx.devstack.lms bash -c 'test -f /edx/src/edxapp-envs/lms.env.json || mv /edx/app/edxapp/lms.{env,auth}.json /edx/src/edxapp-envs/'
+	@docker exec -it edx.devstack.lms bash -c 'ln -sf /edx/src/edxapp-envs/lms.{env,auth}.json /edx/app/edxapp/'
+	@docker exec -it edx.devstack.studio bash -c 'test -f /edx/src/edxapp-envs/cms.env.json || mv /edx/app/edxapp/cms.{env,auth}.json /edx/src/edxapp-envs/'
+	@docker exec -it edx.devstack.studio bash -c 'ln -sf /edx/src/edxapp-envs/cms.{env,auth}.json /edx/app/edxapp/'
+	@sudo chown -R $(USER) $(DEVSTACK_WORKSPACE)/src
+	@make lms-restart
+	@make studio-restart
 
-dev.sync.daemon.start: ## Start the docker-sycn daemon
-	docker-sync start
-
-dev.sync.provision: | dev.sync.daemon.start dev.provision ## Provision with docker-sync enabled
-
-dev.sync.requirements: ## Install requirements
-	gem install docker-sync
-
-dev.sync.up: dev.sync.daemon.start ## Bring up all services with docker-sync enabled
-	docker-compose -f docker-compose.yml -f docker-compose-sync.yml up -d
-
-provision: | dev.provision ## This command will be deprecated in a future release, use dev.provision
-	echo "\033[0;31mThis command will be deprecated in a future release, use dev.provision\033[0m"
+dev.up: | check-memory ## Bring up all services with host volumes
+	docker-compose ${DOCKER_COMPOSE_FILES} up -d
+
+edraak.dev.up.hacks:
+	@# Start: Edraak hacks
+	@# TODO: Add this to `base.in` (thus `development.txt`) and rebuild the docker image
+	@make dev.editable-envs
+	@for container in lms studio lms_watcher studio_watcher; do \
+		 docker exec -it edx.devstack.$$container bash -c 'source /edx/app/edxapp/edxapp_env && pip install python-bidi==0.4.0'; \
+		 docker exec -it edx.devstack.$$container bash -c 'source /edx/app/edxapp/edxapp_env && pip install wand==0.5.1'; \
+		 docker exec -it edx.devstack.$$container bash -c 'source /edx/app/edxapp/edxapp_env && pip install -e /edx/app/edxapp/edx-platform'; \
+    done;
+	@make lms-restart
+	@make studio-restart
+	@# End: Edraak hacks
+
+provision: | dev.provision
 
 stop: ## Stop all services
 	(test -d .docker-sync && docker-sync stop) || true ## Ignore failure here
-	docker-compose stop
-
-stop.watchers: ## Stop asset watchers
-	docker-compose -f docker-compose-watchers.yml stop
-
-stop.all: | stop.analytics_pipeline stop stop.watchers ## Stop all containers, including asset watchers
-
-stop.xqueue:
-	docker-compose -f docker-compose-xqueue.yml stop
+	docker-compose ${DOCKER_COMPOSE_FILES} stop
 
 down: ## Remove all service containers and networks
-	(test -d .docker-sync && docker-sync clean) || true ## Ignore failure here
-	docker-compose -f docker-compose.yml -f docker-compose-watchers.yml -f docker-compose-xqueue.yml -f docker-compose-analytics-pipeline.yml down
+	docker-compose ${DOCKER_COMPOSE_FILES} down
 
 destroy: ## Remove all devstack-related containers, networks, and volumes
 	./destroy.sh
 
 logs: ## View logs from containers running in detached mode
-	docker-compose -f docker-compose.yml -f docker-compose-analytics-pipeline.yml logs -f
+	docker-compose ${DOCKER_COMPOSE_FILES} logs -f --tail 10
 
 %-logs: ## View the logs of the specified service container
-	docker-compose -f docker-compose.yml -f docker-compose-analytics-pipeline.yml logs -f --tail=500 $*
-
-xqueue-logs: ## View logs from containers running in detached mode
-	docker-compose -f docker-compose-xqueue.yml logs -f xqueue
-
-xqueue_consumer-logs: ## View logs from containers running in detached mode
-	docker-compose -f docker-compose-xqueue.yml logs -f xqueue_consumer
-
-pull: ## Update Docker images
-	docker-compose pull --parallel
+	docker-compose ${DOCKER_COMPOSE_FILES} logs -f --tail=500 $*
 
-pull.xqueue: ## Update XQueue Docker images
-	docker-compose -f docker-compose-xqueue.yml pull --parallel
+pull:
+	docker-compose ${DOCKER_COMPOSE_FILES} pull --parallel
 
 validate: ## Validate the devstack configuration
-	docker-compose config
+	docker-compose ${DOCKER_COMPOSE_FILES} config
 
 backup: ## Write all data volumes to the host.
 	docker run --rm --volumes-from edx.devstack.mysql -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql.tar.gz /var/lib/mysql
@@ -132,11 +156,16 @@ restore:  ## Restore all data volumes from the host. WARNING: THIS WILL OVERWRIT
 	docker run --rm --volumes-from edx.devstack.mongo -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz
 	docker run --rm --volumes-from edx.devstack.elasticsearch -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz
 
+%-edbash: ## Run a shell on the specified service container
+	docker exec -it edraak.devstack.$* /bin/bash
+
+
 # TODO: Print out help for this target. Even better if we can iterate over the
 # services in docker-compose.yml, and print the actual service names.
 %-shell: ## Run a shell on the specified service container
 	docker exec -it edx.devstack.$* /bin/bash
 
+
 credentials-shell:
 	docker exec -it edx.devstack.credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash'
 
@@ -202,7 +231,7 @@ lms-static: ## Rebuild static assets for the LMS container
 studio-static: ## Rebuild static assets for the Studio container
 	docker exec -t edx.devstack.studio bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets'
 
-static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers
+static: | lms-static studio-static ## Rebuild static assets for all service containers
 
 healthchecks: ## Run a curl against all services' healthcheck endpoints to make sure they are up. This will eventually be parameterized
 	./healthchecks.sh