Skip to content
This repository has been archived by the owner on Aug 1, 2024. It is now read-only.

Commit

Permalink
feat: remove docker-compose from devstack (#1172)
Browse files Browse the repository at this point in the history
  • Loading branch information
@nsprenkle
Rebecca Graber authored and nsprenkle committed Nov 21, 2023
1 parent e4c3eff commit c41187b
Show file tree
Hide file tree
Showing 35 changed files with 212 additions and 579 deletions.
6 changes: 4 additions & 2 deletions .github/workflows/cli-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,17 @@ jobs:
sudo apt update
sudo apt install docker-ce containerd.io
docker version
docker-compose --version
docker compose --version
# Note: we cannot use Docker Desktop because it has not been licensed for use in GithubActions
- name: Docker installation - Mac
if: ${{ matrix.os.name == 'mac' }}
run: |
brew install lima docker
brew install lima docker docker-compose
limactl start --name=default template://docker
echo "DOCKER_HOST=unix:///Users/runner/.lima/default/sock/docker.sock" >> $GITHUB_ENV
mkdir -p ~/.docker/cli-plugins
ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose
- name: Install Python dependencies
run: make requirements
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/provisioning-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
os:
- ubuntu-20.04 # Ubuntu 20.04 "Focal Fossa"
python-version: [ '3.8' ]
services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue]
services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue, analyticsapi+insights+lms]
fail-fast: false # some services can be flaky; let others run to completion even if one fails

steps:
Expand All @@ -48,7 +48,7 @@ jobs:
sudo apt update
sudo apt install docker-ce containerd.io
docker version
docker-compose --version
docker compose --version
- name: free up disk space
run: sudo apt remove --purge -y ghc-* azure-cli google-cloud-sdk hhvm llvm-* dotnet-* powershell mono-* php* ruby*
Expand Down
100 changes: 50 additions & 50 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ dev.pull.without-deps.%: ## Pull latest Docker images for specific services.
@scripts/send_metrics.py wrap "dev.pull.without-deps.$*"

impl-dev.pull.without-deps.%: ## Pull latest Docker images for specific services.
docker-compose pull $$(echo $* | tr + " ")
docker compose pull $$(echo $* | tr + " ")

dev.pull:
@scripts/send_metrics.py wrap "$@"
Expand All @@ -205,7 +205,7 @@ dev.pull.%: ## Pull latest Docker images for services and their dependencies.
@scripts/send_metrics.py wrap "dev.pull.$*"

impl-dev.pull.%: ## Pull latest Docker images for services and their dependencies.
docker-compose pull --include-deps $$(echo $* | tr + " ")
docker compose pull --include-deps $$(echo $* | tr + " ")

########################################################################################
# Developer interface: Database management.
Expand Down Expand Up @@ -253,18 +253,18 @@ $(foreach db_service,$(DB_SERVICES_LIST),\
dev.migrate: | $(_db_migration_targets) ## Run migrations for applicable default services.

dev.migrate.cms:
docker-compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-cms'
docker compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-cms'

dev.migrate.lms:
docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-lms'
docker compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-lms'

dev.migrate.%: ## Run migrations on a service.
docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate'
docker compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate'

dev.drop-db: _expects-database.dev.drop-db

dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mysql container.
docker-compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\""
docker compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\""


########################################################################################
Expand All @@ -274,7 +274,7 @@ dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mys
dev.up.attach: _expects-service.dev.up.attach

impl-dev.up.attach.%: ## Bring up a service and its dependencies + and attach to it.
docker-compose up $*
docker compose up $*

dev.up.attach.%: ## Bring up a service and its dependencies + and attach to it.
@scripts/send_metrics.py wrap "dev.up.attach.$*"
Expand All @@ -298,7 +298,7 @@ dev.up.with-watchers.%: ## Bring up services and their dependencies + asset watc
dev.up.without-deps: _expects-service-list.dev.up.without-deps

impl-dev.up.without-deps.%: dev.check-memory ## Bring up services by themselves.
docker-compose up -d --no-deps $$(echo $* | tr + " ")
docker compose up -d --no-deps $$(echo $* | tr + " ")

dev.up.without-deps.%: ## Bring up services by themselves.
@scripts/send_metrics.py wrap "dev.up.without-deps.$*"
Expand All @@ -316,7 +316,7 @@ dev.up.large-and-slow: dev.up.$(DEFAULT_SERVICES) ## Bring up default services.
@echo # at least one statement so that dev.up.% doesn't run too

impl-dev.up.%: dev.check-memory ## Bring up services and their dependencies.
docker-compose up -d $$(echo $* | tr + " ")
docker compose up -d $$(echo $* | tr + " ")
ifeq ($(ALWAYS_CACHE_PROGRAMS),true)
make dev.cache-programs
endif
Expand All @@ -326,32 +326,32 @@ dev.up.%:
@scripts/send_metrics.py wrap "dev.up.$*"

dev.ps: ## View list of created services and their statuses.
docker-compose ps
docker compose ps

dev.print-container.%: ## Get the ID of the running container for a given service.
@# Can be run as ``make --silent --no-print-directory dev.print-container.$service`` for just ID.
@echo $$(docker-compose ps --quiet $*)
@echo $$(docker compose ps --quiet $*)

dev.restart-container: ## Restart all service containers.
docker-compose restart $$(echo $* | tr + " ")
docker compose restart $$(echo $* | tr + " ")

dev.restart-container.%: ## Restart specific services' containers.
docker-compose restart $$(echo $* | tr + " ")
docker compose restart $$(echo $* | tr + " ")

dev.stop: ## Stop all running services.
docker-compose stop
docker compose stop

dev.stop.%: ## Stop specific services.
docker-compose stop $$(echo $* | tr + " ")
docker compose stop $$(echo $* | tr + " ")

dev.kill: ## Kill all running services.
docker-compose stop
docker compose stop

dev.kill.%: ## Kill specific services.
docker-compose kill $$(echo $* | tr + " ")
docker compose kill $$(echo $* | tr + " ")

dev.rm-stopped: ## Remove stopped containers. Does not affect running containers.
docker-compose rm --force
docker compose rm --force

dev.down: ## Documentation for a change to naming
@echo "dev.down has been renamed to dev.remove-containers. If this doesn't seem like what you were looking for, you probably want to be using dev.stop instead. See docs for more details."
Expand All @@ -360,10 +360,10 @@ dev.down.%:
@echo "dev.down has been renamed to dev.remove-containers. If this doesn't seem like what you were looking for, you probably want to be using dev.stop instead. See docs for more details."

dev.remove-containers: ## Stop and remove containers and networks for all services.
docker-compose down
docker compose down

dev.remove-containers.%: ## Stop and remove containers for specific services.
docker-compose rm --force --stop $$(echo $* | tr + " ")
docker compose rm --force --stop $$(echo $* | tr + " ")


########################################################################################
Expand All @@ -382,7 +382,7 @@ dev.check.%: # Run checks for a given service or set of services.
$(WINPTY) bash ./check.sh $*

dev.validate: ## Print effective Docker Compose config, validating files in COMPOSE_FILE.
docker-compose config
docker compose config


########################################################################################
Expand All @@ -395,20 +395,20 @@ dev.cache-programs: ## Copy programs from Discovery to Memcached for use in LMS.
dev.restart-devserver: _expects-service.dev.restart-devserver

dev.restart-devserver.forum:
docker-compose exec -T forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")'
docker compose exec -T forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")'

dev.forum.build-indices: ## Build indices for forum service
docker-compose exec -T forum bash -c "cd forum && source ruby_env && source devstack_forum_env && cd cs_comments_service/ && bin/rake search:rebuild_indices"
docker compose exec -T forum bash -c "cd forum && source ruby_env && source devstack_forum_env && cd cs_comments_service/ && bin/rake search:rebuild_indices"

dev.restart-devserver.%: ## Kill an edX service's development server. Watcher should restart it.
# Applicable to Django services only.
docker-compose exec -T $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")'
docker compose exec -T $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")'

dev.logs: ## View logs from running containers.
docker-compose logs -f
docker compose logs -f

dev.logs.%: ## View the logs of the specified service container.
docker-compose logs -f --tail=500 $*
docker compose logs -f --tail=500 $*

dev.attach: _expects-service.dev.attach

Expand All @@ -418,54 +418,54 @@ dev.attach.%: ## Attach to the specified service container process for debugging
dev.shell: _expects-service.dev.shell

dev.shell.credentials:
docker-compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash'
docker compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash'

dev.shell.discovery:
docker-compose exec discovery env TERM=$(TERM) bash -c '/bin/bash'
docker compose exec discovery env TERM=$(TERM) bash -c '/bin/bash'

dev.shell.ecommerce:
docker-compose exec ecommerce env TERM=$(TERM) /bin/bash
docker compose exec ecommerce env TERM=$(TERM) /bin/bash

dev.shell.registrar:
docker-compose exec registrar env TERM=$(TERM) /bin/bash
docker compose exec registrar env TERM=$(TERM) /bin/bash

dev.shell.xqueue:
docker-compose exec xqueue env TERM=$(TERM) /bin/bash
docker compose exec xqueue env TERM=$(TERM) /bin/bash

dev.shell.lms:
docker-compose exec lms env TERM=$(TERM) bash -c '/bin/bash'
docker compose exec lms env TERM=$(TERM) bash -c '/bin/bash'

dev.shell.lms_watcher:
docker-compose exec lms_watcher env TERM=$(TERM) bash -c '/bin/bash'
docker compose exec lms_watcher env TERM=$(TERM) bash -c '/bin/bash'

dev.shell.cms:
docker-compose exec cms env TERM=$(TERM) bash -c '/bin/bash'
docker compose exec cms env TERM=$(TERM) bash -c '/bin/bash'

dev.shell.cms_watcher:
docker-compose exec cms_watcher env TERM=$(TERM) bash -c '/bin/bash'
docker compose exec cms_watcher env TERM=$(TERM) bash -c '/bin/bash'

dev.shell.xqueue_consumer:
docker-compose exec xqueue_consumer env TERM=$(TERM) /bin/bash
docker compose exec xqueue_consumer env TERM=$(TERM) /bin/bash

dev.shell.analyticsapi:
docker exec -it edx.devstack.analyticsapi env TERM=$(TERM) bash -c '/bin/bash'

dev.shell.insights:
docker-compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /bin/bash'
docker compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /bin/bash'

dev.shell.%: ## Run a shell on the specified service's container.
docker-compose exec $* /bin/bash
docker compose exec $* /bin/bash

dev.dbshell:
docker-compose exec mysql57 bash -c "mysql"
docker compose exec mysql57 bash -c "mysql"

dev.dbcopy8.%: ## Copy data from old mysql 5.7 container into a new 8 db
docker-compose exec mysql57 bash -c "mysqldump $*" > .dev/$*.sql
docker-compose exec -T mysql80 bash -c "mysql $*" < .dev/$*.sql
docker compose exec mysql57 bash -c "mysqldump $*" > .dev/$*.sql
docker compose exec -T mysql80 bash -c "mysql $*" < .dev/$*.sql
rm .dev/$*.sql

dev.dbshell.%: ## Run a SQL shell on the given database.
docker-compose exec mysql57 bash -c "mysql $*"
docker compose exec mysql57 bash -c "mysql $*"

# List of Makefile targets to run static asset generation, in the form dev.static.$(service)
# Services will only have their asset generation added here
Expand All @@ -478,13 +478,13 @@ $(foreach asset_service,$(ASSET_SERVICES_LIST),\
dev.static: | $(_asset_compilation_targets)

dev.static.lms:
docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms'
docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms'

dev.static.cms:
docker-compose exec -T cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets cms'
docker compose exec -T cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets cms'

dev.static.%: ## Rebuild static assets for the specified service's container.
docker-compose exec -T $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static'
docker compose exec -T $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static'


########################################################################################
Expand Down Expand Up @@ -605,18 +605,18 @@ docs: ## generate Sphinx HTML documentation, including API docs

validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container.
touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile
docker-compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile
docker compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile
rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile

vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers.
@docker-compose logs chrome 2>&1 | grep "VNC password" | tail -1
@docker-compose logs firefox 2>&1 | grep "VNC password" | tail -1
@docker compose logs chrome 2>&1 | grep "VNC password" | tail -1
@docker compose logs firefox 2>&1 | grep "VNC password" | tail -1

devpi-password: ## Get the root devpi password for the devpi container.
docker-compose exec devpi bash -c "cat /data/server/.serverpassword"
docker compose exec devpi bash -c "cat /data/server/.serverpassword"

hadoop-application-logs-%: ## View hadoop logs by application Id.
docker-compose exec nodemanager yarn logs -applicationId $*
docker compose exec nodemanager yarn logs -applicationId $*

create-test-course: ## Provisions cms, and ecommerce with course(s) in test-course.json.
$(WINPTY) bash ./course-generator/create-courses.sh --cms --ecommerce course-generator/test-course.json
Expand Down
2 changes: 1 addition & 1 deletion check.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ run_check() {
if $cmd; then # Run the command itself and check if it succeeded.
succeeded="$succeeded $check_name"
else
docker-compose logs "$service"
docker compose logs "$service"
failed="$failed $check_name"
fi
set -e # Re-enable exit-on-error
Expand Down
8 changes: 4 additions & 4 deletions course-generator/create-courses.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ echo "Parsing options"
container_error=false
for arg in "$@"; do
if [ $arg == "--cms" ]; then
if [ ! "$(docker-compose exec lms bash -c 'echo "Course will be created for cms"; exit $?')" ]; then
if [ ! "$(docker compose exec lms bash -c 'echo "Course will be created for cms"; exit $?')" ]; then
echo "Issue with cms container"
container_error=true
else
cms=true
fi
elif [ $arg == "--ecommerce" ]; then
if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then
if [ ! "$(docker compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then
echo "Issue with ecommerce container"
container_error=true
else
Expand Down Expand Up @@ -42,10 +42,10 @@ done < "${@: -1}"

if $cms ; then
echo "Creating courses on cms."
docker-compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'"
docker compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'"
fi

if $ecommerce ; then
echo "Creating courses on ecommerce."
docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'"
docker compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'"
fi
Loading

0 comments on commit c41187b

Please sign in to comment.