diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml index f3f9189291..292481e76a 100644 --- a/.github/workflows/cli-tests.yml +++ b/.github/workflows/cli-tests.yml @@ -48,15 +48,17 @@ jobs: sudo apt update sudo apt install docker-ce containerd.io docker version - docker-compose --version + docker compose --version # Note: we cannot use Docker Desktop because it has not been licensed for use in GithubActions - name: Docker installation - Mac if: ${{ matrix.os.name == 'mac' }} run: | - brew install lima docker + brew install lima docker docker-compose limactl start --name=default template://docker echo "DOCKER_HOST=unix:///Users/runner/.lima/default/sock/docker.sock" >> $GITHUB_ENV + mkdir -p ~/.docker/cli-plugins + ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose - name: Install Python dependencies run: make requirements diff --git a/.github/workflows/provisioning-tests.yml b/.github/workflows/provisioning-tests.yml index 3f7a029874..586817d3ca 100644 --- a/.github/workflows/provisioning-tests.yml +++ b/.github/workflows/provisioning-tests.yml @@ -28,7 +28,7 @@ jobs: os: - ubuntu-20.04 # Ubuntu 20.04 "Focal Fossa" python-version: [ '3.8' ] - services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue] + services: [ discovery+lms+forum ,registrar+lms, ecommerce+lms, edx_notes_api+lms, credentials+lms, xqueue, analyticsapi+insights+lms] fail-fast: false # some services can be flaky; let others run to completion even if one fails steps: @@ -48,7 +48,7 @@ jobs: sudo apt update sudo apt install docker-ce containerd.io docker version - docker-compose --version + docker compose --version - name: free up disk space run: sudo apt remove --purge -y ghc-* azure-cli google-cloud-sdk hhvm llvm-* dotnet-* powershell mono-* php* ruby* diff --git a/Makefile b/Makefile index 194792cea1..2f5cbd3afc 100644 --- a/Makefile +++ b/Makefile @@ -189,7 +189,7 @@ dev.pull.without-deps.%: ## Pull latest Docker images for specific services. @scripts/send_metrics.py wrap "dev.pull.without-deps.$*" impl-dev.pull.without-deps.%: ## Pull latest Docker images for specific services. - docker-compose pull $$(echo $* | tr + " ") + docker compose pull $$(echo $* | tr + " ") dev.pull: @scripts/send_metrics.py wrap "$@" @@ -205,7 +205,7 @@ dev.pull.%: ## Pull latest Docker images for services and their dependencies. @scripts/send_metrics.py wrap "dev.pull.$*" impl-dev.pull.%: ## Pull latest Docker images for services and their dependencies. - docker-compose pull --include-deps $$(echo $* | tr + " ") + docker compose pull --include-deps $$(echo $* | tr + " ") ######################################################################################## # Developer interface: Database management. @@ -253,18 +253,18 @@ $(foreach db_service,$(DB_SERVICES_LIST),\ dev.migrate: | $(_db_migration_targets) ## Run migrations for applicable default services. dev.migrate.cms: - docker-compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-cms' + docker compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-cms' dev.migrate.lms: - docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-lms' + docker compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate-lms' dev.migrate.%: ## Run migrations on a service. - docker-compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' + docker compose exec $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make migrate' dev.drop-db: _expects-database.dev.drop-db dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mysql container. - docker-compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\"" + docker compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\"" ######################################################################################## @@ -274,7 +274,7 @@ dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mys dev.up.attach: _expects-service.dev.up.attach impl-dev.up.attach.%: ## Bring up a service and its dependencies + and attach to it. - docker-compose up $* + docker compose up $* dev.up.attach.%: ## Bring up a service and its dependencies + and attach to it. @scripts/send_metrics.py wrap "dev.up.attach.$*" @@ -298,7 +298,7 @@ dev.up.with-watchers.%: ## Bring up services and their dependencies + asset watc dev.up.without-deps: _expects-service-list.dev.up.without-deps impl-dev.up.without-deps.%: dev.check-memory ## Bring up services by themselves. - docker-compose up -d --no-deps $$(echo $* | tr + " ") + docker compose up -d --no-deps $$(echo $* | tr + " ") dev.up.without-deps.%: ## Bring up services by themselves. @scripts/send_metrics.py wrap "dev.up.without-deps.$*" @@ -316,7 +316,7 @@ dev.up.large-and-slow: dev.up.$(DEFAULT_SERVICES) ## Bring up default services. @echo # at least one statement so that dev.up.% doesn't run too impl-dev.up.%: dev.check-memory ## Bring up services and their dependencies. - docker-compose up -d $$(echo $* | tr + " ") + docker compose up -d $$(echo $* | tr + " ") ifeq ($(ALWAYS_CACHE_PROGRAMS),true) make dev.cache-programs endif @@ -326,32 +326,32 @@ dev.up.%: @scripts/send_metrics.py wrap "dev.up.$*" dev.ps: ## View list of created services and their statuses. - docker-compose ps + docker compose ps dev.print-container.%: ## Get the ID of the running container for a given service. @# Can be run as ``make --silent --no-print-directory dev.print-container.$service`` for just ID. - @echo $$(docker-compose ps --quiet $*) + @echo $$(docker compose ps --quiet $*) dev.restart-container: ## Restart all service containers. - docker-compose restart $$(echo $* | tr + " ") + docker compose restart $$(echo $* | tr + " ") dev.restart-container.%: ## Restart specific services' containers. - docker-compose restart $$(echo $* | tr + " ") + docker compose restart $$(echo $* | tr + " ") dev.stop: ## Stop all running services. - docker-compose stop + docker compose stop dev.stop.%: ## Stop specific services. - docker-compose stop $$(echo $* | tr + " ") + docker compose stop $$(echo $* | tr + " ") dev.kill: ## Kill all running services. - docker-compose stop + docker compose stop dev.kill.%: ## Kill specific services. - docker-compose kill $$(echo $* | tr + " ") + docker compose kill $$(echo $* | tr + " ") dev.rm-stopped: ## Remove stopped containers. Does not affect running containers. - docker-compose rm --force + docker compose rm --force dev.down: ## Documentation for a change to naming @echo "dev.down has been renamed to dev.remove-containers. If this doesn't seem like what you were looking for, you probably want to be using dev.stop instead. See docs for more details." @@ -360,10 +360,10 @@ dev.down.%: @echo "dev.down has been renamed to dev.remove-containers. If this doesn't seem like what you were looking for, you probably want to be using dev.stop instead. See docs for more details." dev.remove-containers: ## Stop and remove containers and networks for all services. - docker-compose down + docker compose down dev.remove-containers.%: ## Stop and remove containers for specific services. - docker-compose rm --force --stop $$(echo $* | tr + " ") + docker compose rm --force --stop $$(echo $* | tr + " ") ######################################################################################## @@ -382,7 +382,7 @@ dev.check.%: # Run checks for a given service or set of services. $(WINPTY) bash ./check.sh $* dev.validate: ## Print effective Docker Compose config, validating files in COMPOSE_FILE. - docker-compose config + docker compose config ######################################################################################## @@ -395,20 +395,20 @@ dev.cache-programs: ## Copy programs from Discovery to Memcached for use in LMS. dev.restart-devserver: _expects-service.dev.restart-devserver dev.restart-devserver.forum: - docker-compose exec -T forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' + docker compose exec -T forum bash -c 'kill $$(ps aux | grep "ruby app.rb" | egrep -v "while|grep" | awk "{print \$$2}")' dev.forum.build-indices: ## Build indices for forum service - docker-compose exec -T forum bash -c "cd forum && source ruby_env && source devstack_forum_env && cd cs_comments_service/ && bin/rake search:rebuild_indices" + docker compose exec -T forum bash -c "cd forum && source ruby_env && source devstack_forum_env && cd cs_comments_service/ && bin/rake search:rebuild_indices" dev.restart-devserver.%: ## Kill an edX service's development server. Watcher should restart it. # Applicable to Django services only. - docker-compose exec -T $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' + docker compose exec -T $* bash -c 'kill $$(ps aux | egrep "manage.py ?\w* runserver" | egrep -v "while|grep" | awk "{print \$$2}")' dev.logs: ## View logs from running containers. - docker-compose logs -f + docker compose logs -f dev.logs.%: ## View the logs of the specified service container. - docker-compose logs -f --tail=500 $* + docker compose logs -f --tail=500 $* dev.attach: _expects-service.dev.attach @@ -418,54 +418,54 @@ dev.attach.%: ## Attach to the specified service container process for debugging dev.shell: _expects-service.dev.shell dev.shell.credentials: - docker-compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' + docker compose exec credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash' dev.shell.discovery: - docker-compose exec discovery env TERM=$(TERM) bash -c '/bin/bash' + docker compose exec discovery env TERM=$(TERM) bash -c '/bin/bash' dev.shell.ecommerce: - docker-compose exec ecommerce env TERM=$(TERM) /bin/bash + docker compose exec ecommerce env TERM=$(TERM) /bin/bash dev.shell.registrar: - docker-compose exec registrar env TERM=$(TERM) /bin/bash + docker compose exec registrar env TERM=$(TERM) /bin/bash dev.shell.xqueue: - docker-compose exec xqueue env TERM=$(TERM) /bin/bash + docker compose exec xqueue env TERM=$(TERM) /bin/bash dev.shell.lms: - docker-compose exec lms env TERM=$(TERM) bash -c '/bin/bash' + docker compose exec lms env TERM=$(TERM) bash -c '/bin/bash' dev.shell.lms_watcher: - docker-compose exec lms_watcher env TERM=$(TERM) bash -c '/bin/bash' + docker compose exec lms_watcher env TERM=$(TERM) bash -c '/bin/bash' dev.shell.cms: - docker-compose exec cms env TERM=$(TERM) bash -c '/bin/bash' + docker compose exec cms env TERM=$(TERM) bash -c '/bin/bash' dev.shell.cms_watcher: - docker-compose exec cms_watcher env TERM=$(TERM) bash -c '/bin/bash' + docker compose exec cms_watcher env TERM=$(TERM) bash -c '/bin/bash' dev.shell.xqueue_consumer: - docker-compose exec xqueue_consumer env TERM=$(TERM) /bin/bash + docker compose exec xqueue_consumer env TERM=$(TERM) /bin/bash dev.shell.analyticsapi: docker exec -it edx.devstack.analyticsapi env TERM=$(TERM) bash -c '/bin/bash' dev.shell.insights: - docker-compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /bin/bash' + docker compose exec insights env TERM=$(TERM) bash -c 'eval $$(source /edx/app/insights/insights_env; echo PATH="$$PATH";) && /bin/bash' dev.shell.%: ## Run a shell on the specified service's container. - docker-compose exec $* /bin/bash + docker compose exec $* /bin/bash dev.dbshell: - docker-compose exec mysql57 bash -c "mysql" + docker compose exec mysql57 bash -c "mysql" dev.dbcopy8.%: ## Copy data from old mysql 5.7 container into a new 8 db - docker-compose exec mysql57 bash -c "mysqldump $*" > .dev/$*.sql - docker-compose exec -T mysql80 bash -c "mysql $*" < .dev/$*.sql + docker compose exec mysql57 bash -c "mysqldump $*" > .dev/$*.sql + docker compose exec -T mysql80 bash -c "mysql $*" < .dev/$*.sql rm .dev/$*.sql dev.dbshell.%: ## Run a SQL shell on the given database. - docker-compose exec mysql57 bash -c "mysql $*" + docker compose exec mysql57 bash -c "mysql $*" # List of Makefile targets to run static asset generation, in the form dev.static.$(service) # Services will only have their asset generation added here @@ -478,13 +478,13 @@ $(foreach asset_service,$(ASSET_SERVICES_LIST),\ dev.static: | $(_asset_compilation_targets) dev.static.lms: - docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms' + docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms' dev.static.cms: - docker-compose exec -T cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets cms' + docker compose exec -T cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets cms' dev.static.%: ## Rebuild static assets for the specified service's container. - docker-compose exec -T $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' + docker compose exec -T $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static' ######################################################################################## @@ -605,18 +605,18 @@ docs: ## generate Sphinx HTML documentation, including API docs validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container. touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile - docker-compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile + docker compose exec -T lms ls /edx/app/edxapp/edx-platform/testfile rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile vnc-passwords: ## Get the VNC passwords for the Chrome and Firefox Selenium containers. - @docker-compose logs chrome 2>&1 | grep "VNC password" | tail -1 - @docker-compose logs firefox 2>&1 | grep "VNC password" | tail -1 + @docker compose logs chrome 2>&1 | grep "VNC password" | tail -1 + @docker compose logs firefox 2>&1 | grep "VNC password" | tail -1 devpi-password: ## Get the root devpi password for the devpi container. - docker-compose exec devpi bash -c "cat /data/server/.serverpassword" + docker compose exec devpi bash -c "cat /data/server/.serverpassword" hadoop-application-logs-%: ## View hadoop logs by application Id. - docker-compose exec nodemanager yarn logs -applicationId $* + docker compose exec nodemanager yarn logs -applicationId $* create-test-course: ## Provisions cms, and ecommerce with course(s) in test-course.json. $(WINPTY) bash ./course-generator/create-courses.sh --cms --ecommerce course-generator/test-course.json diff --git a/check.sh b/check.sh index 0113cb6263..bbad6eb037 100755 --- a/check.sh +++ b/check.sh @@ -46,7 +46,7 @@ run_check() { if $cmd; then # Run the command itself and check if it succeeded. succeeded="$succeeded $check_name" else - docker-compose logs "$service" + docker compose logs "$service" failed="$failed $check_name" fi set -e # Re-enable exit-on-error diff --git a/course-generator/create-courses.sh b/course-generator/create-courses.sh index 861ce50d27..c4c714a5ec 100755 --- a/course-generator/create-courses.sh +++ b/course-generator/create-courses.sh @@ -7,14 +7,14 @@ echo "Parsing options" container_error=false for arg in "$@"; do if [ $arg == "--cms" ]; then - if [ ! "$(docker-compose exec lms bash -c 'echo "Course will be created for cms"; exit $?')" ]; then + if [ ! "$(docker compose exec lms bash -c 'echo "Course will be created for cms"; exit $?')" ]; then echo "Issue with cms container" container_error=true else cms=true fi elif [ $arg == "--ecommerce" ]; then - if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then + if [ ! "$(docker compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then echo "Issue with ecommerce container" container_error=true else @@ -42,10 +42,10 @@ done < "${@: -1}" if $cms ; then echo "Creating courses on cms." - docker-compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" + docker compose exec lms bash -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker generate_courses '$course_json'" fi if $ecommerce ; then echo "Creating courses on ecommerce." - docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" + docker compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'" fi diff --git a/credentials/generate_program_certificate.sh b/credentials/generate_program_certificate.sh index 2347b03d03..0f1b0dda51 100755 --- a/credentials/generate_program_certificate.sh +++ b/credentials/generate_program_certificate.sh @@ -2,27 +2,27 @@ echo 'Attempting to award a program certificate to the edX user' echo 'Updating Discovery...' echo 'Adding assets to the edX demo organization' -docker-compose exec -T discovery bash -c 'mkdir /edx/app/discovery/discovery/provision-temp' +docker compose exec -T discovery bash -c 'mkdir /edx/app/discovery/discovery/provision-temp' docker cp ./assets edx.devstack.discovery:/edx/app/discovery/discovery/provision-temp/assets -docker-compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py add_logos_to_organization --partner=edX --logo=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-logo.png --certificate_logo=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-certificate-logo.png --banner_image=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-banner-image.png' -docker-compose exec -T discovery bash -c 'rm -rf /edx/app/discovery/discovery/provision-temp' +docker compose exec -T discovery bash -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py add_logos_to_organization --partner=edX --logo=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-logo.png --certificate_logo=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-certificate-logo.png --banner_image=/edx/app/discovery/discovery/provision-temp/assets/demo-asset-banner-image.png' +docker compose exec -T discovery bash -c 'rm -rf /edx/app/discovery/discovery/provision-temp' echo 'Updating credentials...' echo 'setting catalog and lms base urls' -docker-compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py create_or_update_site --site-domain example.com --site-name example.com --platform-name edX --tos-url https://www.edx.org/edx-terms-service --privacy-policy-url https://www.edx.org/edx-privacy-policy --homepage-url https://www.edx.org --company-name "edX Inc." --certificate-help-url https://edx.readthedocs.org/projects/edx-guide-for-students/en/latest/SFD_certificates.html#web-certificates --lms-url-root http://edx.devstack.lms:18000/ --catalog-api-url http://edx.devstack.discovery:18381/api/v1/ --theme-name edx.org' +docker compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py create_or_update_site --site-domain example.com --site-name example.com --platform-name edX --tos-url https://www.edx.org/edx-terms-service --privacy-policy-url https://www.edx.org/edx-privacy-policy --homepage-url https://www.edx.org --company-name "edX Inc." --certificate-help-url https://edx.readthedocs.org/projects/edx-guide-for-students/en/latest/SFD_certificates.html#web-certificates --lms-url-root http://edx.devstack.lms:18000/ --catalog-api-url http://edx.devstack.discovery:18381/api/v1/ --theme-name edx.org' echo 'copying discovery catalog' -docker-compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py copy_catalog' +docker compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py copy_catalog' echo 'creating a program certificate configuration' -docker-compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py create_program_certificate_configuration' +docker compose exec -T credentials bash -c 'source /edx/app/credentials/credentials_env && python /edx/app/credentials/credentials/manage.py create_program_certificate_configuration' echo 'Updating LMS...' echo 'creating a credentials API connection' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms create_credentials_api_configuration' +docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms create_credentials_api_configuration' echo 'changing edX user enrollment in demo course from audit to verified' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms change_enrollment -u edx -c course-v1:edX+DemoX+Demo_Course --from audit --to verified' +docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms change_enrollment -u edx -c course-v1:edX+DemoX+Demo_Course --from audit --to verified' echo 'manually ID verifying edX user' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms manual_verifications --email edx@example.com' +docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms manual_verifications --email edx@example.com' echo 'generating course certificate' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms cert_generation -u 3 -c course-v1:edX+DemoX+Demo_Course' +docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms cert_generation -u 3 -c course-v1:edX+DemoX+Demo_Course' echo 'notifying credentials' -docker-compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker notify_credentials --courses course-v1:edX+DemoX+Demo_Course --notify_programs' +docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker notify_credentials --courses course-v1:edX+DemoX+Demo_Course --notify_programs' diff --git a/destroy.sh b/destroy.sh index 1c92e23784..235367f7bd 100755 --- a/destroy.sh +++ b/destroy.sh @@ -4,5 +4,5 @@ set -eu -o pipefail read -p "This will delete all data in your devstack. Would you like to proceed? [y/n] " -r if [[ $REPLY =~ ^[Yy]$ ]] then - docker-compose down -v + docker compose down -v fi diff --git a/docs/developing_on_named_release_branches.rst b/docs/developing_on_named_release_branches.rst index b48a8f25da..4dbd5950c1 100644 --- a/docs/developing_on_named_release_branches.rst +++ b/docs/developing_on_named_release_branches.rst @@ -13,10 +13,19 @@ By default, the startup steps in :doc:`getting_started` will install the devstac checkout of each service repository #. Continue with step 3 in :doc:`getting_started` to pull the correct docker images. -All ``make`` target and ``docker-compose`` calls should now use the correct +All ``make`` target and ``docker compose`` calls should now use the correct images until you change or unset ``OPENEDX_RELEASE`` again. To work on the master branches and ``latest`` images, unset ``OPENEDX_RELEASE`` or set it to an empty string. +Note that older versions of devstack may have different prerequisites. In particular, +releases before Quince will need support for the ``docker-compose`` syntax as +well as the newer ``docker compose``. The easiest way to do this is to add +is to add a shell script with the following and put it on the PATH under the name docker-compose: + + .. code:: sh + + #!/bin/bash + docker compose "$@" How do I run multiple named Open edX releases on same machine? -------------------------------------------------------------- diff --git a/docs/devstack_faq.rst b/docs/devstack_faq.rst index 13ecd9fd9c..1665718c8b 100644 --- a/docs/devstack_faq.rst +++ b/docs/devstack_faq.rst @@ -54,7 +54,7 @@ starts, you have a few options: automatically on a regular basis. See `building images for devstack`_ for more information. * You can update your requirements files as appropriate and then build your own updated image for the service as described above, tagging it such that - ``docker-compose`` will use it instead of the last image you downloaded. + ``docker compose`` will use it instead of the last image you downloaded. (Alternatively, you can temporarily edit ``docker-compose.yml`` to replace the ``image`` entry for that service with the ID of your new image.) You should be sure to modify the variable override for the version of the @@ -145,8 +145,8 @@ Then bring your mysql container down and back up by running: .. code:: sh - docker-compose stop mysql57 - docker-compose up -d mysql57 + docker compose stop mysql57 + docker compose up -d mysql57 Then connect using the values below. Note that the username and password will vary depending on the database. For all of the options, see ``provision.sql``. diff --git a/docs/devstack_interface.rst b/docs/devstack_interface.rst index 5aa38fe860..e1dc02ba08 100644 --- a/docs/devstack_interface.rst +++ b/docs/devstack_interface.rst @@ -1,7 +1,7 @@ Devstack Interface ------------------ -Devstack comes built in with many useful make commands that act as an user interface. This UI is essentially a make wrapper around docker-compose commands. We attempt to give a short summary of what the make commands do below, but it would be a good idea for you to familiarize yourself with some knowledge of docker-compose. +Devstack comes built in with many useful make commands that act as an user interface. This UI is essentially a make wrapper around ``docker compose`` commands. We attempt to give a short summary of what the make commands do below, but it would be a good idea for you to familiarize yourself with some knowledge of docker compose. Due to the organic nature of how this user interface came into being, there are often multiple ways to do the same thing. The two main variants of commands are of the form: ``dev.ACTION.SERVICE`` vs ``SERVICE-ACTION``. The ``SERVICE-ACTION`` variant tends to be more tab-completion friendly. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index b34251c9c1..c8697ac930 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -8,21 +8,21 @@ You will need to have the following installed: - make - Python 3.8 -- Docker, including ``docker-compose`` +- Docker, including ``docker compose`` -This project requires **Docker 17.06+ CE**. We recommend Docker Stable, but +This project requires **Docker 19.03+ CE**. We recommend Docker Stable, but Docker Edge should work as well. Ensure that your Docker installation includes -``docker-compose``; on some operating systems (e.g. Ubuntu Linux) this may require +``docker compose``; on some operating systems (e.g. Ubuntu Linux) this may require a separate package. **NOTE:** Switching between Docker Stable and Docker Edge will remove all images and settings. Don't forget to restore your memory setting and be prepared to provision. -For macOS users, please use `Docker for Mac`_. Previous Mac-based tools (e.g. -boot2docker) are *not* supported. Please be aware that the `licensing terms`_ for -Docker for Mac (aka Docker Desktop) may mean that it is no longer -free for your organization's use. +For macOS users, please use `Docker for Mac`_, which comes with ``docker +compose``. Previous Mac-based tools (e.g. boot2docker) are *not* supported. +Please be aware that the `licensing terms`_ for Docker for Mac (aka Docker +Desktop) may mean that it is no longer free for your organization's use. Since a Docker-based devstack runs many containers, you should configure Docker with a sufficient @@ -32,9 +32,14 @@ does work. `Docker for Windows`_ may work but has not been tested and is *not* supported. -If you are using Linux, use the ``overlay2`` storage driver, kernel version -4.0+ and *not* ``overlay``. To check which storage driver your -``docker-daemon`` uses, run the following command. +If you are using Linux, developers on Ubuntu (and Debian) should ensure +they've uninstalled docker.io and docker-compose from the main Ubuntu +repositories and instead install docker-ce and docker-compose-plugin from the +official Docker package repository: +https://docs.docker.com/engine/install/ubuntu/. Also they should use the +``overlay2`` storage driver, kernel version 4.0+ and *not* ``overlay``. To +check which storage driver your ``docker-daemon`` uses, run the following +command. .. code:: sh diff --git a/docs/troubleshoot_general_tips.rst b/docs/troubleshoot_general_tips.rst index 17435e0cef..d036d52297 100644 --- a/docs/troubleshoot_general_tips.rst +++ b/docs/troubleshoot_general_tips.rst @@ -134,7 +134,7 @@ for example: The most common culprit is an infinite restart loop where an error during service startup causes the process to exit, but we've configured -``docker-compose`` to immediately try starting it again (so the container will +``docker compose`` to immediately try starting it again (so the container will stay running long enough for you to use a shell to investigate and fix the problem). Make sure the set of packages installed in the container matches what your current code branch expects; you may need to rerun ``pip`` on a diff --git a/enterprise/provision.sh b/enterprise/provision.sh index 18b8947a44..dcc5c08b96 100755 --- a/enterprise/provision.sh +++ b/enterprise/provision.sh @@ -2,7 +2,7 @@ set -eu -o pipefail set -x -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com --staff' -cat enterprise/worker_permissions.py | docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user enterprise_worker enterprise_worker@example.com --staff' +cat enterprise/worker_permissions.py | docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "enterprise-backend-service-key" --client-secret "enterprise-backend-service-secret" enterprise-backend-service enterprise_worker' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "enterprise-backend-service-key" --client-secret "enterprise-backend-service-secret" enterprise-backend-service enterprise_worker' diff --git a/programs/provision.sh b/programs/provision.sh index 32697100d7..ca7cd50d19 100755 --- a/programs/provision.sh +++ b/programs/provision.sh @@ -45,7 +45,7 @@ docker_exec() { /edx/app/$app/$repo/manage.py $cmd " - docker-compose exec -T "$service" bash -e -c "$CMDS" + docker compose exec -T "$service" bash -e -c "$CMDS" } provision_ida() { diff --git a/provision-analyticsapi.sh b/provision-analyticsapi.sh index 0e4fffc373..3d564ad629 100755 --- a/provision-analyticsapi.sh +++ b/provision-analyticsapi.sh @@ -7,16 +7,16 @@ set -x name=analyticsapi port=19001 -docker-compose up -d ${name} +docker compose up -d ${name} echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make develop' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make develop' -- ${name} echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && export DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.devstack" && cd /edx/app/analytics_api/analytics_api && make migrate-all' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && export DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.devstack" && cd /edx/app/analytics_api/analytics_api && make migrate-all' -- ${name} echo -e "${GREEN}Creating default user and authentication token for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && python manage.py set_api_key edx edx' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && python manage.py set_api_key edx edx' -- ${name} echo -e "${GREEN}Loading test data for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make loaddata' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/analytics_api/analytics_api_env && cd /edx/app/analytics_api/analytics_api && make loaddata' -- ${name} diff --git a/provision-coursegraph.sh b/provision-coursegraph.sh index 4bbc5d0602..337c99c234 100755 --- a/provision-coursegraph.sh +++ b/provision-coursegraph.sh @@ -9,14 +9,14 @@ set -x # Also, this gives us a chance to refresh the container in case it's gotten into # a weird state. echo -e "${GREEN} Ensuring Coursegraph image is up to date...${NC}" -docker-compose rm --force --stop coursegraph -docker-compose pull coursegraph +docker compose rm --force --stop coursegraph +docker compose pull coursegraph echo -e "${GREEN} Starting Coursegraph and CMS...${NC}" -docker-compose up -d coursegraph cms +docker compose up -d coursegraph cms sleep 10 # Give Neo4j some time to boot up. echo -e "${GREEN} Updating CMS courses in Coursegraph...${NC}" -docker-compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && ./manage.py cms dump_to_neo4j --host coursegraph.devstack.edx --user neo4j --password edx' +docker compose exec cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && ./manage.py cms dump_to_neo4j --host coursegraph.devstack.edx --user neo4j --password edx' echo -e "${GREEN} Coursegraph is now up-to-date with CMS!${NC}" diff --git a/provision-credentials.sh b/provision-credentials.sh index 77fd1ce02e..f73da9a6eb 100755 --- a/provision-credentials.sh +++ b/provision-credentials.sh @@ -10,26 +10,26 @@ set -x name=credentials port=18150 -docker-compose up -d $name +docker compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements' -- "$name" +docker compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" +docker compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" +docker compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$name" echo -e "${GREEN}Configuring site for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' +docker compose exec -T ${name} bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && ./manage.py create_or_update_site --site-id=1 --site-domain=localhost:18150 --site-name="Open edX" --platform-name="Open edX" --company-name="Open edX" --lms-url-root=http://localhost:18000 --catalog-api-url=http://edx.devstack.discovery:18381/api/v1/ --tos-url=http://localhost:18000/tos --privacy-policy-url=http://localhost:18000/privacy --homepage-url=http://localhost:18000 --certificate-help-url=http://localhost:18000/faq --records-help-url=http://localhost:18000/faq --theme-name=openedx' ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker compose exec -T ${name} bash -e -c ' if ! source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make static 2>creds_make_static.err; then echo "------- Last 100 lines of stderr"; tail creds_make_static.err -n 100; echo "-------"; fi;' -- "$name" # Restart credentials devserver. make dev.restart-devserver.credentials diff --git a/provision-discovery.sh b/provision-discovery.sh index 04f959d619..8b947a47c9 100755 --- a/provision-discovery.sh +++ b/provision-discovery.sh @@ -3,24 +3,24 @@ set -eu -o pipefail set -x -docker-compose up -d lms -docker-compose up -d cms -docker-compose up -d ecommerce +docker compose up -d lms +docker compose up -d cms +docker compose up -d ecommerce sleep 5 # Give above services some time to boot up ./provision-ida.sh discovery discovery 18381 -docker-compose exec -T discovery bash -e -c 'rm -rf /edx/var/discovery/*' -docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --lms-coursemode-api-url "http://edx.devstack.lms:18000/api/course_modes/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/" --studio-url "http://edx.devstack.cms:18010/" --publisher-url "http://edx.devstack.frontend-app-publisher:18400/"' +docker compose exec -T discovery bash -e -c 'rm -rf /edx/var/discovery/*' +docker compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py create_or_update_partner --site-id 1 --site-domain localhost:18381 --code edx --name edX --courses-api-url "http://edx.devstack.lms:18000/api/courses/v1/" --lms-coursemode-api-url "http://edx.devstack.lms:18000/api/course_modes/v1/" --ecommerce-api-url "http://edx.devstack.ecommerce:18130/api/v2/" --organizations-api-url "http://edx.devstack.lms:18000/api/organizations/v0/" --lms-url "http://edx.devstack.lms:18000/" --studio-url "http://edx.devstack.cms:18010/" --publisher-url "http://edx.devstack.frontend-app-publisher:18400/"' set +e # FIXME[bash-e]: Bash scripts should use -e -- but this script fails # (after many retries) when trying to talk to ecommerce -docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' -docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py add_provisioning_data' +docker compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py refresh_course_metadata' +docker compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py add_provisioning_data' set -e -docker-compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' +docker compose exec -T discovery bash -e -c 'source /edx/app/discovery/discovery_env && python /edx/app/discovery/discovery/manage.py update_index --disable-change-limit' # Add demo program ./programs/provision.sh discovery diff --git a/provision-ecommerce.sh b/provision-ecommerce.sh index 85cda52306..13a6e094dc 100755 --- a/provision-ecommerce.sh +++ b/provision-ecommerce.sh @@ -8,6 +8,6 @@ set -x ./provision-ida.sh ecommerce ecommerce 18130 # Configure ecommerce -docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998' -docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' -docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' +docker compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email staff@example.com --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998' +docker compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only' +docker compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX' diff --git a/provision-forum.sh b/provision-forum.sh index 342aa49875..98a8104eed 100755 --- a/provision-forum.sh +++ b/provision-forum.sh @@ -2,5 +2,5 @@ set -eu -o pipefail set -x -docker-compose up -d forum -docker-compose exec -T forum bash -e -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/ && bin/rake search:initialize' +docker compose up -d forum +docker compose exec -T forum bash -e -c 'source /edx/app/forum/ruby_env && source /edx/app/forum/devstack_forum_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/ && bin/rake search:initialize' diff --git a/provision-ida-user.sh b/provision-ida-user.sh index ccc2ecf0dc..df68df6fce 100755 --- a/provision-ida-user.sh +++ b/provision-ida-user.sh @@ -13,8 +13,8 @@ client_port=$3 echo -e "${GREEN}Creating service user and OAuth2 applications for ${app_name}...${NC}" # Create the service user. -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1_worker $1_worker@example.com --staff --superuser' -- "$app_name" # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "http://localhost:$3/complete/edx-oauth2/" --client-id "$1-sso-key" --client-secret "$1-sso-secret" --scopes "user_id" $1-sso $1_worker' -- "$app_name" "$client_name" "$client_port" +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application --grant-type client-credentials --client-id "$1-backend-service-key" --client-secret "$1-backend-service-secret" $1-backend-service $1_worker' -- "$app_name" "$client_name" "$client_port" diff --git a/provision-ida.sh b/provision-ida.sh index 483b54ce3f..2236c65d75 100755 --- a/provision-ida.sh +++ b/provision-ida.sh @@ -12,17 +12,17 @@ container_name=${4:-$1} # (Optional) The name of the container. If missing, wil make dev.up.$app_name echo -e "${GREEN}Installing requirements for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" +docker compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$app_name" echo -e "${GREEN}Running migrations for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" +docker compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make migrate' -- "$app_name" echo -e "${GREEN}Creating super-user for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" +docker compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/$1/$1/manage.py shell' -- "$app_name" ./provision-ida-user.sh $app_name $client_name $client_port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${app_name}...${NC}" -docker-compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" +docker compose exec -T ${container_name} bash -e -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make static' -- "$app_name" diff --git a/provision-insights.sh b/provision-insights.sh index 864eb0a85e..17959eae1c 100755 --- a/provision-insights.sh +++ b/provision-insights.sh @@ -7,20 +7,20 @@ set -x name=insights port=18110 -docker-compose up -d insights +docker compose up -d insights echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights && make develop' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights && make develop' -- ${name} # # Install Insights npm dependencies -docker-compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights/ && npm ci && ./npm-post-install.sh' +docker compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights/ && npm ci && ./npm-post-install.sh' echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && export DJANGO_SETTINGS_MODULE="analytics_dashboard.settings.devstack" && cd /edx/app/insights/insights && make migrate' -- ${name} +docker compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && export DJANGO_SETTINGS_MODULE="analytics_dashboard.settings.devstack" && cd /edx/app/insights/insights && make migrate' -- ${name} ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights && make static' -- "$name" +docker compose exec -T ${name} bash -e -c 'source /edx/app/insights/insights_env && cd /edx/app/insights/insights && make static' -- "$name" diff --git a/provision-lms.sh b/provision-lms.sh index ffd48b9e14..78247d1147 100755 --- a/provision-lms.sh +++ b/provision-lms.sh @@ -12,47 +12,47 @@ cms_port=18010 # Bring edxapp containers online for app in "${apps[@]}"; do - docker-compose up -d $app + docker compose up -d $app done # install git for both LMS and CMS for app in "${apps[@]}"; do - docker-compose exec -T $app bash -e -c 'apt-get update && apt-get -y install --no-install-recommends git' + docker compose exec -T $app bash -e -c 'apt-get update && apt-get -y install --no-install-recommends git' - docker-compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' + docker compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs' #Installing prereqs crashes the process - docker-compose restart $app + docker compose restart $app done # Run edxapp migrations first since they are needed for the service users and OAuth clients # Make migrate runs migrations for both lms and cms. -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && make migrate' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && make migrate' -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py lms showmigrations --database student_module_history --traceback --pythonpath=. --settings devstack_docker' -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py lms migrate --database student_module_history --noinput --traceback --pythonpath=. --settings devstack_docker' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py lms showmigrations --database student_module_history --traceback --pythonpath=. --settings devstack_docker' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py lms migrate --database student_module_history --noinput --traceback --pythonpath=. --settings devstack_docker' -docker-compose exec -T cms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py cms showmigrations --database student_module_history --traceback --pythonpath=. --settings devstack_docker' -docker-compose exec -T cms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py cms migrate --database student_module_history --noinput --traceback --pythonpath=. --settings devstack_docker' +docker compose exec -T cms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py cms showmigrations --database student_module_history --traceback --pythonpath=. --settings devstack_docker' +docker compose exec -T cms bash -e -c 'source /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py cms migrate --database student_module_history --noinput --traceback --pythonpath=. --settings devstack_docker' # Create a superuser for edxapp -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell --settings=devstack_docker' # Create an enterprise service user for edxapp and give them appropriate permissions ./enterprise/provision.sh # Enable the LMS-E-Commerce integration -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce' # Create demo course and users -#docker-compose exec -T lms bash -e -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' +#docker compose exec -T lms bash -e -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"' if [[ ${DEVSTACK_SKIP_DEMO-false} == "true" ]] then echo "Skipping import of demo course. DEVSTACK_SKIP_DEMO is set to true" else - docker-compose exec -T lms bash -e -c 'git clone https://github.com/openedx/edx-demo-course.git /tmp/edx-demo-course' - docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /edx/var/edxapp/data /tmp/edx-demo-course && rm -rf /tmp/edx-demo-course' + docker compose exec -T lms bash -e -c 'git clone https://github.com/openedx/edx-demo-course.git /tmp/edx-demo-course' + docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py cms --settings=devstack_docker import /edx/var/edxapp/data /tmp/edx-demo-course && rm -rf /tmp/edx-demo-course' fi demo_hashed_password='pbkdf2_sha256$20000$TjE34FJjc3vv$0B7GUmH8RwrOc/BvMoxjb5j8EgnWTt3sxorDANeF7Qw=' @@ -60,24 +60,24 @@ for user in honor audit verified staff ; do email="$user@example.com" # Set staff flag for staff user if [[ $user == "staff" ]] ; then - docker-compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms manage_user $user $email --initial-password-hash '$demo_hashed_password' --staff" + docker compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms manage_user $user $email --initial-password-hash '$demo_hashed_password' --staff" else - docker-compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms manage_user $user $email --initial-password-hash '$demo_hashed_password'" + docker compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms manage_user $user $email --initial-password-hash '$demo_hashed_password'" fi if [[ "${DEVSTACK_SKIP_DEMO-false}" != "true" ]] then # Enroll users in the demo course - docker-compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms enroll_user_in_course -e $email -c course-v1:edX+DemoX+Demo_Course" + docker compose exec -T lms bash -e -c "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker --service-variant lms enroll_user_in_course -e $email -c course-v1:edX+DemoX+Demo_Course" fi done # Fix missing vendor file by clearing the cache -docker-compose exec -T lms bash -e -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' +docker compose exec -T lms bash -e -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1' # Create static assets for both LMS and CMS for app in "${apps[@]}"; do - docker-compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' + docker compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker' done # Allow LMS SSO for CMS diff --git a/provision-notes.sh b/provision-notes.sh index 81075a6499..8c824858db 100755 --- a/provision-notes.sh +++ b/provision-notes.sh @@ -9,24 +9,24 @@ name=edx_notes_api port=18734 client_name=edx-notes # The name of the Oauth client stored in the edxapp DB. -docker-compose up -d $name +docker compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make requirements' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make migrate' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cho "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/notes/manage.py shell' -- "$name" +docker compose exec -T ${name} bash -e -c 'cho "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\"edx\", \"edx@example.com\", \"edx\") if not User.objects.filter(username=\"edx\").exists() else None" | python /edx/app/notes/manage.py shell' -- "$name" ./provision-ida-user.sh $name $client_name $port # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make static' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/notes && make static' -- "$name" # This will build the elasticsearch index for notes. echo -e "${GREEN}Creating indexes for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/notes/ && python manage.py search_index --rebuild -f' +docker compose exec -T ${name} bash -e -c 'cd /edx/app/notes/ && python manage.py search_index --rebuild -f' diff --git a/provision-registrar.sh b/provision-registrar.sh index 4d695c747b..1e527eff3f 100755 --- a/provision-registrar.sh +++ b/provision-registrar.sh @@ -7,20 +7,20 @@ set -x name=registrar port=18734 -docker-compose up -d $name +docker compose up -d $name echo -e "${GREEN}Installing requirements for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make requirements' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make requirements' -- "$name" echo -e "${GREEN}Running migrations for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make migrate' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make migrate' -- "$name" echo -e "${GREEN}Creating super-user for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make createsuperuser' -- "$name" +docker compose exec -T ${name} bash -e -c 'cd /edx/app/registrar && make createsuperuser' -- "$name" ./provision-ida-user.sh ${name} ${name} ${port} # Compile static assets last since they are absolutely necessary for all services. This will allow developers to get # started if they do not care about static assets echo -e "${GREEN}Compiling static assets for ${name}...${NC}" -docker-compose exec -T ${name} bash -e -c ' if ! cd /edx/app/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" +docker compose exec -T ${name} bash -e -c ' if ! cd /edx/app/registrar && make static 2>registrar_make_static.err; then echo "------- Last 100 lines of stderr"; tail regsitrar_make_static.err -n 100; echo "-------"; fi;' -- "$name" diff --git a/provision-retirement-user.sh b/provision-retirement-user.sh index 2df6d814ee..812d92b1c5 100755 --- a/provision-retirement-user.sh +++ b/provision-retirement-user.sh @@ -9,5 +9,5 @@ app_name=$1 user_name=$2 echo -e "${GREEN}Creating retirement service user ${user_name} and DOT Application ${app_name}...${NC}" -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" -docker-compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user $1 $1@example.com --staff --superuser' -- "$user_name" +docker compose exec -T lms bash -e -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker create_dot_application $1 $2' -- "$app_name" "$user_name" diff --git a/provision-xqueue.sh b/provision-xqueue.sh index 8fde5dafce..d983cc57c2 100755 --- a/provision-xqueue.sh +++ b/provision-xqueue.sh @@ -3,11 +3,11 @@ set -eu -o pipefail set -x # Bring up XQueue, we don't need the consumer for provisioning -docker-compose up -d xqueue +docker compose up -d xqueue # Update dependencies -docker-compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && make requirements' +docker compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && make requirements' # Run migrations -docker-compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' +docker compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py migrate' # Add users that graders use to fetch data, there's one default user in Ansible which is part of our settings -docker-compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py update_users' +docker compose exec -T xqueue bash -e -c 'source /edx/app/xqueue/xqueue_env && cd /edx/app/xqueue/xqueue && python manage.py update_users' diff --git a/provision.sh b/provision.sh index 9e63f23c02..e22c97f670 100755 --- a/provision.sh +++ b/provision.sh @@ -122,15 +122,15 @@ fi echo -e "${GREEN}Will provision the following:\n ${to_provision_ordered}${NC}" # Bring the databases online. -docker-compose up -d mysql57 -docker-compose up -d mysql80 +docker compose up -d mysql57 +docker compose up -d mysql80 if needs_mongo "$to_provision_ordered"; then - docker-compose up -d mongo + docker compose up -d mongo fi # Ensure the MySQL5 server is online and usable echo "${GREEN}Waiting for MySQL 5.7.${NC}" -until docker-compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null +until docker compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -138,7 +138,7 @@ done # Ensure the MySQL8 server is online and usable echo "${GREEN}Waiting for MySQL 8.0.${NC}" -until docker-compose exec -T mysql80 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null +until docker compose exec -T mysql80 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -149,7 +149,7 @@ done sleep 10 echo "${GREEN}Waiting for MySQL 5.7 to restart.${NC}" -until docker-compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null +until docker compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -158,7 +158,7 @@ done echo -e "${GREEN}MySQL5 ready.${NC}" echo "${GREEN}Waiting for MySQL 8.0 to restart.${NC}" -until docker-compose exec -T mysql80 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null +until docker compose exec -T mysql80 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -169,23 +169,23 @@ echo -e "${GREEN}MySQL8 ready.${NC}" # Ensure that the MySQL databases and users are created for all IDAs. # (A no-op for databases and users that already exist). echo -e "${GREEN}Ensuring MySQL 5.7 databases and users exist...${NC}" -docker-compose exec -T mysql57 bash -e -c "mysql -uroot mysql" < provision.sql +docker compose exec -T mysql57 bash -e -c "mysql -uroot mysql" < provision.sql echo -e "${GREEN}Ensuring MySQL 8.0 databases and users exist...${NC}" -docker-compose exec -T mysql80 bash -e -c "mysql -uroot mysql" < provision-mysql80.sql +docker compose exec -T mysql80 bash -e -c "mysql -uroot mysql" < provision-mysql80.sql # If necessary, ensure the MongoDB server is online and usable # and create its users. if needs_mongo "$to_provision_ordered"; then echo -e "${GREEN}Waiting for MongoDB...${NC}" # mongo container and mongo process/shell inside the container - until docker-compose exec -T mongo mongo --eval "db.serverStatus()" &> /dev/null + until docker compose exec -T mongo mongo --eval "db.serverStatus()" &> /dev/null do printf "." sleep 1 done echo -e "${GREEN}MongoDB ready.${NC}" echo -e "${GREEN}Creating MongoDB users...${NC}" - docker-compose exec -T mongo bash -e -c "mongo" < mongo-provision.js + docker compose exec -T mongo bash -e -c "mongo" < mongo-provision.js else echo -e "${GREEN}MongoDB preparation not required; skipping.${NC}" fi diff --git a/requirements/base.in b/requirements/base.in index 8fce5cc217..1b9e6a1a3c 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,4 +1,4 @@ -c constraints.txt -docker-compose # For launching the devstack Docker containers -PyYAML # For parsing configuration files while generating offline installers +# Support for Apple Silicon begins with 6.0.0 +PyYAML>=6.0.0 # For parsing configuration files while generating offline installers diff --git a/requirements/base.txt b/requirements/base.txt index 5f746bdb4d..a0fda06220 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,69 +4,5 @@ # # make upgrade # -attrs==23.1.0 - # via jsonschema -bcrypt==4.0.1 - # via paramiko -certifi==2023.7.22 - # via requests -cffi==1.15.1 - # via - # cryptography - # pynacl -charset-normalizer==3.2.0 - # via requests -cryptography==41.0.3 - # via paramiko -distro==1.8.0 - # via docker-compose -docker[ssh]==6.1.3 - # via docker-compose -docker-compose==1.29.2 +pyyaml==6.0.1 # via -r requirements/base.in -dockerpty==0.4.1 - # via docker-compose -docopt==0.6.2 - # via docker-compose -idna==3.4 - # via requests -jsonschema==3.2.0 - # via docker-compose -packaging==23.1 - # via docker -paramiko==3.3.1 - # via docker -pycparser==2.21 - # via cffi -pynacl==1.5.0 - # via paramiko -pyrsistent==0.19.3 - # via jsonschema -python-dotenv==0.21.1 - # via docker-compose -pyyaml==5.4.1 - # via - # -r requirements/base.in - # docker-compose -requests==2.31.0 - # via - # docker - # docker-compose -six==1.16.0 - # via - # dockerpty - # jsonschema - # websocket-client -texttable==1.6.7 - # via docker-compose -urllib3==2.0.4 - # via - # docker - # requests -websocket-client==0.59.0 - # via - # docker - # docker-compose - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/dev.txt b/requirements/dev.txt index 3ff53a1a5b..7d6e5ab75c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,71 +4,16 @@ # # make upgrade # -attrs==23.1.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # jsonschema -bcrypt==4.0.1 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # paramiko build==1.0.0 # via # -r requirements/pip-tools.txt # pip-tools -certifi==2023.7.22 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # requests -cffi==1.15.1 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # cryptography - # pynacl -charset-normalizer==3.2.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # requests click==8.1.7 # via # -r requirements/pip-tools.txt # pip-tools -cryptography==41.0.3 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # paramiko distlib==0.3.7 # via virtualenv -distro==1.8.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose -docker[ssh]==6.1.3 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose -docker-compose==1.29.2 - # via - # -r requirements/base.txt - # -r requirements/test.txt -dockerpty==0.4.1 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose -docopt==0.6.2 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose exceptiongroup==1.1.3 # via # -r requirements/test.txt @@ -77,11 +22,6 @@ filelock==3.12.3 # via # tox # virtualenv -idna==3.4 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # requests importlib-metadata==6.8.0 # via # -r requirements/pip-tools.txt @@ -90,25 +30,13 @@ iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -jsonschema==3.2.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose packaging==23.1 # via - # -r requirements/base.txt # -r requirements/pip-tools.txt # -r requirements/test.txt # build - # docker # pytest # tox -paramiko==3.3.1 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker pexpect==4.8.0 # via -r requirements/test.txt pip-tools==7.3.0 @@ -126,56 +54,18 @@ ptyprocess==0.7.0 # pexpect py==1.11.0 # via tox -pycparser==2.21 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # cffi -pynacl==1.5.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # paramiko pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pyrsistent==0.19.3 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # jsonschema pytest==7.4.1 # via -r requirements/test.txt -python-dotenv==0.21.1 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose -pyyaml==5.4.1 +pyyaml==6.0.1 # via # -r requirements/base.txt # -r requirements/test.txt - # docker-compose -requests==2.31.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker - # docker-compose six==1.16.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # dockerpty - # jsonschema - # tox - # websocket-client -texttable==1.6.7 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker-compose + # via tox tomli==2.0.1 # via # -r requirements/pip-tools.txt @@ -194,20 +84,8 @@ tox-battery==0.6.2 # via -r requirements/dev.in typing-extensions==4.7.1 # via filelock -urllib3==2.0.4 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker - # requests virtualenv==20.24.4 # via tox -websocket-client==0.59.0 - # via - # -r requirements/base.txt - # -r requirements/test.txt - # docker - # docker-compose wheel==0.41.2 # via # -r requirements/pip-tools.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index 4806175b39..eb0767effb 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -8,59 +8,20 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -attrs==23.1.0 - # via - # -r requirements/base.txt - # jsonschema babel==2.12.1 # via # pydata-sphinx-theme # sphinx -bcrypt==4.0.1 - # via - # -r requirements/base.txt - # paramiko beautifulsoup4==4.12.2 # via pydata-sphinx-theme bleach==6.0.0 # via readme-renderer certifi==2023.7.22 - # via - # -r requirements/base.txt - # requests -cffi==1.15.1 - # via - # -r requirements/base.txt - # cryptography - # pynacl + # via requests charset-normalizer==3.2.0 - # via - # -r requirements/base.txt - # requests -cryptography==41.0.3 - # via - # -r requirements/base.txt - # paramiko -distro==1.8.0 - # via - # -r requirements/base.txt - # docker-compose + # via requests doc8==1.1.1 # via -r requirements/doc.in -docker[ssh]==6.1.3 - # via - # -r requirements/base.txt - # docker-compose -docker-compose==1.29.2 - # via -r requirements/base.txt -dockerpty==0.4.1 - # via - # -r requirements/base.txt - # docker-compose -docopt==0.6.2 - # via - # -r requirements/base.txt - # docker-compose docutils==0.19 # via # doc8 @@ -69,37 +30,21 @@ docutils==0.19 # restructuredtext-lint # sphinx idna==3.4 - # via - # -r requirements/base.txt - # requests + # via requests imagesize==1.4.1 # via sphinx importlib-metadata==6.8.0 # via sphinx jinja2==3.1.2 # via sphinx -jsonschema==3.2.0 - # via - # -r requirements/base.txt - # docker-compose markupsafe==2.1.3 # via jinja2 packaging==23.1 # via - # -r requirements/base.txt - # docker # pydata-sphinx-theme # sphinx -paramiko==3.3.1 - # via - # -r requirements/base.txt - # docker pbr==5.11.1 # via stevedore -pycparser==2.21 - # via - # -r requirements/base.txt - # cffi pydata-sphinx-theme==0.13.3 # via sphinx-book-theme pygments==2.16.1 @@ -109,41 +54,18 @@ pygments==2.16.1 # pydata-sphinx-theme # readme-renderer # sphinx -pynacl==1.5.0 - # via - # -r requirements/base.txt - # paramiko -pyrsistent==0.19.3 - # via - # -r requirements/base.txt - # jsonschema -python-dotenv==0.21.1 - # via - # -r requirements/base.txt - # docker-compose pytz==2023.3.post1 # via babel -pyyaml==5.4.1 - # via - # -r requirements/base.txt - # docker-compose +pyyaml==6.0.1 + # via -r requirements/base.txt readme-renderer==41.0 # via -r requirements/doc.in requests==2.31.0 - # via - # -r requirements/base.txt - # docker - # docker-compose - # sphinx + # via sphinx restructuredtext-lint==1.4.0 # via doc8 six==1.16.0 - # via - # -r requirements/base.txt - # bleach - # dockerpty - # jsonschema - # websocket-client + # via bleach snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 @@ -169,28 +91,13 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx stevedore==5.1.0 # via doc8 -texttable==1.6.7 - # via - # -r requirements/base.txt - # docker-compose tomli==2.0.1 # via doc8 typing-extensions==4.7.1 # via pydata-sphinx-theme urllib3==2.0.4 - # via - # -r requirements/base.txt - # docker - # requests + # via requests webencodings==0.5.1 # via bleach -websocket-client==0.59.0 - # via - # -r requirements/base.txt - # docker - # docker-compose zipp==3.16.2 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/test.txt b/requirements/test.txt index caf1c195d1..dc33211a25 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,125 +4,21 @@ # # make upgrade # -attrs==23.1.0 - # via - # -r requirements/base.txt - # jsonschema -bcrypt==4.0.1 - # via - # -r requirements/base.txt - # paramiko -certifi==2023.7.22 - # via - # -r requirements/base.txt - # requests -cffi==1.15.1 - # via - # -r requirements/base.txt - # cryptography - # pynacl -charset-normalizer==3.2.0 - # via - # -r requirements/base.txt - # requests -cryptography==41.0.3 - # via - # -r requirements/base.txt - # paramiko -distro==1.8.0 - # via - # -r requirements/base.txt - # docker-compose -docker[ssh]==6.1.3 - # via - # -r requirements/base.txt - # docker-compose -docker-compose==1.29.2 - # via -r requirements/base.txt -dockerpty==0.4.1 - # via - # -r requirements/base.txt - # docker-compose -docopt==0.6.2 - # via - # -r requirements/base.txt - # docker-compose exceptiongroup==1.1.3 # via pytest -idna==3.4 - # via - # -r requirements/base.txt - # requests iniconfig==2.0.0 # via pytest -jsonschema==3.2.0 - # via - # -r requirements/base.txt - # docker-compose packaging==23.1 - # via - # -r requirements/base.txt - # docker - # pytest -paramiko==3.3.1 - # via - # -r requirements/base.txt - # docker + # via pytest pexpect==4.8.0 # via -r requirements/test.in pluggy==1.3.0 # via pytest ptyprocess==0.7.0 # via pexpect -pycparser==2.21 - # via - # -r requirements/base.txt - # cffi -pynacl==1.5.0 - # via - # -r requirements/base.txt - # paramiko -pyrsistent==0.19.3 - # via - # -r requirements/base.txt - # jsonschema pytest==7.4.1 # via -r requirements/test.in -python-dotenv==0.21.1 - # via - # -r requirements/base.txt - # docker-compose -pyyaml==5.4.1 - # via - # -r requirements/base.txt - # docker-compose -requests==2.31.0 - # via - # -r requirements/base.txt - # docker - # docker-compose -six==1.16.0 - # via - # -r requirements/base.txt - # dockerpty - # jsonschema - # websocket-client -texttable==1.6.7 - # via - # -r requirements/base.txt - # docker-compose +pyyaml==6.0.1 + # via -r requirements/base.txt tomli==2.0.1 # via pytest -urllib3==2.0.4 - # via - # -r requirements/base.txt - # docker - # requests -websocket-client==0.59.0 - # via - # -r requirements/base.txt - # docker - # docker-compose - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/tests/warn_default.py b/tests/warn_default.py index c02a275a95..7645cd22f5 100644 --- a/tests/warn_default.py +++ b/tests/warn_default.py @@ -14,7 +14,7 @@ def test_warn_default(): p.expect(r'Are you sure you want to run this command') p.sendline('') - p.expect(r'Pulling lms') + p.expect(r'docker compose pull --include-deps') # Send ^C, don't wait for it to finish p.sendintr()