Skip to content

Commit

Permalink
chore: remove metrics work from devstack
Browse files Browse the repository at this point in the history
  • Loading branch information
huniafatima-arbi committed Oct 21, 2024
1 parent 309f147 commit d241277
Show file tree
Hide file tree
Showing 10 changed files with 47 additions and 891 deletions.
60 changes: 9 additions & 51 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -163,17 +163,11 @@ dev.checkout: ## Check out "openedx-release/$OPENEDX_RELEASE" in each repo if se

dev.clone: dev.clone.ssh ## Clone service repos to the parent directory.

impl-dev.clone.https: ## Clone service repos using HTTPS method to the parent directory.
./repo.sh clone

dev.clone.https: ## Clone service repos using HTTPS method to the parent directory.
@scripts/send_metrics.py wrap "$@"

impl-dev.clone.ssh: ## Clone service repos using SSH method to the parent directory.
./repo.sh clone_ssh
./repo.sh clone

dev.clone.ssh: ## Clone service repos using SSH method to the parent directory.
@scripts/send_metrics.py wrap "$@"
./repo.sh clone_ssh

########################################################################################
# Developer interface: Docker image management.
Expand All @@ -186,46 +180,30 @@ dev.prune: ## Prune dangling docker images, containers, and networks. Useful whe
dev.pull.without-deps: _expects-service-list.dev.pull.without-deps

dev.pull.without-deps.%: ## Pull latest Docker images for specific services.
@scripts/send_metrics.py wrap "dev.pull.without-deps.$*"

impl-dev.pull.without-deps.%: ## Pull latest Docker images for specific services.
docker compose pull $$(echo $* | tr + " ")

dev.pull:
@scripts/send_metrics.py wrap "$@"

impl-dev.pull:
@scripts/make_warn_default_large.sh "dev.pull"

dev.pull.large-and-slow: dev.pull.$(DEFAULT_SERVICES) ## Pull latest Docker images required by default services.
@echo # at least one statement so that dev.pull.% doesn't run too

# Wildcards must be below anything they could match
dev.pull.%: ## Pull latest Docker images for services and their dependencies.
@scripts/send_metrics.py wrap "dev.pull.$*"

impl-dev.pull.%: ## Pull latest Docker images for services and their dependencies.
docker compose pull --include-deps $$(echo $* | tr + " ")

########################################################################################
# Developer interface: Database management.
########################################################################################

impl-dev.provision: ## Provision dev environment with default services, and then stop them.
dev.provision: ## Provision dev environment with default services, and then stop them.
make dev.check-memory
$(WINPTY) bash ./provision.sh $(DEFAULT_SERVICES)
make dev.stop

dev.provision: ## Provision dev environment with default services, and then stop them.
@scripts/send_metrics.py wrap "$@"

impl-dev.provision.%: dev.check-memory ## Provision specified services.
dev.provision.%: dev.check-memory ## Provision specified services.
echo $*
$(WINPTY) bash ./provision.sh $*

dev.provision.%: ## Provision specified services.
@scripts/send_metrics.py wrap "dev.provision.$*"

dev.backup: dev.up.mysql57+mysql80+mongo+elasticsearch710+opensearch12+coursegraph ## Write all data volumes to the host.
docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql57) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql57.tar.gz /var/lib/mysql
docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql80) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql80.tar.gz /var/lib/mysql
Expand Down Expand Up @@ -273,11 +251,8 @@ dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mys

dev.up.attach: _expects-service.dev.up.attach

impl-dev.up.attach.%: ## Bring up a service and its dependencies + and attach to it.
docker compose up $*

dev.up.attach.%: ## Bring up a service and its dependencies + and attach to it.
@scripts/send_metrics.py wrap "dev.up.attach.$*"
docker compose up $*

dev.up.shell: _expects-service.dev.up.shell

Expand All @@ -297,12 +272,9 @@ dev.up.with-watchers.%: ## Bring up services and their dependencies + asset watc

dev.up.without-deps: _expects-service-list.dev.up.without-deps

impl-dev.up.without-deps.%: dev.check-memory ## Bring up services by themselves.
dev.up.without-deps.%: dev.check-memory ## Bring up services by themselves.
docker compose up -d --no-deps $$(echo $* | tr + " ")

dev.up.without-deps.%: ## Bring up services by themselves.
@scripts/send_metrics.py wrap "dev.up.without-deps.$*"

dev.up.without-deps.shell: _expects-service.dev.up.without-deps.shell

dev.up.without-deps.shell.%: ## Bring up a service by itself + shell into it.
Expand All @@ -315,15 +287,12 @@ dev.up:
dev.up.large-and-slow: dev.up.$(DEFAULT_SERVICES) ## Bring up default services.
@echo # at least one statement so that dev.up.% doesn't run too

impl-dev.up.%: dev.check-memory ## Bring up services and their dependencies.
dev.up.%: dev.check-memory ## Bring up services and their dependencies.
docker compose up -d $$(echo $* | tr + " ")
ifeq ($(ALWAYS_CACHE_PROGRAMS),true)
make dev.cache-programs
endif

# Wildcards must be below anything they could match
dev.up.%:
@scripts/send_metrics.py wrap "dev.up.$*"

dev.ps: ## View list of created services and their statuses.
docker compose ps
Expand Down Expand Up @@ -493,10 +462,10 @@ $(foreach asset_service,$(ASSET_SERVICES_LIST),\
dev.static: | $(_asset_compilation_targets)

dev.static.lms:
docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets lms'
docker compose exec -T lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && npm run build && ./manage.py lms collectstatic --noinput'

dev.static.cms:
docker compose exec -T cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_assets cms'
docker compose exec -T cms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && npm run build && ./manage.py cms collectstatic --no-input'

dev.static.%: ## Rebuild static assets for the specified service's container.
docker compose exec -T $* bash -c 'source /edx/app/$*/$*_env && cd /edx/app/$*/$*/ && make static'
Expand Down Expand Up @@ -599,17 +568,6 @@ _expects-database.%:
@echo " make $*.edxapp"


########################################################################################
# Convenient ways to opt in or out of devstack usage metrics reporting
########################################################################################

metrics-opt-in: ## To opt into basic data collection to help improve devstack
@./scripts/send_metrics.py opt-in

metrics-opt-out: ## To opt out of metrics data collection
@./scripts/send_metrics.py opt-out


########################################################################################
# Miscellaneous targets.
# These are useful, but don't fit nicely to the greater Devstack interface.
Expand Down
4 changes: 2 additions & 2 deletions docker-compose-watchers.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
services:
lms_watcher:
command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done'
command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do npm run watch --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done'
container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher"
environment:
FRONTEND_TEST_SERVER_HOSTNAME: edx.devstack.lms_watcher
Expand All @@ -18,7 +18,7 @@ services:
- edx.devstack.lms_watcher

cms_watcher:
command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done'
command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do npm run watch --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done'
container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.cms_watcher"
environment:
FRONTEND_TEST_SERVER_HOSTNAME: edx.devstack.cms_watcher
Expand Down
4 changes: 2 additions & 2 deletions docs/devstack_faq.rst
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ Unlike the ``node_modules`` directory, the ``virtualenv`` used to run Python
code in a Docker container only exists inside that container. Changes made to
a container's filesystem are not saved when the container exits, so if you
manually install or upgrade Python packages in a container (via
``pip install``, ``paver install_python_prereqs``, etc.), they will no
``pip install``, ``pip install -r requirements/edx/base.txt``, etc.), they will no
longer be present if you restart the container. (Devstack Docker containers
lose changes made to the filesystem when you reboot your computer, run
``make remove-containers``, restart or upgrade Docker itself, etc.) If you want to ensure
Expand Down Expand Up @@ -78,7 +78,7 @@ JavaScript packages for Node.js are installed into the ``node_modules``
directory of the local git repository checkout which is synced into the
corresponding Docker container. Hence these can be upgraded via any of the
usual methods for that service (``npm install``,
``paver install_node_prereqs``, etc.), and the changes will persist between
``npm clean-install``, etc.), and the changes will persist between
container restarts.

How do I rebuild static assets?
Expand Down
4 changes: 2 additions & 2 deletions docs/service_list.rst
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@ Some common service combinations include:
.. _frontend-app-publisher: https://github.com/openedx/frontend-app-publisher
.. _frontend-app-gradebook: https://github.com/openedx/frontend-app-gradebook
.. _lms: https://github.com/openedx/edx-platform
.. _frontend-app-program-console: https://github.com/openedx/frontend-app-program-console
.. _registrar: https://github.com/openedx/registrar
.. _frontend-app-program-console: https://github.com/edx/frontend-app-program-console
.. _registrar: https://github.com/edx/registrar
.. _cms: https://github.com/openedx/edx-platform
.. _frontend-app-learner-dashboard: https://github.com/openedx/frontend-app-learner-dashboard
.. _frontend-app-learner-record: https://github.com/openedx/frontend-app-learner-record
Expand Down
15 changes: 2 additions & 13 deletions docs/testing_and_debugging.rst
Original file line number Diff line number Diff line change
Expand Up @@ -49,19 +49,8 @@ Running LMS and CMS Tests
----------------------------

After entering a shell for the appropriate service via ``make lms-shell`` or
``make cms-shell``, you can run any of the usual paver commands from the
`edx-platform testing documentation`_. Examples:

.. code:: sh
paver run_quality
paver test_a11y
paver test_bokchoy
paver test_js
paver test_lib
paver test_python
Tests can also be run individually. Example:
``make cms-shell``, you can run commands from the `edx-platform testing documentation`_
Tests can be run individually. Example:

.. code:: sh
Expand Down
8 changes: 4 additions & 4 deletions docs/troubleshoot_general_tips.rst
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,13 @@ To fix this situation, change the owner back to yourself outside of the containe
Running LMS commands within a container
---------------------------------------

Most of the ``paver`` commands require a settings flag. If omitted, the flag defaults to
``devstack``. If you run into issues running ``paver`` commands in a docker container, you should append
Most of the commands require a settings flag. If omitted, the flag defaults to
``devstack``. If you run into issues running commands in a docker container, you should append
the ``devstack_docker`` flag. For example:

.. code:: sh
$ paver update_assets --settings=devstack_docker
$ npm run build && ./manage.py lms collectstatic --no-input --settings=devstack_docker
Resource busy or locked
-----------------------
Expand Down Expand Up @@ -266,7 +266,7 @@ The fix is to get a new auth session. You can do any of the following:
Missing vendor file node_modules/backbone.paginator/lib/backbone.paginator.js
-----------------------------------------------------------------------------
This message sometimes appears when provisioning. The root cause of this is as yet unknown but the most effective workaround seems to be
to shell into the LMS (``make lms-shell`` in devstack) and run ``npm ci``, followed by ``paver update_assets``.
to shell into the LMS (``make lms-shell`` in devstack) and run ``npm ci``, followed by ``npm run build && ./manage.py lms collectstatic --no-input && ./manage.py cms collectstatic``.
See `the github issue`_ to follow the work being done on the resolution.

.. _the github issue: https://github.com/openedx/devstack/issues/1072
Expand Down
6 changes: 3 additions & 3 deletions provision-lms.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ done
for app in "${apps[@]}"; do
docker compose exec -T $app bash -e -c 'apt-get update && apt-get -y install --no-install-recommends git'

docker compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 paver install_prereqs'
docker compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PYTHON_UNINSTALL=1 pip install -e . -r requirements/edx/base.txt -r requirements/edx/assets.txt && npm clean-install'

#Installing prereqs crashes the process
docker compose restart $app
Expand Down Expand Up @@ -75,11 +75,11 @@ done


# Fix missing vendor file by clearing the cache
docker compose exec -T lms bash -e -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1'
# docker compose exec -T lms bash -e -c 'rm /edx/app/edxapp/edx-platform/.prereqs_cache/Node_prereqs.sha1'

# Create static assets for both LMS and CMS
for app in "${apps[@]}"; do
docker compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker'
docker compose exec -T $app bash -e -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && npm run build && ./manage.py lms collectstatic --noinput --settings devstack_docker && ./manage.py cms collectstatic --noinput --settings devstack_docker'
done

# Allow LMS SSO for CMS
Expand Down
34 changes: 23 additions & 11 deletions repo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ repos=(
"https://github.com/openedx/xqueue.git"
"https://github.com/edx/edx-analytics-dashboard.git"
"https://github.com/openedx/frontend-app-gradebook.git"
"https://github.com/openedx/frontend-app-learner-dashboard"
"https://github.com/openedx/frontend-app-learner-record"
"https://github.com/openedx/frontend-app-learner-dashboard.git"
"https://github.com/openedx/frontend-app-learner-record.git"
"https://github.com/edx/frontend-app-payment.git"
"https://github.com/openedx/frontend-app-publisher.git"
"https://github.com/edx/edx-analytics-dashboard.git"
Expand All @@ -43,8 +43,8 @@ non_release_repos=(
"https://github.com/openedx/frontend-app-course-authoring.git"
"https://github.com/openedx/frontend-app-learning.git"
"https://github.com/openedx/frontend-app-library-authoring.git"
"https://github.com/openedx/registrar.git"
"https://github.com/openedx/frontend-app-program-console.git"
"https://github.com/edx/registrar.git"
"https://github.com/edx/frontend-app-program-console.git"
"https://github.com/openedx/frontend-app-account.git"
"https://github.com/openedx/frontend-app-profile.git"
"https://github.com/openedx/frontend-app-ora-grading.git"
Expand Down Expand Up @@ -73,8 +73,8 @@ non_release_ssh_repos=(
"[email protected]:openedx/frontend-app-course-authoring.git"
"[email protected]:openedx/frontend-app-learning.git"
"[email protected]:openedx/frontend-app-library-authoring.git"
"[email protected]:openedx/registrar.git"
"[email protected]:openedx/frontend-app-program-console.git"
"[email protected]:edx/registrar.git"
"[email protected]:edx/frontend-app-program-console.git"
"[email protected]:openedx/frontend-app-account.git"
"[email protected]:openedx/frontend-app-profile.git"
"[email protected]:openedx/frontend-app-ora-grading.git"
Expand All @@ -87,7 +87,7 @@ else
ssh_repos+=("${non_release_ssh_repos[@]}")
fi

name_pattern=".*/(.*).git"
name_pattern=".*/(.*).git$"

_checkout ()
{
Expand All @@ -97,7 +97,10 @@ _checkout ()
do
# Use Bash's regex match operator to capture the name of the repo.
# Results of the match are saved to an array called $BASH_REMATCH.
[[ $repo =~ $name_pattern ]]
if [[ ! $repo =~ $name_pattern ]]; then
echo "Cannot perform checkout on repo; URL did not match expected pattern: $repo"
exit 1
fi
name="${BASH_REMATCH[1]}"

# If a directory exists and it is nonempty, assume the repo has been cloned.
Expand All @@ -122,7 +125,10 @@ _clone ()
do
# Use Bash's regex match operator to capture the name of the repo.
# Results of the match are saved to an array called $BASH_REMATCH.
[[ $repo =~ $name_pattern ]]
if [[ ! $repo =~ $name_pattern ]]; then
echo "Cannot clone repo; URL did not match expected pattern: $repo"
exit 1
fi
name="${BASH_REMATCH[1]}"

# If a directory exists and it is nonempty, assume the repo has been checked out
Expand Down Expand Up @@ -196,7 +202,10 @@ reset ()

for repo in ${repos[*]}
do
[[ $repo =~ $name_pattern ]]
if [[ ! $repo =~ $name_pattern ]]; then
echo "Cannot reset repo; URL did not match expected pattern: $repo"
exit 1
fi
name="${BASH_REMATCH[1]}"

if [ -d "$name" ]; then
Expand Down Expand Up @@ -227,7 +236,10 @@ status ()
currDir=$(pwd)
for repo in ${repos[*]}
do
[[ $repo =~ $name_pattern ]]
if [[ ! $repo =~ $name_pattern ]]; then
echo "Cannot check repo status; URL did not match expected pattern: $repo"
exit 1
fi
name="${BASH_REMATCH[1]}"

if [ -d "$name" ]; then
Expand Down
Loading

0 comments on commit d241277

Please sign in to comment.