From ea65f074b3cc5d8c70940f50ce0b032779e3fc8b Mon Sep 17 00:00:00 2001 From: Maari Tamm Date: Tue, 26 Mar 2024 10:19:55 +0100 Subject: [PATCH] feat: Drop the use of the Tubular repository The Tubular repository has been deplrecated and the relevant scripts have been moved to the edx-platform codebase. * Drop the custom image build and rely on the openedx image. * Add support for Tutor 18 / Open edX Redwood (breaking change) Fixes: #32 --- CHANGELOG.md | 7 +++++++ README.md | 1 + setup.py | 2 +- tutorretirement/patches/k8s-jobs | 12 ++++++++--- .../patches/kustomization-configmapgenerator | 3 ++- .../local-docker-compose-jobs-services | 4 ++-- tutorretirement/plugin.py | 20 ++----------------- .../retirement}/config.yml | 0 .../retirement/run_retirement_pipeline.sh | 13 ++++++++++++ .../retirement/build/retirement/Dockerfile | 11 ---------- .../retirement/run_retirement_pipeline.sh | 12 ----------- 11 files changed, 37 insertions(+), 48 deletions(-) rename tutorretirement/templates/retirement/{build/retirement/pipeline_config => apps/retirement}/config.yml (100%) create mode 100644 tutorretirement/templates/retirement/apps/retirement/run_retirement_pipeline.sh delete mode 100644 tutorretirement/templates/retirement/build/retirement/Dockerfile delete mode 100644 tutorretirement/templates/retirement/build/retirement/run_retirement_pipeline.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c1f74d..b59f219 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## Unreleased + +* [BREAKING CHANGE] Add support for Tutor 18 and Open edX Redwood. + The Tubular repository has been deprecated and the relevant scripts + have been moved to the `edx-platform` repository (https://github.com/openedx/axim-engineering/issues/881). Thus, starting with Tutor 18, this plugin no longer requires a custom + image build and the `retirement-job` will be run based on the `openedx` image. + ## Version 3.3.1 (2024-07-03) * [Bug fix] Drop support for retiring users in the [Open edX E-Commerce Service](https://github.com/openedx/ecommerce). diff --git a/README.md b/README.md index 67c8222..64d8b30 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ appropriate one: | Olive | `>=15.0, <16` | `main` | 2.x.x | | Palm | `>=16.0, <17` | `main` | 3.x.x | | Quince | `>=17.0, <18` | `main` | 3.x.x | +| Redwood | `>=18.0, <19` | `main` | 4.x.x | [^1]: For Open edX Maple and Tutor 13, you must run version 13.2.0 or  later. That is because this plugin uses the Tutor v1 plugin API, diff --git a/setup.py b/setup.py index 620693e..770dd2d 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def load_readme(): packages=find_packages(exclude=["tests*"]), include_package_data=True, python_requires=">=3.8", - install_requires=["tutor <18, >=16.1.7"], + install_requires=["tutor <19, >=16.1.7"], setup_requires=["setuptools-scm"], entry_points={ "tutor.plugin.v1": [ diff --git a/tutorretirement/patches/k8s-jobs b/tutorretirement/patches/k8s-jobs index 82919d2..4ed75c1 100644 --- a/tutorretirement/patches/k8s-jobs +++ b/tutorretirement/patches/k8s-jobs @@ -15,17 +15,23 @@ spec: spec: containers: - name: retirement - image: {{ RETIREMENT_DOCKER_IMAGE }} + image: {{ DOCKER_IMAGE_OPENEDX }} command: - 'bash' - '-e' - - 'run_retirement_pipeline.sh' + - 'scripts/user_retirement/plugin/run_retirement_pipeline.sh' - '{{ RETIREMENT_COOL_OFF_DAYS }}' volumeMounts: + - name: pipeline-run + mountPath: 'scripts/user_retirement/plugin/run_retirement_pipeline.sh' + subPath: run_retirement_pipeline.sh - name: pipeline-config - mountPath: '/tubular/pipeline_config/config.yml' + mountPath: 'scripts/user_retirement/plugin/config.yml' subPath: config.yml volumes: + - name: pipeline-run + configMap: + name: retirement-pipeline-configmap - name: pipeline-config configMap: name: retirement-pipeline-configmap diff --git a/tutorretirement/patches/kustomization-configmapgenerator b/tutorretirement/patches/kustomization-configmapgenerator index 2c92c66..fa109b5 100644 --- a/tutorretirement/patches/kustomization-configmapgenerator +++ b/tutorretirement/patches/kustomization-configmapgenerator @@ -1,3 +1,4 @@ - name: retirement-pipeline-configmap files: - - plugins/retirement/build/retirement/pipeline_config/config.yml + - plugins/retirement/apps/retirement/config.yml + - plugins/retirement/apps/retirement/run_retirement_pipeline.sh diff --git a/tutorretirement/patches/local-docker-compose-jobs-services b/tutorretirement/patches/local-docker-compose-jobs-services index e8b97db..3825494 100644 --- a/tutorretirement/patches/local-docker-compose-jobs-services +++ b/tutorretirement/patches/local-docker-compose-jobs-services @@ -1,7 +1,7 @@ ############# Retirement worker service retirement-job: - image: {{ RETIREMENT_DOCKER_IMAGE }} + image: {{ DOCKER_IMAGE_OPENEDX }} volumes: - - ../plugins/retirement/build/retirement/pipeline_config:/tubular/pipeline_config + - ../plugins/retirement/apps/retirement:/openedx/edx-platform/scripts/user_retirement/plugin depends_on: - lms diff --git a/tutorretirement/plugin.py b/tutorretirement/plugin.py index 3ad7fd9..3cc9b9a 100644 --- a/tutorretirement/plugin.py +++ b/tutorretirement/plugin.py @@ -17,31 +17,14 @@ }, "defaults": { "VERSION": __version__, - "DOCKER_IMAGE": "{{ DOCKER_REGISTRY }}retirement:{{ RETIREMENT_VERSION }}", # noqa: E501 "EDX_OAUTH2_CLIENT_ID": "retirement_service_worker", "COOL_OFF_DAYS": 30, - "TUBULAR_VERSION": "{{ OPENEDX_COMMON_VERSION }}", "K8S_CRONJOB_HISTORYLIMIT_FAILURE": 1, "K8S_CRONJOB_HISTORYLIMIT_SUCCESS": 3, "K8S_CRONJOB_SCHEDULE": "0 0 * * *", }, } -hooks.Filters.IMAGES_BUILD.add_item(( - "retirement", - ("plugins", "retirement", "build", "retirement"), - "{{ RETIREMENT_DOCKER_IMAGE }}", - (), -)) -hooks.Filters.IMAGES_PULL.add_item(( - "retirement", - "{{ RETIREMENT_DOCKER_IMAGE }}", -)) -hooks.Filters.IMAGES_PUSH.add_item(( - "retirement", - "{{ RETIREMENT_DOCKER_IMAGE }}", -)) - @local_command_group.command(help="Run the retirement pipeline") @click.pass_obj @@ -51,7 +34,8 @@ def retire_users(context): cool_off_days = config["RETIREMENT_COOL_OFF_DAYS"] job_runner.run_task( service="retirement", - command=f"bash -e run_retirement_pipeline.sh {cool_off_days}" + command="bash -e scripts/user_retirement/plugin" + f"/run_retirement_pipeline.sh {cool_off_days}" ) diff --git a/tutorretirement/templates/retirement/build/retirement/pipeline_config/config.yml b/tutorretirement/templates/retirement/apps/retirement/config.yml similarity index 100% rename from tutorretirement/templates/retirement/build/retirement/pipeline_config/config.yml rename to tutorretirement/templates/retirement/apps/retirement/config.yml diff --git a/tutorretirement/templates/retirement/apps/retirement/run_retirement_pipeline.sh b/tutorretirement/templates/retirement/apps/retirement/run_retirement_pipeline.sh new file mode 100644 index 0000000..c7d4aa6 --- /dev/null +++ b/tutorretirement/templates/retirement/apps/retirement/run_retirement_pipeline.sh @@ -0,0 +1,13 @@ +cool_off_days=$1 +pip install --upgrade pip && pip install -r scripts/user_retirement/requirements/base.txt --exists-action w +python scripts/user_retirement/get_learners_to_retire.py --config_file=scripts/user_retirement/plugin/config.yml --output_dir=scripts/user_retirement/plugin/learners_to_retire --cool_off_days="$cool_off_days" +for filename in scripts/user_retirement/plugin/learners_to_retire/*;do + if test ! -f "$filename";then + echo "No users to retire" + continue + else + IFS="=" read -r name username <"$filename" + echo "Retiring $username" + python scripts/user_retirement/retire_one_learner.py --config_file=scripts/user_retirement/plugin/config.yml --username="$username" + fi +done diff --git a/tutorretirement/templates/retirement/build/retirement/Dockerfile b/tutorretirement/templates/retirement/build/retirement/Dockerfile deleted file mode 100644 index ea20f04..0000000 --- a/tutorretirement/templates/retirement/build/retirement/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM python:3.8 -ENV PYTHONUNBUFFERED 1 -RUN python3 -m venv /retirement/venv/ -ENV PATH "/retirement/venv/bin:$PATH" -RUN apt-get update && \ - git clone --depth 1 --branch {{ RETIREMENT_TUBULAR_VERSION }} https://github.com/openedx/tubular.git ./tubular -WORKDIR ./tubular -COPY run_retirement_pipeline.sh . -RUN pip install --upgrade pip && \ - pip install -r requirements.txt --exists-action w && \ - mkdir learners_to_retire diff --git a/tutorretirement/templates/retirement/build/retirement/run_retirement_pipeline.sh b/tutorretirement/templates/retirement/build/retirement/run_retirement_pipeline.sh deleted file mode 100644 index 8bcfbd5..0000000 --- a/tutorretirement/templates/retirement/build/retirement/run_retirement_pipeline.sh +++ /dev/null @@ -1,12 +0,0 @@ -cool_off_days=$1 -python scripts/get_learners_to_retire.py --config_file=./pipeline_config/config.yml --output_dir=learners_to_retire --cool_off_days="$cool_off_days" -for filename in learners_to_retire/*;do - if test ! -f "$filename";then - echo "No users to retire" - continue - else - IFS="=" read -r name username <"$filename" - echo "Retiring $username" - python scripts/retire_one_learner.py --config_file=./pipeline_config/config.yml --username="$username" - fi -done