diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36126c1..4efbf80 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ variables: TUTOR_PLUGIN: discovery TUTOR_IMAGES: discovery TUTOR_PYPI_PACKAGE: tutor-discovery - OPENEDX_RELEASE: olive + OPENEDX_RELEASE: palm GITHUB_REPO: overhangio/tutor-discovery include: diff --git a/README.rst b/README.rst index 7ac2ea9..3f79808 100644 --- a/README.rst +++ b/README.rst @@ -61,16 +61,21 @@ Re-indexing courses Caching programs ~~~~~~~~~~~~~~~~ -In order to cache programs in the LMS, you will need to manually create a catalog integration. This step should be performed just once:: +In order to cache programs in the LMS, you will need to manually create a catalog integration. Make sure you use staff user for the below command. If ``lms_catalog_service_user`` is not a staff user, then make it a staff user in your LMS User model. This step should be performed just once:: tutor local run lms ./manage.py lms create_catalog_integrations --enabled \ --internal_api_url="" \ --service_username=lms_catalog_service_user -Then:: +Then run the below command, this command will cause errors every time as it tries to cache programs from all sites that are added to your LMS sites model:: tutor local run lms ./manage.py lms cache_programs +If you don't want the errors, then make use of an extra argumet to the command .i.e. ``--domain``. This argument will be equal to ``local.overhang.io`` if you are running tutor local and ``local.overhang.io:8000`` if you are running tutor dev:: + tutor local run lms ./manage.py lms cache_programs --domain="local.overhang.io" + or + tutor dev run lms ./manage.py lms cache_programs --domain="local.overhang.io:8000" + This last step should be performed every time you create new or make changes to existing programs. Install extra requirements diff --git a/setup.py b/setup.py index 183f853..e35e949 100644 --- a/setup.py +++ b/setup.py @@ -34,8 +34,8 @@ long_description_content_type="text/x-rst", packages=find_packages(exclude=["tests*"]), include_package_data=True, - install_requires=["tutor>=15.0.0,<16.0.0"], - python_requires=">=3.7", + install_requires=["tutor>=16.0.0,<17.0.0"], + python_requires=">=3.8", entry_points={"tutor.plugin.v1": ["discovery = tutordiscovery.plugin"]}, classifiers=[ "Development Status :: 5 - Production/Stable", @@ -43,9 +43,9 @@ "License :: OSI Approved :: GNU Affero General Public License v3", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], ) diff --git a/tutordiscovery/__about__.py b/tutordiscovery/__about__.py index 9d7ed14..60b692a 100644 --- a/tutordiscovery/__about__.py +++ b/tutordiscovery/__about__.py @@ -1,4 +1,4 @@ -__version__ = "15.0.0" +__version__ = "16.0.0" # Handle version suffix for nightly, just like tutor core. __version_suffix__ = "" diff --git a/tutordiscovery/patches/local-docker-compose-jobs-services b/tutordiscovery/patches/local-docker-compose-jobs-services index a880498..09c77ab 100644 --- a/tutordiscovery/patches/local-docker-compose-jobs-services +++ b/tutordiscovery/patches/local-docker-compose-jobs-services @@ -4,4 +4,6 @@ discovery-job: DEFAULT_PARTNER_CODE: openedx volumes: - ../plugins/discovery/apps/settings/tutor:/openedx/discovery/course_discovery/settings/tutor:ro - depends_on: {{ [("lms", RUN_LMS), ("mysql", RUN_MYSQL)]|list_if }} + depends_on: + - lms + {% if RUN_MYSQL %}- mysql{% endif %} diff --git a/tutordiscovery/patches/local-docker-compose-services b/tutordiscovery/patches/local-docker-compose-services index f158d4a..6bd8e89 100644 --- a/tutordiscovery/patches/local-docker-compose-services +++ b/tutordiscovery/patches/local-docker-compose-services @@ -5,4 +5,7 @@ discovery: restart: unless-stopped volumes: - ../plugins/discovery/apps/settings/tutor:/openedx/discovery/course_discovery/settings/tutor:ro - depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("lms", RUN_LMS), ("mysql", RUN_MYSQL)]|list_if }} + depends_on: + - lms + {% if RUN_MYSQL %}- mysql{% endif %} + {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} diff --git a/tutordiscovery/plugin.py b/tutordiscovery/plugin.py index e6bae19..3e36638 100644 --- a/tutordiscovery/plugin.py +++ b/tutordiscovery/plugin.py @@ -1,6 +1,9 @@ +from __future__ import annotations + from glob import glob import os import pkg_resources +import typing as t from tutor import hooks as tutor_hooks @@ -74,11 +77,26 @@ ) ) + +REPO_NAME = "course-discovery" +APP_NAME = "discovery" + + # Automount /openedx/discovery folder from the container @tutor_hooks.Filters.COMPOSE_MOUNTS.add() def _mount_course_discovery(mounts, name): - if name == "course-discovery": - mounts.append(("discovery", "/openedx/discovery")) + if name == REPO_NAME: + mounts.append((APP_NAME, "/openedx/discovery")) + return mounts + + +# Bind-mount repo at build-time, both for prod and dev images +@tutor_hooks.Filters.IMAGES_BUILD_MOUNTS.add() +def _mount_course_discovery_on_build(mounts: list[tuple[str, str]], host_path: str) -> list[tuple[str, str]]: + path_basename = os.path.basename(host_path) + if path_basename == REPO_NAME: + mounts.append((APP_NAME, f"{APP_NAME}-src")) + mounts.append((f"{APP_NAME}-dev", f"{APP_NAME}-src")) return mounts @@ -115,3 +133,12 @@ def _mount_course_discovery(mounts, name): tutor_hooks.Filters.CONFIG_OVERRIDES.add_items( list(config.get("overrides", {}).items()) ) + + +@tutor_hooks.Filters.APP_PUBLIC_HOSTS.add() +def _print_discovery_public_hosts(hosts: list[str], context_name: t.Literal["local", "dev"]) -> list[str]: + if context_name == "dev": + hosts += ["{{ DISCOVERY_HOST }}:8381"] + else: + hosts += ["{{ DISCOVERY_HOST }}"] + return hosts diff --git a/tutordiscovery/templates/discovery/build/discovery/Dockerfile b/tutordiscovery/templates/discovery/build/discovery/Dockerfile index f425a89..92fa176 100644 --- a/tutordiscovery/templates/discovery/build/discovery/Dockerfile +++ b/tutordiscovery/templates/discovery/build/discovery/Dockerfile @@ -1,17 +1,23 @@ -FROM docker.io/ubuntu:20.04 +{% if is_buildkit_enabled() %}# syntax=docker/dockerfile:1.4{% endif %} +###### Minimal image with base system requirements for most stages +FROM docker.io/ubuntu:20.04 as minimal ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ - apt install -y curl git-core language-pack-en python3 python3-pip python3-venv \ - build-essential libcairo2 libffi-dev libmysqlclient-dev libxml2-dev libxslt-dev libjpeg-dev libssl-dev +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \ + apt update && \ + apt install -y curl git-core language-pack-en python3 python3-pip python3-venv \ + build-essential libcairo2 libffi-dev libmysqlclient-dev libxml2-dev libxslt-dev libjpeg-dev libssl-dev ENV LC_ALL en_US.UTF-8 ARG APP_USER_ID=1000 +RUN if [ "$APP_USER_ID" = 0 ]; then echo "app user may not be root" && false; fi RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER_ID} app USER ${APP_USER_ID} +###### Git-clone course-discovery repo ARG DISCOVERY_REPOSITORY=https://github.com/edx/course-discovery.git -ARG DISCOVERY_VERSION={{ OPENEDX_COMMON_VERSION }} +ARG DISCOVERY_VERSION='{{ OPENEDX_COMMON_VERSION }}' RUN mkdir -p /openedx/discovery && \ git clone $DISCOVERY_REPOSITORY --branch $DISCOVERY_VERSION --depth 1 /openedx/discovery WORKDIR /openedx/discovery @@ -23,13 +29,15 @@ ENV DISCOVERY_CFG /openedx/config.yml # Install python venv RUN python3 -m venv ../venv/ ENV PATH "/openedx/venv/bin:$PATH" -# https://pypi.org/project/setuptools/ -# https://pypi.org/project/pip/ -# https://pypi.org/project/wheel/ -RUN pip install setuptools==65.5.1 pip==22.3.1 wheel==0.38.4 + +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ + # https://pypi.org/project/setuptools/ + # https://pypi.org/project/pip/ + # https://pypi.org/project/wheel/ + setuptools==67.7.2 pip==23.1.2. wheel==0.40.0 # Install a recent version of nodejs -RUN pip install nodeenv +RUN pip install nodeenv==1.7.0 # nodejs version picked from https://github.com/openedx/course-discovery/blob/master/Dockerfile RUN nodeenv /openedx/nodeenv --node=16.14.2 --prebuilt ENV PATH /openedx/nodeenv/bin:${PATH} @@ -37,21 +45,22 @@ ENV PATH /openedx/nodeenv/bin:${PATH} # Install python and nodejs requirements # This is identical to "make production-requirements" but it was split in multiple # instructions to benefit from docker image caching -RUN pip install -r requirements.txt -{% for extra_requirements in DISCOVERY_EXTRA_PIP_REQUIREMENTS %}RUN pip install '{{ extra_requirements }}' +# Install base requirements +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements.txt +{% for extra_requirement in DISCOVERY_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}' {% endfor %} -ARG NPM_REGISTRY={{ NPM_REGISTRY }} -RUN npm install --verbose --registry=$NPM_REGISTRY --production -RUN ./node_modules/.bin/bower install --allow-root --production - -# Install django-redis for using redis as a django cache -# https://pypi.org/project/django-redis/ -RUN pip install django-redis==5.2.0 +# Install npm, bower requirements +ARG NPM_REGISTRY='{{ NPM_REGISTRY }}' +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.npm/,sharing=shared,uid=${APP_USER_ID} {% endif %}npm clean-install --verbose --no-audit --registry=$NPM_REGISTRY --production +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/bower,sharing=shared,uid=${APP_USER_ID} {% endif %}./node_modules/.bin/bower install --allow-root --production -# Install uwsgi -# https://pypi.org/project/uWSGI/ -RUN pip install uwsgi==2.0.21 +# Install extra requirements +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ + # Use redis as a django cache https://pypi.org/project/django-redis/ + django-redis==5.2.0 \ + # uwsgi server https://pypi.org/project/uWSGI/ + uwsgi==2.0.21 # Collect static assets COPY --chown=app:app assets.py ./course_discovery/settings/assets.py