Update build code; convert from rye to uv (#169)

.github/workflows/ci.yml

@@ -1,6 +1,6 @@
 ---
 name: CI
-on: [push]
+on: [push, workflow_dispatch]
 
 jobs:
   tests:
@@ -12,7 +12,7 @@ jobs:
         - "3.9"
         - "3.10"
         - "3.11"
-        #- "3.12"
+        # - "3.12"
 
     # Service containers to run
     services:
@@ -41,21 +41,21 @@ jobs:
     - name: Apt install
       run: sudo apt-get install -y libxmlsec1-dev
 
-    - uses: eifinger/setup-rye@v4
-      id: setup-rye
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
 
     - name: Pin python-version ${{ matrix.python-version }}
-      run: rye pin ${{ matrix.python-version }}
+      run: uv python pin ${{ matrix.python-version }}
 
     - name: Install dependencies
-      run: rye sync
+      run: uv sync
 
     - name: Tests
-      run: rye test
+      run: uv run pytest
 
     - name: Changelog
       if: ${{ !contains(github.event.head_commit.author, 'renovate') }}
-      run: rye run build changelog check
+      run: PYTHONPATH=build-support/bin uv run build changelog check
 
       # - name: Upload coverage to CodeCov
       #   uses: codecov/codecov-action@v1
@@ -70,8 +70,8 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - uses: eifinger/setup-rye@v4
-      id: setup-rye
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
 
     # extract the app name out of the tag's ref value
     - id: get-app-name
@@ -83,7 +83,7 @@ jobs:
           core.setOutput('app-package', appName.replace("mitol-django-", "").replace(/-/g, "_"))
 
     - name: Build
-      run: rye build "src/${{steps.get-app-name.outputs.app-package}}:"
+      run: PYTHONPATH=build-support/bin uv build "src/${{steps.get-app-name.outputs.app-package}}"
 
     - uses: pypa/gh-action-pypi-publish@release/v1
       with:

.github/workflows/renovate.yml

@@ -17,8 +17,8 @@ jobs:
     - name: Apt install
       run: sudo apt-get update && sudo apt-get install -y libxmlsec1-dev
 
-    - uses: eifinger/setup-rye@v4
-      id: setup-rye
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
 
     - name: Self-hosted Renovate
       uses: renovatebot/[email protected]

.gitignore

@@ -68,3 +68,6 @@ testapp/settings/dev.py
 /.pids
 /.pants.workdir.file_lock*
 *.pex
+
+# SSH keys/etc.
+ssh/*

Dockerfile

@@ -17,24 +17,35 @@ WORKDIR /tmp
 COPY apt.txt /tmp/apt.txt
 RUN xargs apt-get install -y <apt.txt
 
-RUN useradd dev
+COPY --from=ghcr.io/astral-sh/uv:0.4.25 /uv /uvx /bin/
+
+RUN useradd -G ubuntu dev
 USER dev
 WORKDIR /home/dev
 
 # ===================================================================
-FROM base as rye
+FROM base as uv
 ARG PYTHON_VERSION=3.11
 
 USER dev
-ENV PATH="${PATH}:/home/dev/.rye/shims"
-RUN curl -sSf https://rye.astral.sh/get | RYE_INSTALL_OPTION="--yes" bash &&\
-    rye pin ${PYTHON_VERSION}
 
-# ===================================================================
+RUN uv python install $PYTHON_VERSION
 
 # the pants installer puts things in ~/cache/nce and it needs to be persistent
 RUN mkdir -p .cache && chown dev:dev .cache
 
-
 VOLUME /home/dev/.cache
 WORKDIR /home/dev/src
+
+# ===================================================================
+FROM uv as release
+
+USER dev
+ENV PYTHONPATH="build-support/bin/"
+
+WORKDIR /home/dev
+RUN mkdir -m 0750 .ssh
+COPY --chown=dev:dev . /home/dev/src
+
+WORKDIR /home/dev/src
+RUN uv sync

README.md

@@ -1,25 +1,42 @@
 ### Open Learning Django Apps
 
-This repository is the home of MIT Open Learning's reusable django apps
+This repository is the home of MIT Open Learning's reusable django apps.
 
-### Changelogs
+### Getting Started
 
-We maintain changelogs in `changelog.d/` directories with each app. To create a new changelog for your changes, run:
+This set of libraries is managed using [uv](https://docs.astral.sh/uv/).
 
-- `mkdir ./src/mitol/{APPNAME}/changelog.d`
-- `rye run build changelog create --app APPNAME`
-  - `APPNAME`: the name of an application directory
+#### Use on your host system
 
-Then fill out the new file that was generated with information about your changes. These changes will all be merged down into `CHANGELOG.md` when a release is generated.
+- Install `xmlsec` native libraries for your OS: https://xmlsec.readthedocs.io/en/stable/install.html
+- Install `uv` as described in the manual: https://docs.astral.sh/uv/
+- Bootstrap the `uv` environment: `uv python install 3.11 ; uv sync`
 
-### Releases
+> [!WARNING]
+> If you're running `uv` locally, any calls to the `build` command will need help.
+> Prepend `PYTHONPATH=build-support/bin` to the command so uv will launch it properly.
 
-Changelogs are maintained according to [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
-Versioning uses a date-based versioning scheme with incremental builds on the same day.
-Version tags follow `{package-name}/v{version}`
-To perform a release, run:
-- `rye run build release create --app APPNAME --push`:
-  - `APPNAME`: the name of an application directory
+#### Use the Docker Compose environment (recommended)
+
+The Compose environment includes a container for general use called `shell` and one specifically for building releases called `release`. In either case, you'll get a shell with `uv` already set up, and with a PostgreSQL database available.
+
+- Ensure that 'other' users can write to the repo directory: `chmod -R o+w .`
+- Build the containers: `docker compose build`
+- Get a shell in the `shell` container: `docker compose run --rm -ti shell bash`
+
+#### Using the `release` container
+
+The `release` container is special and is set up to run `build` commands, including generating releases. It's special because it _does not_ mount your local copy of the codebase (mainly because of file permission issues). So, it requires a bit more care before using.
+
+**One-time setup:**
+1. Copy the SSH private key you use for GitHub to the `ssh` folder and name it appropriately (e.g. `id_ed25519`, etc.)
+2. Set permissions on the key so that it is group-readable (`0640`).
+
+**Using:**
+1. Build the images, so the source code in the image is up to date: `docker compose build`
+2. Get a shell: `docker compose run --rm -ti release bash`
+3. Run your command: `uv run build release create etc. etc. etc.`
+4. If you've done things that involve Git, make sure you `git pull` when you leave the session.
 
 ### Navigating this repository
 
@@ -28,68 +45,60 @@ To perform a release, run:
   - Module paths follow the pattern `mitol.{name}`
   - The app itself is installable to `INSTALLED_APPS` as `"{name}"`.
 
-### Prerequisites
+### Adding a new app
 
-#### Use the docker-compose container (recommended)
+Apps go in the `src/` folder. Test suites for apps go in the `tests/` folder (which is a Django app for this purpose).
 
-- You'll need to make sure other users (the docker container user), can write to the repo root directory. Run `chmod o+w .`
-- Run `docker compose run --rm shell bash` to get a clean sandbox environment
+Per convention, use `_` for spaces within your app name if you must use spaces.
 
-#### Use on your host system
+To add a new one, it's easiest to copy one of the existing apps. There's one called `uvtestapp` that has (basically) nothing in it, and can be used for this purpose.
 
-- Install `xmlsec` native libraries for your OS: https://xmlsec.readthedocs.io/en/stable/install.html
-- Install `rye` following the instructions at https://rye.astral.sh/
+1. Duplicate the `uvtestapp` folder, and rename the copy to the name you wish to use.
+2. Update things within the folder to use the new name. This will include:
+   * The folder under `mitol`
+   * `README.md`
+   * `pyproject.toml`
+   * `mitol/<appname>/__init__.py`
+   * `mitol/<appname>/apps.py`
+3. Update the root `pyproject.toml`
+   * Under `[project]`, add the new app into `dependencies` in the same format that's already there.
+   * Under `[tool.uv.sources]`, add a new entry for the new app, using (again) the same format as the other entries.
+4. Test building: `uv build --package mitol-django-<appname>` . (This ensures that uv is OK with your changes.)
+5. Add space for the app in the `tests` app: `mkdir tests/mitol/<appname>` and add a blank `__init__.py` to it.
+6. Add the app to `testapp/settings/shared.py`
+   * You must add it to `INSTALLED_APPS`.
+   * If your app has configuration settings, add to the `import_settings_module` call at the top too.
 
+You can now add your code and tests.
 
-### Usage
+If you need to run Django commands,
 
-We use [`rye`](https://rye.astral.sh/) to manage apps and releases.
+### Running tests
 
-Useful commands:
-```shell
-# run all tests
-rye test
-# run only common app tests
-rye test -p mitol-django-common
+Run `uv run pytest`. This should run all the tests. If you want to run a specific one, specify with a file path as per usual. Use the whole path (so `tests/mitol/<appname>/etc`).
 
-# build a package
-rye build -p mitol-django-common
+### Changelogs
 
-# format code (isort + black)
-rye fmt
+We maintain changelogs in `changelog.d/` directories with each app. To create a new changelog for your changes, run:
 
-# run lints
-rye lint
+- `uv run build changelog create --app APPNAME`
+  - `APPNAME`: the name of an application directory
 
-# run django management scripts
-rye run tests/manage.py -- ARGS
-# run a django shell
-rye run tests/manage.py -- shell
-# create migrations
-rye run tests/manage.py -- makemigrations
-# run a django migrate
-rye run tests/manage.py -- migrate
-```
+Note warning above about `PYTHONPATH`. You will need to adjust permissions/ownership on the new file if you're using the Compose setup.
 
-### Migrations
+Then fill out the new file that was generated with information about your changes. These changes will all be merged down into `CHANGELOG.md` when a release is generated. **Do this before you put up a PR for your changes.**
 
-To generate migrations for a reusable app, run the standard:
+### Releases
 
-```
-rye run tests/manage.py -- makemigrations APP_NAME --name MIGRATION_NAME
-```
+Changelogs are maintained according to [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+Versioning uses a date-based versioning scheme with incremental builds on the same day.
+Version tags follow `{package-name}/v{version}`
+To perform a release, run:
+- `uv run build release create --app APPNAME --push`:
+  - `APPNAME`: the name of an application directory
 
-where `APP_NAME` matches the `name` attribute from your `apps.py` app.
+`release` expects to be run on the `main` branch and it expects you to not have changes pending.
 
-#### Adding a new app
+You should probably avoid running this within the `shell` container - Git will be pretty unhappy about the permissions of the `.git` folder and you may run into other permissions issues. Either run this on your local machine or use the `release` container for this as described above.
 
-- Create the project:
-```shell
-NAME="[APP-NAME]"
-mkdir src/mitol/$NAME
-pants run :django-admin -- startapp $NAME src/mitol/$NAME
-```
-- Add a `BUILD` file, following using the same files from other projects as a guideline
-- Add the project as a dependency in `src/BUILD`
-- Add the project to the testapp
-  - Add the project app in `tests/testapp/settings/shared.py` under `INSTALLED_APPS`
+Supplying the `--push` flag will tag the release appropriately and push it, and a GitHub action should publish it to PyPI.

build-support/bin/mitol/build_support/cli.py

@@ -16,3 +16,6 @@ def cli(ctx: Context):
 cli.add_command(changelog)
 cli.add_command(release)
 cli.add_command(version)
+
+if __name__ == "__main__":
+    cli()

build-support/bin/mitol/build_support/commands/release.py

@@ -8,7 +8,6 @@
     no_require_main,
     pass_app,
     pass_project,
-    require_no_changes,
 )
 from mitol.build_support.project import Project
 
@@ -19,7 +18,7 @@ def release():
 
 
 @release.command()
-@require_no_changes
+# @require_no_changes
 @no_require_main
 @app_option
 @option(
@@ -36,7 +35,8 @@ def create(ctx: Context, project: Project, app: App, push: bool):  # noqa: FBT00
 
     ctx.invoke(changelog.check)
     ctx.invoke(version.update)
-    ctx.invoke(changelog.collect, version=app.version)
+    # keep=True so we can remove these properly (i.e. in Git) later
+    ctx.invoke(changelog.collect, version=app.version, keep=True)
 
     # copy and remove irrelevant params
     params = ctx.params.copy()
@@ -59,11 +59,18 @@ def commit_and_tag(project: Project, app: App):
 
     repo.index.add(
         [
-            app.app_dir / "__init__.py",
+            app.app_dir / "mitol" / app.module_name / "__init__.py",
             app.app_dir / "pyproject.toml",
             app.app_dir / "CHANGELOG.md",
         ]
     )
+
+    repo.index.remove(
+        [
+            app.app_dir / "changelog.d" / "*.md",
+        ]
+    )
+
     repo.index.commit(f"Release {tag_name}")
 
     echo(f"Tagging {tag_name}")

docker-compose.yml

@@ -11,7 +11,7 @@ services:
 
   shell:
     build:
-      target: rye
+      target: uv
     volumes:
     - .:/home/dev/src
     - cache:/home/dev/.cache/
@@ -20,5 +20,16 @@ services:
     depends_on:
     - db
 
+  release:
+    build:
+      target: release
+    volumes:
+    - cache:/home/dev/.cache/
+    - ./ssh:/home/dev/.ssh:ro
+    environment:
+      DATABASE_URL: postgres://postgres:postgres@db:5432/postgres
+    depends_on:
+    - db
+
 volumes:
   cache: