Merge branch 'main' into build-push-actions

.github/workflows/build-and-test-2.10-py3.yml

@@ -0,0 +1,10 @@
+name: Build and test CKAN 2.10 images (python)
+
+on: push
+
+jobs:
+  call-reusable-workflow:
+    uses: ./.github/workflows/reusable-build-and-test.yml
+    with:
+      ckan-major-version: "2.10"
+      docker-file: "Dockerfile.py3.10"

.github/workflows/build-and-test-2.10.yml

@@ -0,0 +1,9 @@
+name: Build and test CKAN 2.10 images (alpine)
+
+on: push
+
+jobs:
+  call-reusable-workflow:
+    uses: ./.github/workflows/reusable-build-and-test.yml
+    with:
+      ckan-major-version: "2.10"

.github/workflows/build-and-test-2.11.yml

@@ -0,0 +1,9 @@
+name: Build and test CKAN 2.11 images
+
+on: push
+
+jobs:
+  call-reusable-workflow:
+    uses: ./.github/workflows/reusable-build-and-test.yml
+    with:
+      ckan-major-version: "2.11"

.github/workflows/build-and-test-2.9-py3.yml

@@ -0,0 +1,10 @@
+name: Build and test CKAN 2.9 images (python)
+
+on: push
+
+jobs:
+  call-reusable-workflow:
+    uses: ./.github/workflows/reusable-build-and-test.yml
+    with:
+      ckan-major-version: "2.9"
+      docker-file: "Dockerfile.py3.9"

.github/workflows/build-and-test-2.9.yml

@@ -0,0 +1,9 @@
+name: Build and test CKAN 2.9 images (alpine)
+
+on: push
+
+jobs:
+  call-reusable-workflow:
+    uses: ./.github/workflows/reusable-build-and-test.yml
+    with:
+      ckan-major-version: "2.9"

.github/workflows/build-and-test-master.yml

@@ -0,0 +1,9 @@
+name: Build and test CKAN master images
+
+on: push
+
+jobs:
+  call-reusable-workflow:
+    uses: ./.github/workflows/reusable-build-and-test.yml
+    with:
+      ckan-major-version: "master"

.github/workflows/reusable-build-and-test.yml

@@ -0,0 +1,106 @@
+name: Reusable CKAN images build and test workflow
+
+on:
+  workflow_call:
+    inputs:
+      ckan-major-version:
+        required: true
+        type: string
+      docker-file:
+        required: false
+        default: 'Dockerfile'
+        type: string
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+    services:
+      solr:
+        image: ckan/ckan-solr:${{ inputs.ckan-major-version }}-solr9
+        ports:
+          - 8983:8983
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ inputs.ckan-major-version }}
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+          image: redis:7
+          ports:
+            - 6379:6379
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Read version from VERSION file
+        run: |
+          VERSION_VALUE=$(cat "ckan-${{ inputs.ckan-major-version }}/VERSION.txt")
+          if [[ $VERSION_VALUE != "master" ]]; then VERSION_VALUE="ckan-$VERSION_VALUE"; fi
+          echo "Using ref $VERSION_VALUE in next steps"
+          echo "CKAN_REF=$VERSION_VALUE" >> $GITHUB_ENV
+
+      - name: Build base image
+        uses: docker/build-push-action@v6
+        with:
+          push: false
+          load: true
+          context: "{{defaultContext}}:ckan-${{ inputs.ckan-major-version }}"
+          file: ${{ inputs.docker-file }}
+          build-args: |
+            CKAN_REF=${{ env.CKAN_REF }}
+            ENV=base
+          tags: |
+            ckan-base-${{ inputs.ckan-major-version }}-local
+
+      - name: Test base image
+        run: |
+          docker run \
+            --rm \
+            --net=host \
+            -e CKAN_SQLALCHEMY_URL=postgresql://ckan_default:pass@localhost/ckan_test \
+            -e CKAN_DATASTORE_WRITE_URL=postgresql://datastore_write:pass@localhost/datastore_test \
+            -e CKAN_DATASTORE_READ_URL=postgresql://datastore_read:pass@localhost/datastore_test \
+            -e CKAN_SOLR_URL=http://localhost:8983/solr/ckan \
+            -e CKAN_REDIS_URL=redis://localhost:6379/1 \
+            -e CKAN_INI=/srv/app/src/ckan/test-core.ini \
+            --entrypoint "" \
+            ckan-base-${{ inputs.ckan-major-version }}-local \
+            ckan --help
+
+      - name: Build dev image
+        uses: docker/build-push-action@v6
+        with:
+          push: false
+          load: true
+          context: "{{defaultContext}}:ckan-${{ inputs.ckan-major-version }}"
+          file: ${{ inputs.docker-file }}
+          build-args: |
+            CKAN_REF=${{ env.CKAN_REF }}
+            ENV=dev
+          tags: |
+            ckan-dev-${{ inputs.ckan-major-version }}-local
+
+      - name: Test dev image
+        run: |
+          docker run \
+            --rm \
+            --net=host \
+            -e CKAN_SQLALCHEMY_URL=postgresql://ckan_default:pass@localhost/ckan_test \
+            -e CKAN_DATASTORE_WRITE_URL=postgresql://datastore_write:pass@localhost/datastore_test \
+            -e CKAN_DATASTORE_READ_URL=postgresql://datastore_read:pass@localhost/datastore_test \
+            -e CKAN_SOLR_URL=http://localhost:8983/solr/ckan \
+            -e CKAN_REDIS_URL=redis://localhost:6379/1 \
+            -e CKAN_INI=/srv/app/src/ckan/test-core.ini \
+            --entrypoint "" \
+            ckan-dev-${{ inputs.ckan-major-version }}-local \
+            pytest --ckan-ini=/srv/app/src/ckan/test-core.ini -v /srv/app/src/ckan/ckan/tests/logic/action/test_create.py::TestDatasetCreate

README.md

@@ -25,24 +25,49 @@ The following CKAN versions are available in base or dev forms. They are disting
 Older CKAN versions might be available as [image tags](https://hub.docker.com/r/ckan/ckan-base/tags) but note that these are not supported as per [CKAN's release policy](https://docs.ckan.org/en/latest/maintaining/releases.html#supported-versions).
 
 
+### Repo structure
+
+Each CKAN version is located in separate repo containing:
+
+```
+ckan-X.XX
+├── Dockerfile          # Dockerfile for the image (in older versions there might be one for alpine and one for python)
+├── VERSION.txt         # Full CKAN version built (eg. 2.11.0, 2.10.5)
+├── PYTHON_VERSION.txt  # Python version used (eg. 3.10)
+└── setup               # Setup scripts used by the images
+    ├── prerun.py
+    ├── start_ckan_development.sh
+    ├── start_ckan.sh
+    ├── unsafe.cert
+    └── unsafe.key
+
+```
+
 ### Building and Pushing the images
 
 The images can be built locally and tagged appropriately so they can then be pushed into the CKAN DockerHub repo
-assuming you have the correct permission to do so
+assuming you have the correct permission to do so.
 
-For CKAN 2.10 base images, go to the `ckan-2.10/base` directory and use the Makefile included:
+All operations are done using the `build.sh` script located at the root of the repository.
 
+```
+Usage: ./build.sh <action> [<params>]
+Available actions:
+  versions                   - Shows the current CKAN versions used
+  build <version> [base|dev] - Builds images for a CKAN version
+                               Pass 'base' or 'dev' to just build these.
+  push  <version>            - Pushes images to the Docker Hub
 
-    cd ckan-2.10/base
-    make build (can then use locally)
-    make push (if you have enough credentials)
+```
 
+For instance:
 
-For CKAN 2.10 dev images, go to the `ckan-2.10/dev` directory and use the Makefile included:
+```
+./build.sh build 2.11
 
+./build.sh build master
+./build.sh build 2.10 base
+./build.sh build 2.9 dev
+```
 
-    cd ckan-2.10/dev
-    make build (can then use locally)
-    make push (if you have enough credentials)
 
-The process is the same for other CKAN versions and the master branch (`ckan-master`).

build.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+
+set -e
+
+
+set_vars() {
+    local ckan_version_ref="$1"
+    local env="$2"
+
+    ckan_version=$(cat "ckan-$ckan_version_ref/VERSION.txt")
+    python_version=$(cat "ckan-$ckan_version_ref/PYTHON_VERSION.txt")
+    python_dockerfile=Dockerfile.py$python_version
+    tag_name="ckan/ckan-$env:$ckan_version"
+    python_tag_name="ckan/ckan-$env:$ckan_version-py$python_version"
+    if [ "$ckan_version" = "master" ]; then
+        ckan_tag=$ckan_version
+        alt_tag_name=$tag_name
+        python_alt_tag_name=$python_tag_name
+    else
+        ckan_tag=ckan-$ckan_version
+        ckan_version_major=$(echo "$ckan_version" | cut -d'.' -f1)
+        ckan_version_minor=$(echo "$ckan_version" | cut -d'.' -f2)
+        alt_tag_name="ckan/ckan-$env:$ckan_version_major.$ckan_version_minor"
+        python_alt_tag_name="ckan/ckan-$env:$ckan_version_major.$ckan_version_minor-py$python_version"
+    fi
+
+}
+
+
+build_images() {
+    local ckan_version_ref="$1"
+    local env="$2"
+
+    set_vars "$ckan_version_ref" "$env"
+
+    if [ -f "ckan-$ckan_version_ref/$python_dockerfile" ]; then
+        # Build Python image if there's a separate .pyXX Dockerfile
+        DOCKER_BUILDKIT=1 docker build \
+            --build-arg="ENV=$env" \
+            --build-arg="CKAN_REF=$ckan_tag" \
+            -t "$python_tag_name" \
+            -t "$python_alt_tag_name" \
+            -f "ckan-$ckan_version_ref/$python_dockerfile" \
+            "ckan-$ckan_version_ref"
+    fi
+
+    # Build image 
+    DOCKER_BUILDKIT=1 docker build \
+        --build-arg="ENV=$env" \
+        --build-arg="CKAN_REF=$ckan_tag" \
+         -t "$tag_name" \
+         -t "$alt_tag_name" \
+         -t "$python_tag_name" \
+         -t "$python_alt_tag_name" \
+         "ckan-$ckan_version_ref"
+}
+
+
+push_images() {
+    local ckan_version_ref="$1"
+    local env="$2"
+
+    set_vars "$ckan_version_ref" "$env"
+
+    echo "Pushing $tag_name"
+    docker push "$tag_name"
+    echo "Pushing $alt_tag_name"
+    docker push "$alt_tag_name"
+    echo "Pushing $python_tag_name"
+    docker push "$python_tag_name"
+    echo "Pushing $python_alt_tag_name"
+    docker push "$python_alt_tag_name"
+
+}
+
+
+show_versions() {
+    echo "Versions built currently:"
+    cat ckan-*/VERSION.txt | sort -t. -k1,1n -k2,2n | sed 's/^/* /'
+}
+
+
+show_usage() {
+    echo "Usage: $0 <action> [<params>]"
+    echo "Available actions:"
+    echo "  versions                   - Shows the current CKAN versions used"
+    echo "  build <version> [base|dev] - Builds images for a CKAN version"
+    echo "                               Pass 'base' or 'dev' to just build these."
+    echo "  push  <version>            - Pushes images to the Docker Hub"
+    exit 1
+}
+
+check_push_msg="
+**********************************************************************
+*                                                                    *
+* Warning: Pushing images to the Docker Hub should generally be done *
+* via automated GitHub Actions.                                      *
+*                                                                    *
+**********************************************************************
+
+Are you sure you want to proceed? [y/N]"
+
+
+# Check if at least one parameter is provided
+if [ $# -eq 0 ]; then
+    show_usage
+fi
+
+action=$1
+
+case "$action" in
+    "versions")
+           show_versions 
+        ;;
+    "build")
+        ckan_version_ref=$2
+
+        if [ -z "$ckan_version_ref" ]; then
+            echo "Missing version"
+            show_usage
+            exit 1
+        fi
+
+        if [ ! -d "ckan-$ckan_version_ref" ]; then
+            echo "Unknown version: $ckan_version_ref"
+            exit 1
+        fi
+
+        base_or_dev=$3
+
+        case "$base_or_dev" in
+            "base")
+                build_base=true
+                ;;
+            "dev")
+                build_dev=true
+                ;;
+            "")
+                build_base=true
+                build_dev=true
+                ;;
+            *)
+                echo "Please enter 'base' or 'dev'"
+                show_usage
+                exit 1
+        esac
+
+        if [ "$build_base" = true ]; then
+            build_images "$ckan_version_ref" "base"
+        fi
+        if [ "$build_dev" = true ]; then
+            build_images "$ckan_version_ref" "dev"
+        fi
+        ;;
+    "push")
+
+        ckan_version_ref=$2
+
+        read -p "$check_push_msg" -n 1 -r answer
+        echo
+        if [[ $answer =~ ^[Yy]$ ]]; then
+            push_images "$ckan_version_ref" "base"
+            push_images "$ckan_version_ref" "dev"
+        fi
+        ;;
+    *)
+        echo "Error: Unknown action '$action'"
+        show_usage
+        ;;
+esac