Skip to content

Commit

Permalink
Merge branch 'main' into Update-fairdatapoint-extension-
Browse files Browse the repository at this point in the history
  • Loading branch information
Hans-Chrstian committed Aug 13, 2024
2 parents fc1e7b8 + 08ad2a6 commit a4b99fd
Show file tree
Hide file tree
Showing 17 changed files with 183 additions and 39 deletions.
15 changes: 9 additions & 6 deletions .env.example → .env
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,13 @@ CKAN_DB_USER=ckandbuser
CKAN_DB_PASSWORD=ckandbpassword
CKAN_DB=ckandb

# CKAN
# CKAN core
CKAN_VERSION=2.10.4
CKAN_SITE_ID=default
CKAN_SITE_URL=http://localhost:5500
CKAN__CORS__ORIGIN_WHITELIST=http://localhost:3000
CKAN_PORT=5000
CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
CKAN__AUTH__USER_CREATE_ORGANIZATIONS=false
# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
Expand All @@ -36,15 +35,13 @@ CKAN_SMTP_PASSWORD=pass
CKAN_SMTP_MAIL_FROM=ckan@localhost
CKAN_SQLALCHEMY_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB}
TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@${POSTGRES_HOST}/ckan_test
TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@${POSTGRES_HOST}/datastore_test
TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@${POSTGRES_HOST}/datastore_test
CKAN_SOLR_URL=http://solr:8983/solr/ckan
TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan
CKAN_REDIS_URL=redis://redis:6379/1
TEST_CKAN_REDIS_URL=redis://redis:6379/1

# Redis
REDIS_VERSION=6
CKAN_REDIS_URL=redis://redis:6379/1
TEST_CKAN_REDIS_URL=redis://redis:6379/1

# Extensions
CKANEXT__OIDC_PKCE__BASE_URL=https://keycloak-test.healthdata.nl/realms/ckan/protocol/openid-connect
Expand All @@ -63,3 +60,9 @@ CKAN__HARVEST__MQ__TYPE=redis
CKAN__HARVEST__MQ__HOSTNAME=redis
CKAN__HARVEST__MQ__PORT=6379
CKAN__HARVEST__MQ__REDIS_DB=1

# Debug
CKAN___DEBUG=true

#Logger
CKAN___LOGGER_CKANEXT__LEVEL=DEBUG
12 changes: 6 additions & 6 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: oss-review-toolkit/ort-ci-github-action@7f23c1f8d169dad430e41df223d3b8409c7a156e
- uses: oss-review-toolkit/ort-ci-github-action@v1
with:
allow-dynamic-versions: "true"
fail-on: "issues"
Expand All @@ -39,7 +39,7 @@ jobs:
uses: actions/checkout@v4

- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
Expand Down Expand Up @@ -67,7 +67,7 @@ jobs:
echo "last_tag=${TAGS[LAST_INDEX]}" >> $GITHUB_OUTPUT
- name: Build Docker image
uses: docker/build-push-action@v5
uses: docker/build-push-action@v6
with:
context: ./ckan
push: false
Expand All @@ -76,7 +76,7 @@ jobs:
labels: ${{ steps.meta.outputs.labels }}

- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
uses: aquasecurity/trivy-action@0.24.0
with:
image-ref: "${{ steps.tag.outputs.last_tag }}"
format: "table"
Expand All @@ -86,7 +86,7 @@ jobs:
severity: "CRITICAL"

- name: Push Docker image
uses: docker/build-push-action@v5
uses: docker/build-push-action@v6
with:
context: ./ckan
push: true
Expand All @@ -95,7 +95,7 @@ jobs:

- name: Deploy to Azure Web App
id: deploy-to-webapp
uses: azure/webapps-deploy@85270a1854658d167ab239bce43949edb336fa7c
uses: azure/webapps-deploy@v3
with:
app-name: ${{ env.AZURE_WEBAPP_NAME }}
publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE }}
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: oss-review-toolkit/ort-ci-github-action@7f23c1f8d169dad430e41df223d3b8409c7a156e
- uses: oss-review-toolkit/ort-ci-github-action@v1
with:
allow-dynamic-versions: "true"
fail-on: "issues"
Expand All @@ -37,15 +37,15 @@ jobs:
uses: actions/checkout@v4

- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
uses: docker/metadata-action@v5
with:
images: ${{ env.IMAGE_NAME }}
tags: |
Expand All @@ -65,7 +65,7 @@ jobs:
echo "last_tag=${TAGS[LAST_INDEX]}" >> $GITHUB_OUTPUT
- name: Build Docker image for scanning
uses: docker/build-push-action@v5
uses: docker/build-push-action@v6
with:
context: ./ckan
push: false
Expand All @@ -74,7 +74,7 @@ jobs:
labels: ${{ steps.meta.outputs.labels }}

- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
uses: aquasecurity/trivy-action@0.24.0
with:
image-ref: "${{ steps.tag.outputs.last_tag }}"
format: "table"
Expand All @@ -84,7 +84,7 @@ jobs:
severity: "CRITICAL"

- name: Build and push Docker image
uses: docker/build-push-action@v5
uses: docker/build-push-action@v6
with:
context: ./ckan
push: true
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: REUSE Compliance Check
uses: fsfe/reuse-action@v2
uses: fsfe/reuse-action@v4
72 changes: 72 additions & 0 deletions .github/workflows/weekly_vuln_scan.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
# SPDX-FileCopyrightText: 2024 PNED G.I.E.
# SPDX-License-Identifier: Apache-2.0

name: Scan Latest Two Semantically Versioned Image with Trivy and ORT

on:
schedule:
- cron: "0 14 * * 1" # Works on every Monday at 14:00 UTC
workflow_dispatch:

env:
REGISTRY: ghcr.io
IMAGE_NAME: ghcr.io/genomicdatainfrastructure/gdi-userportal-ckan-docker
LATEST_PATCHES: ""

jobs:
fetch-and-scan:
runs-on: ubuntu-latest
steps:
- name: Setup Docker
run: sudo service docker start

- name: Log in to the Container registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: List Docker Image Tags and Fetch Latest Two
run: |
IMAGE="ghcr.io/genomicdatainfrastructure/gdi-userportal-ckan-docker"
echo "Fetching Docker image tags..."
RESPONSE=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/orgs/genomicdatainfrastructure/packages/container/gdi-userportal-ckan-docker/versions")
# Log the response for debugging
echo "Response from curl: $RESPONSE"
# Extract tags and creation dates, then sort by creation date
TAGS_WITH_DATES=$(echo "$RESPONSE" | jq -r '.[] | .metadata.container.tags[] as $tag | select($tag | test("^sha-[a-z0-9]+$")) | "\($tag) \(.created_at)"' | sort -k2 -r)
# Get the latest tag from each of the latest two minor versions
LATEST_TAGS=$(echo "$TAGS_WITH_DATES" | awk '{split($1, a, "."); key=a[1] "." a[2]; if (!seen[key]++) print $1}' | head -n 2 | tr '\n' ' ')
# Print the results
echo "Tags fetched: $LATEST_TAGS"
echo "LATEST_PATCHES=$LATEST_TAGS" >> $GITHUB_ENV
- name: Pull and Scan Docker Images with Trivy
run: |
IFS=' ' read -ra TAGS <<< "${{ env.LATEST_PATCHES }}"
for TAG in "${TAGS[@]}"
do
IMAGE_TAG="ghcr.io/genomicdatainfrastructure/gdi-userportal-ckan-docker:$TAG"
echo "Pulling $IMAGE_TAG"
docker pull $IMAGE_TAG || exit 1
echo "Scanning $IMAGE_TAG with Trivy"
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:latest image --severity CRITICAL,HIGH --exit-code 1 "$IMAGE_TAG" || exit 1
done
- name: OSS Review Toolkit (ORT) Analysis
uses: actions/checkout@v4
with:
repository: ${{ github.repository }}

- uses: oss-review-toolkit/ort-ci-github-action@v1
with:
allow-dynamic-versions: "true"
fail-on: "issues"
run: "cache-dependencies,cache-scan-results,labels,analyzer,evaluator,advisor,reporter,upload-results"
3 changes: 1 addition & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,4 @@ _service-provider/*
_solr/schema.xml
_src/*
local/*
.env
venv
venv
9 changes: 2 additions & 7 deletions ckan/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@

FROM ckan/ckan-base:2.10.4

# Install any extensions needed by your CKAN instance
# See Dockerfile.dev for more details and examples

RUN pip3 install -e git+https://github.com/GenomicDataInfrastructure/[email protected]#egg=ckanext-gdi-userportal && \
pip3 install -r ${APP_DIR}/src/ckanext-gdi-userportal/requirements.txt

Expand Down Expand Up @@ -72,7 +69,5 @@ COPY --chown=ckan:ckan setup/prerun.py ${APP_DIR}
# TODO
# 1. Remove this command once the issue is fixed
# https://github.com/ckan/ckan-docker-base/issues/41
RUN mkdir /var/lib/ckan/storage
RUN mkdir /var/lib/ckan/storage/uploads
RUN mkdir /var/lib/ckan/storage/uploads/group
RUN chmod -R u+rwx "/var/lib/ckan"
RUN mkdir -p /var/lib/ckan/storage/uploads/group && \
chmod -R u+rwx "/var/lib/ckan"
3 changes: 3 additions & 0 deletions ckan/Dockerfile.dev
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# SPDX-FileCopyrightText: 2006-2024 Open Knowledge Foundation and contributors
# SPDX-FileContributor: PNED G.I.E.
# SPDX-FileContributor: Stichting Health-RI
#
# SPDX-License-Identifier: AGPL-3.0-only

Expand Down Expand Up @@ -60,3 +61,5 @@ RUN for d in $APP_DIR/patches/*; do \
done

ENV CKAN__PLUGINS="envvars image_view text_view recline_view scheming_datasets scheming_organizations gdi_userportal dcat harvest ckan_harvester dcat_rdf_harvester dcat_json_harvester dcat_json_interface oidc_pkce fairdatapointharvester activity"

COPY --chown=ckan:ckan setup/prerun.py ${APP_DIR}
2 changes: 1 addition & 1 deletion ckan/docker-entrypoint.d/common_vocabulary_tags.csv
Original file line number Diff line number Diff line change
Expand Up @@ -1091,7 +1091,7 @@ https://ncit.nci.nih.gov/ncitbrowser/ConceptReport.jsp?dictionary=NCI_Thesaurus&
https://ncim.nci.nih.gov/ncimbrowser/ConceptReport.jsp?code=C0853879,"Invasive Breast Carcinoma",en
http://purl.bioontology.org/ontology/ICD10CM/J96.2,"Acute and chronic respiratory failure",en
access_rights,"Access Rights",en
organization,"Catalogues",en
organization,"Organizations",en
publisher_name,"Publishers",en
res_format,"File Formats",en
spatial,"Spatial Coverage",en
Expand Down
33 changes: 33 additions & 0 deletions ckan/docker-entrypoint.d/setup_default_user_permissions.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/bash

# SPDX-FileCopyrightText: 2024 PNED G.I.E.
#
# SPDX-License-Identifier: AGPL-3.0-only

# Update the config file with each extension config-options
echo "[ckan.auth] Setting up config-options"
ckan config-tool $CKAN_INI -s app:main \
"ckan.auth.create_user_via_api = False"\
"ckan.auth.create_user_via_web = False"\
"ckan.auth.user_create_groups = False"\
"ckan.auth.user_create_organizations = False"\
"ckan.auth.user_delete_groups = False"\
"ckan.auth.user_delete_organizations = False"\
"ckan.auth.anon_create_dataset = False"\
"ckan.auth.create_unowned_dataset = False"\
"ckan.auth.create_dataset_if_not_in_organization = False"\
"ckan.auth.roles_that_cascade_to_sub_groups = admin"\
"ckan.auth.public_user_details = False"\
"ckan.auth.public_activity_stream_detail = False"\
"ckan.auth.allow_dataset_collaborators = False"\
"ckan.auth.allow_admin_collaborators = False"\
"ckan.auth.allow_collaborators_to_change_owner_org = False"\
"ckan.auth.create_default_api_keys = False"\
"ckan.auth.reveal_private_datasets = False"\
"ckan.auth.enable_cookie_auth_in_api = False"\
"ckan.auth.route_after_login = home.index"\
"ckan.user_reset_landing_page = home.index"\
"ckan.upload.user.types = image"\
"ckan.upload.user.mimetypes = image/png image/jpeg"\
"ckan.upload.group.types = image text/svg"\
"ckan.upload.group.mimetypes = image/png text/svg image/svg+xml"
12 changes: 12 additions & 0 deletions ckan/docker-entrypoint.d/setup_logger_level.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/sh

# SPDX-FileCopyrightText: 2024 PNED G.I.E.
#
# SPDX-License-Identifier: AGPL-3.0-only


echo "[DEFAULT] Setting up debug"
#ckan config-tool $CKAN_INI -s DEFAULT "debug = $CKAN___DEBUG"

echo "[logger_ckanext] Setting up level"
#ckan config-tool $CKAN_INI -s logger_ckanext "level = $CKAN___LOGGER_CKANEXT__LEVEL"
2 changes: 1 addition & 1 deletion ckan/docker-entrypoint.d/setup_scheming.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@ ckan config-tool $CKAN_INI -s app:main \
"scheming.presets = ckanext.scheming:presets.json"\
"scheming.dataset_fallback = false"\
"ckanext.dcat.rdf.profiles = euro_dcat_ap_2 fairdatapoint_dcat_ap"\
"ckanext.dcat.compatibility_mode = true"
"ckanext.dcat.compatibility_mode = false"
9 changes: 2 additions & 7 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,8 @@ services:
postgres:
build:
context: postgresql/
environment:
- POSTGRES_USER
- POSTGRES_PASSWORD
- POSTGRES_DB
- CKAN_DB_USER
- CKAN_DB_PASSWORD
- CKAN_DB
env_file:
- .env
volumes:
- pg_data:/var/lib/postgresql/data
restart: unless-stopped
Expand Down
2 changes: 1 addition & 1 deletion postgresql/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: AGPL-3.0-only

FROM postgres:12-alpine
FROM postgres:16-alpine

# Include extra setup scripts (eg datastore)
COPY --chown=postgres:postgres docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
29 changes: 29 additions & 0 deletions renovate.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"packageRules": [
{
"depTypeList": ["devDependencies", "require-dev"],
"updateTypes": ["patch", "minor", "major"],
"groupName": "devDependencies"
},
{
"description": "Trigger fix release for patch updates",
"matchPackageNames": [
"renovate/renovate",
"ghcr.io/renovatebot/renovate"
],
"automerge": true,
"matchUpdateTypes": ["patch"],
"semanticCommitType": "fix",
"automergeType": "pr"
}
],
"extends": [
"config:base",
":preserveSemverRanges",
":dependencyDashboard",
":rebaseStalePrs",
":enableVulnerabilityAlertsWithLabel('security')",
"group:recommended"
]
}
3 changes: 3 additions & 0 deletions renovate.json.license
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
SPDX-FileCopyrightText: 2024 PNED G.I.E.

SPDX-License-Identifier: Apache-2.0
2 changes: 1 addition & 1 deletion src/ckanext-gdi-userportal

0 comments on commit a4b99fd

Please sign in to comment.