diff --git a/.github/workflows/algorithm-evaluation.yml b/.github/workflows/algorithm-evaluation.yml new file mode 100644 index 0000000..feef483 --- /dev/null +++ b/.github/workflows/algorithm-evaluation.yml @@ -0,0 +1,83 @@ +name: Execute algorithm and evaluate metrics + +on: + push: + branches: [ "main" ] + paths: + - 'algorithms/*.yaml' + - '.github/workflows/algorithm-evaluation.yml' + pull_request: + branches: [ "main" ] + paths: + - 'algorithms/*.yaml' + - '.github/workflows/algorithm-evaluation.yml' + +jobs: + execute: + runs-on: ubuntu-latest + steps: + - name: Find updated algorithm + id: find-algorithm + uses: tj-actions/changed-files@v44 + with: + files: algorithms/**.yaml + + - uses: actions/checkout@v4 + + - name: Execute algorithm container + run: | + # iterate over all datasets in S3 + for recording in $(aws s3api list-objects --bucket ${AWS_BUCKET} --prefix datasets --output text --query 'Contents[].[Key]' | grep '.*edf'); do + + if [ ${{ steps.find-algorithm.outputs.any_changed }} == "false" ]; then + break + fi + + # save each recording (dataset slice) to a temp file + aws s3 cp s3://${AWS_BUCKET}/${recording} ./data/tmp.edf + + # Run inference on recording for every updated algorithm + for algo in ${{ steps.find-algorithm.outputs.all_changed_files }}; do + + IMAGE=$(grep '^image: ' $algo | sed 's/^image: \+//' | tr -d \'\") + ALGO_NAME=$(echo "$IMAGE" | iconv -t ascii//TRANSLIT | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | tr A-Z a-z) + + mkdir -p ./predictions + chmod -R 777 ./predictions/ + + echo "Running inference for $ALGO_NAME" + docker run \ + -e INPUT=tmp.edf \ + -e OUTPUT=tmp.tsv \ + -v ./predictions:/output:rw \ + -v ./data:/data:ro \ + "${IMAGE}" + + # Upload predictions to S3 + subpath=${recording#*/} + prediction=${subpath%_eeg.edf}_events.tsv + aws s3 cp \ + ./predictions/tmp.tsv \ + "s3://${AWS_BUCKET}/submissions/${ALGO_NAME}/${prediction}" + rm ./data/tmp.edf ./predictions/tmp.tsv + done + done + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + AWS_BUCKET: ${{secrets.AWS_BUCKET }} + + evaluate: + runs-on: ubuntu-latest + needs: [execute] + container: + image: ghcr.io/${{ github.repository }}-evaluator:main + credentials: + username: ${{ github.actor }} + password: ${{ secrets.github_token }} + + steps: + + - name: Evaluate algorithm predictions + run: python __main__.py diff --git a/.github/workflows/build-website.yml b/.github/workflows/build-website.yml new file mode 100644 index 0000000..ee2a345 --- /dev/null +++ b/.github/workflows/build-website.yml @@ -0,0 +1,82 @@ +name: Build website +on: + workflow_run: + workflows: ["Execute algorithm and evaluate metrics"] + branches: [main] + types: + - completed + + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +jobs: + download: + runs-on: ubuntu-latest + steps: + - name: Download results + run: | + aws s3 cp \ + s3://${AWS_BUCKET}/results/results.json \ + website/data/sampleEval.json + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + AWS_BUCKET: ${{secrets.AWS_BUCKET }} + + - name: upload artifact + uses: actions/upload-artifact@v4 + with: + name: web-data + path: website/data/ + + + build: + needs: download + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}-site-builder:main + credentials: + username: ${{ github.actor }} + password: ${{ secrets.github_token }} + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: download artifact + uses: actions/download-artifact@v4 + with: + name: web-data + path: website/data/ + + - name: Build website + run: python script.py + + + deploy: + needs: build + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Pages + uses: actions/configure-pages@v5 + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: 'website/public' + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index b7f4908..7909af3 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -1,4 +1,4 @@ -name: PR checks fon algorithm submission +name: PR checks on algorithm submission on: pull_request: diff --git a/.gitignore b/.gitignore index a1ca18a..0bae38d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ *$py.class .DS_Store *.csv +website/data/* # C extensions *.so @@ -153,4 +154,4 @@ dmypy.json .pytype/ # Cython debug symbols -cython_debug/ \ No newline at end of file +cython_debug/ diff --git a/README.md b/README.md index 8d68eaf..0578a48 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ flowchart LR subgraph S3 EDF[edf] TSVr[TSV ref] - TSVh[TSV s3hyp] + TSVh[TSV hyp] end subgraph Github Actions CI diff --git a/algorithms/gotman.yaml b/algorithms/gotman.yaml index c529c12..cc39354 100644 --- a/algorithms/gotman.yaml +++ b/algorithms/gotman.yaml @@ -13,7 +13,7 @@ date-released: "1982-01-01" abstract: > During prolonged EEG monitoring of epileptic patients, the continuous EEG tracing may be replaced by a selective recording of ictal and interictal - epileptic activity. We have described previously methods for the EEG + epileptic activity. We have previously described methods for the EEG recording of seizures with overt clinical manifestations and for the automatic detection of spikes. This paper describes a method for the automatic detection of seizures in the EEG, independently of the presence of clinical signs; it is diff --git a/config/template.Dockerfile b/config/template.Dockerfile index e96ec44..27fe36b 100644 --- a/config/template.Dockerfile +++ b/config/template.Dockerfile @@ -3,11 +3,6 @@ ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION}-slim as base -# Read aws creds -ENV AWS_ACCESS_KEY -ENV AWS_SECRET_KEY -ENV AWS_BUCKET - # Prevents Python from writing pyc files. ENV PYTHONDONTWRITEBYTECODE=1 # Keeps Python from buffering stdout and stderr to avoid situations where @@ -30,40 +25,29 @@ RUN adduser \ # Install S3 dependencies RUN apt-get update -y && \ apt-get install -y \ - s3fs \ - libfuse-dev \ - libcurl4-openssl-dev \ - libxml2-dev \ - libssl-dev \ - mime-support \ - automake \ - libtool \ wget \ tar \ git \ unzip && apt-get clean -RUN pip --no-cache-dir install --upgrade awscli -RUN mkdir -p /mnt/s3 - - # Download dependencies as a separate step to take advantage of Docker's caching. # Leverage a cache mount to /root/.cache/pip to speed up subsequent builds. # Leverage a bind mount to requirements.txt to avoid having to copy them into # into this layer. RUN --mount=type=cache,target=/root/.cache/pip \ --mount=type=bind,source=algo/,target=algo/ \ - python -m pip install ./algo + python -m pip install numpy # <-- install your algorithm # Switch to the non-privileged user to run the application. USER appuser +VOLUME ["/data"] VOLUME ["/output"] # Define input / output files -ENV INPUT_FILE="" -ENV OUTPUT_FILE="" +ENV INPUT="" +ENV OUTPUT="" # Run the application # NOTE: edit the second command -CMD s3fs ${AWS_BUCKET} /mnt/s3; python3 -m algo "/mnt/s3/input/${INPUT_FILE}" "/output/${OUTPUT_FILE}" +CMD python3 -m algo "/data/${INPUT_FILE}" "/output/${OUTPUT_FILE}" diff --git a/website/data/sampleEval.json b/website/data/sampleEval.json deleted file mode 100644 index 3f2e99d..0000000 --- a/website/data/sampleEval.json +++ /dev/null @@ -1,70 +0,0 @@ -[{ - "algo_id": "ANN", - "datasets": [ - { - "dataset": "chbmit", - "sample_results": { - "sensitivity": 1, - "precision": 1, - "f1": 1, - "fpRate": 0 - }, - "event_results": { - "sensitivity": 1, - "precision": 1, - "f1": 1, - "fpRate": 0 - } - }, - { - "dataset": "group2", - "sample_results": { - "sensitivity": 1, - "precision": 1, - "f1": 1, - "fpRate": 0 - }, - "event_results": { - "sensitivity": 1, - "precision": 1, - "f1": 1, - "fpRate": 0 - } - } - ] -}, -{ - "algo_id": "ANN", - "datasets": [ - { - "dataset": "chbmit", - "sample_results": { - "sensitivity": 1, - "precision": 1, - "f1": 1, - "fpRate": 0 - }, - "event_results": { - "sensitivity": 1, - "precision": 1, - "f1": 1, - "fpRate": 0 - } - }, - { - "dataset": "group2", - "sample_results": { - "sensitivity": 1, - "precision": 1, - "f1": 1, - "fpRate": 0 - }, - "event_results": { - "sensitivity": 1, - "precision": 1, - "f1": 1, - "fpRate": 0 - } - } - ] -}] \ No newline at end of file diff --git a/website/index.html b/website/index.html deleted file mode 100644 index a805108..0000000 --- a/website/index.html +++ /dev/null @@ -1,78 +0,0 @@ - - - - - - Epilepsy Benchmarks - - - - - -

Epilepsy Benchmarks

-

Standardising benchmarking procedures across epilepsy models. Datasets, performance scores

-
-
- -
- - - - - - - -
- -
- Performance Metrics -
- - -
-
- - -
-
- - -
-
- - -
-
- -
- Scoring Type - -
- - -
- -
- - -
-
- -
-
- -
-
-
-
- - - - - \ No newline at end of file diff --git a/website/public/ann.html b/website/public/ann.html new file mode 100644 index 0000000..b187da6 --- /dev/null +++ b/website/public/ann.html @@ -0,0 +1 @@ +

ANN page

\ No newline at end of file diff --git a/website/public/index.html b/website/public/index.html new file mode 100644 index 0000000..b71cfcb --- /dev/null +++ b/website/public/index.html @@ -0,0 +1,78 @@ + + + + + + Epilepsy Benchmarks + + + + + +

Epilepsy Benchmarks

+

Standardising benchmarking procedures across epilepsy models. Datasets, performance scores

+
+
+ +
+ + + + + + + +
+ +
+ Performance Metrics +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ Scoring Type + +
+ + +
+ +
+ + +
+
+ +
+
+ +
+
+
+
+ + + + + \ No newline at end of file diff --git a/website/script.py b/website/script.py index 8976d0c..1fbee27 100644 --- a/website/script.py +++ b/website/script.py @@ -2,8 +2,12 @@ import pandas as pd from layout import layout_with_figures import json, re +from algo_details_script import create_algo_page + +github_pages_root_url = "https://esl-epfl.github.io/szcore/" path_to_eval = './data/sampleEval.json' +path_to_algo_yaml = "../algorithms/" file = open(path_to_eval) metrics = ["Sensitivity", "Precision", "F1 Score", "fpRate"] # hardcoded @@ -17,7 +21,7 @@ dataset_name = dataset["dataset"] datasets.add(dataset_name) sample_results = dataset["sample_results"] - algo_html = "" + re.sub(r'[^a-zA-Z0-9]', '', algo_id) + "" + algo_html = "" + algo_id + "" row = {"algo_id": algo_html, "dataset": dataset_name, **sample_results} data_for_df.append(row) @@ -39,7 +43,8 @@ complete_html = layout_with_figures(plotly_html, datasets) # Save everything into a single HTML file -with open("index.html", "w") as file: +with open("./public/index.html", "w") as file: file.write(complete_html) # Create second HTML file for algo details (from yaml) +create_algo_page(path_to_algo_yaml) \ No newline at end of file