From 79b03c0d75740f4e92279cf0d390bded51526324 Mon Sep 17 00:00:00 2001 From: Sarthak Pati Date: Wed, 3 Apr 2024 09:03:54 -0400 Subject: [PATCH 1/3] updated test command to prepare data for ci --- .github/workflows/fets-challenge.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fets-challenge.yml b/.github/workflows/fets-challenge.yml index 4d517057f8..32a75ff7c1 100644 --- a/.github/workflows/fets-challenge.yml +++ b/.github/workflows/fets-challenge.yml @@ -42,7 +42,8 @@ jobs: pip uninstall onnx -y cat ./GANDLF/version.py echo "Download data and Split CSVs into training and validation" - python -c "from testing.test_full import test_generic_download_data, test_generic_constructTrainingCSV; test_generic_download_data(); test_generic_constructTrainingCSV()" + # python -c "from testing.test_full import test_generic_download_data, test_generic_constructTrainingCSV; test_generic_download_data(); test_generic_constructTrainingCSV()" + pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" head -n 1 testing/data/train_2d_rad_segmentation.csv > /home/runner/work/openfl/openfl/valid.csv tail -n +9 testing/data/train_2d_rad_segmentation.csv >> /home/runner/work/openfl/openfl/valid.csv head -n 8 testing/data/train_2d_rad_segmentation.csv > /home/runner/work/openfl/openfl/train.csv From 73969b84553b251081d2cb236988e5ce32336d80 Mon Sep 17 00:00:00 2001 From: Sarthak Pati Date: Wed, 3 Apr 2024 09:04:26 -0400 Subject: [PATCH 2/3] Delete .github/workflows/fets-challenge.yml --- .github/workflows/fets-challenge.yml | 75 ---------------------------- 1 file changed, 75 deletions(-) delete mode 100644 .github/workflows/fets-challenge.yml diff --git a/.github/workflows/fets-challenge.yml b/.github/workflows/fets-challenge.yml deleted file mode 100644 index 32a75ff7c1..0000000000 --- a/.github/workflows/fets-challenge.yml +++ /dev/null @@ -1,75 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: GaNDLF TaskRunner - -on: - pull_request: - branches: [ develop ] - -permissions: - contents: read - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install torch==2.1.0+cpu torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu - pip install . - - name: Install GaNDLF - run: | - git clone https://github.com/MLCommons/GaNDLF.git ./gandlf - cd gandlf - git fetch --tags - echo "Checkout the latest GaNDLF tag" - latestTag=$(git describe --tags "$(git rev-list --tags --max-count=1)") - git checkout $latestTag - - name: GaNDLF Task Runner Test - run: | - cd gandlf - pwd - pip install -e . - pip uninstall onnx -y - cat ./GANDLF/version.py - echo "Download data and Split CSVs into training and validation" - # python -c "from testing.test_full import test_generic_download_data, test_generic_constructTrainingCSV; test_generic_download_data(); test_generic_constructTrainingCSV()" - pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" - head -n 1 testing/data/train_2d_rad_segmentation.csv > /home/runner/work/openfl/openfl/valid.csv - tail -n +9 testing/data/train_2d_rad_segmentation.csv >> /home/runner/work/openfl/openfl/valid.csv - head -n 8 testing/data/train_2d_rad_segmentation.csv > /home/runner/work/openfl/openfl/train.csv - cp testing/config_segmentation.yaml /home/runner/work/openfl/openfl/config_segmentation.yaml - echo "DEBUG display the config file" - cat /home/runner/work/openfl/openfl/config_segmentation.yaml - echo "Initialize OpenFL plan" - ## from docs - export WORKSPACE_TEMPLATE=gandlf_seg_test - export WORKSPACE_PATH=./my_federation - fx workspace create --prefix ${WORKSPACE_PATH} --template ${WORKSPACE_TEMPLATE} - cd ${WORKSPACE_PATH} - mkdir ./data/one - mkdir ./data/two - cp /home/runner/work/openfl/openfl/*.csv ./data/one/ - cp /home/runner/work/openfl/openfl/*.csv ./data/two/ - ## from docs - # fx plan initialize --gandlf_config ../testing/config_segmentation.yaml - cd /home/runner/work/openfl/openfl - ls - file "/home/runner/work/openfl/openfl/config_segmentation.yaml" - ## for 2d data, only a single change is needed in the gandlf config - sed -i 's/# n_channels: 3/num_channels: 3/g' "/home/runner/work/openfl/openfl/config_segmentation.yaml" - ## for 3d data, the following changes are needed in the gandlf config -- commented out for now - # sed -i 's/dimension: 2/dimension: 3/g' "/home/runner/work/openfl/openfl/config_segmentation.yaml" - # sed -i 's/0,255/0,1/g' "/home/runner/work/openfl/openfl/config_segmentation.yaml" - # sed -i 's/128,128/32,32,32/g' "/home/runner/work/openfl/openfl/config_segmentation.yaml" - python -m tests.github.test_gandlf --template gandlf_seg_test --fed_workspace aggregator --col1 one --col2 two --rounds-to-train 1 --gandlf_config "/home/runner/work/openfl/openfl/config_segmentation.yaml" - From 2f59730cd3be3a4be0a008321bde723d1c7dfe91 Mon Sep 17 00:00:00 2001 From: Sarthak Pati Date: Wed, 3 Apr 2024 09:04:50 -0400 Subject: [PATCH 3/3] renamed file --- .github/workflows/gandlf.yml | 75 ++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/gandlf.yml diff --git a/.github/workflows/gandlf.yml b/.github/workflows/gandlf.yml new file mode 100644 index 0000000000..32a75ff7c1 --- /dev/null +++ b/.github/workflows/gandlf.yml @@ -0,0 +1,75 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: GaNDLF TaskRunner + +on: + pull_request: + branches: [ develop ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install torch==2.1.0+cpu torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu + pip install . + - name: Install GaNDLF + run: | + git clone https://github.com/MLCommons/GaNDLF.git ./gandlf + cd gandlf + git fetch --tags + echo "Checkout the latest GaNDLF tag" + latestTag=$(git describe --tags "$(git rev-list --tags --max-count=1)") + git checkout $latestTag + - name: GaNDLF Task Runner Test + run: | + cd gandlf + pwd + pip install -e . + pip uninstall onnx -y + cat ./GANDLF/version.py + echo "Download data and Split CSVs into training and validation" + # python -c "from testing.test_full import test_generic_download_data, test_generic_constructTrainingCSV; test_generic_download_data(); test_generic_constructTrainingCSV()" + pytest --cov=. --cov-report=xml -k "prepare_data_for_ci" + head -n 1 testing/data/train_2d_rad_segmentation.csv > /home/runner/work/openfl/openfl/valid.csv + tail -n +9 testing/data/train_2d_rad_segmentation.csv >> /home/runner/work/openfl/openfl/valid.csv + head -n 8 testing/data/train_2d_rad_segmentation.csv > /home/runner/work/openfl/openfl/train.csv + cp testing/config_segmentation.yaml /home/runner/work/openfl/openfl/config_segmentation.yaml + echo "DEBUG display the config file" + cat /home/runner/work/openfl/openfl/config_segmentation.yaml + echo "Initialize OpenFL plan" + ## from docs + export WORKSPACE_TEMPLATE=gandlf_seg_test + export WORKSPACE_PATH=./my_federation + fx workspace create --prefix ${WORKSPACE_PATH} --template ${WORKSPACE_TEMPLATE} + cd ${WORKSPACE_PATH} + mkdir ./data/one + mkdir ./data/two + cp /home/runner/work/openfl/openfl/*.csv ./data/one/ + cp /home/runner/work/openfl/openfl/*.csv ./data/two/ + ## from docs + # fx plan initialize --gandlf_config ../testing/config_segmentation.yaml + cd /home/runner/work/openfl/openfl + ls + file "/home/runner/work/openfl/openfl/config_segmentation.yaml" + ## for 2d data, only a single change is needed in the gandlf config + sed -i 's/# n_channels: 3/num_channels: 3/g' "/home/runner/work/openfl/openfl/config_segmentation.yaml" + ## for 3d data, the following changes are needed in the gandlf config -- commented out for now + # sed -i 's/dimension: 2/dimension: 3/g' "/home/runner/work/openfl/openfl/config_segmentation.yaml" + # sed -i 's/0,255/0,1/g' "/home/runner/work/openfl/openfl/config_segmentation.yaml" + # sed -i 's/128,128/32,32,32/g' "/home/runner/work/openfl/openfl/config_segmentation.yaml" + python -m tests.github.test_gandlf --template gandlf_seg_test --fed_workspace aggregator --col1 one --col2 two --rounds-to-train 1 --gandlf_config "/home/runner/work/openfl/openfl/config_segmentation.yaml" +