Merge pull request #27 from FR-DC/0.0.6

0.0.6

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,6 @@
+[core]
+    remote = frdc-ds
+    autostage = true
+['remote "frdc-ds"']
+    url = gs://frdc-ds/
+    version_aware = true

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

.github/workflows/python-package.yml → .github/workflows/basic-tests.yml

@@ -38,6 +38,7 @@ jobs:
           python -m pip install flake8 pytest poetry          
           poetry export --with dev --without-hashes -o requirements.txt
           pip install -r requirements.txt
+          pip install torch torchaudio torchvision lightning
 
       - name: Lint with flake8
         run: |

.github/workflows/model.yml → .github/workflows/model-tests.yml

@@ -2,6 +2,9 @@ name: Model Training
 
 on:
   pull_request:
+    branches: ['main']
+  workflow_dispatch:
+
 
 jobs:
   build:
@@ -12,7 +15,9 @@ jobs:
       volumes:
         - /home/runner/work/frdc-ml/_github_home:/root
       env:
+        # This is where setup-python will install and cache the venv
         AGENT_TOOLSDIRECTORY: "/root/venv"
+      options: --gpus all
 
     steps:
       - uses: actions/checkout@v3
@@ -34,7 +39,10 @@ jobs:
           pip3 install -r requirements.txt
           pip3 install torch torchvision torchaudio
 
-      - name: Set up gcloud
+      - name: Check CUDA is available
+        run: nvidia-smi
+
+      - name: Auth gcloud
         id: 'auth'
         uses: 'google-github-actions/auth@v1'
         with:
@@ -47,13 +55,15 @@ jobs:
         run: |
           echo "WANDB_API_KEY=${{ secrets.WANDB_API_KEY }}" >> $GITHUB_ENV
 
+      # Our project has src as a source path, explicitly add that in.
       - name: Add src as PYTHONPATH
         run: |
           echo "PYTHONPATH=src" >> $GITHUB_ENV
 
+      # Do not do cd as it'll break PYTHONPATH.
       - name: Run Model Training
         run: |
-          python3 -m tests.model_tests.chestnut_dec_may.main
+          python3 -m tests.model_tests.chestnut_dec_may.train
 
       - name: Comment results via CML
         run: |

.gitignore

@@ -165,4 +165,8 @@ cython_debug/
 rsc/**/*.tif
 
 **/*/lightning_logs
-*.zip
+*.zip
+*.sh
+*.ckpt
+/rsc
+**/wandb/

pyproject.toml

@@ -12,31 +12,27 @@ python = ">=3.9,<3.12"
 numpy = "^1.25.2"
 scikit-image = "0.22.0"
 scikit-learn = "^1.3.0"
-seaborn = "^0.12.2"
+seaborn = "^0.13.0"
 matplotlib = "^3.7.3"
 google-cloud-storage = "^2.10.0"
 tqdm = "^4.66.1"
 xxhash = "^3.4.1"
 
-
-[[tool.poetry.source]]
-name = "pytorch"
-url = "https://download.pytorch.org/whl/cu121"
-priority = "explicit"
-
+#torch = {version="^2.1.0", source="pytorch"}
+#torchvision = {version="^0.16.0", source="pytorch"}
+#torchaudio = {version="^2.1.0", source="pytorch"}
+#lightning = "^2.0.9.post0"
 
 [tool.poetry.group.dev.dependencies]
-torch = "^2.1.0"
-torchvision = "^0.16.0"
-torchaudio = "^2.1.0"
-lightning = "^2.0.9.post0"
 pytest = "^7.4.2"
 pre-commit = "^3.5.0"
 black = "^23.10.0"
 flake8 = "^6.1.0"
 wandb = "^0.16.0"
 
 
+
+
 [tool.poetry.group.glcm.dependencies]
 glcm-cupy = "0.2.1"
 cupy-cuda12x = "^12.2.0"