Merge pull request #78 from FR-DC/0.1.2

Update Docs

.env.example

@@ -0,0 +1,5 @@
+LABEL_STUDIO_API_KEY=
+LABEL_STUDIO_HOST=10.97.41.70
+LABEL_STUDIO_PORT=8080
+GCS_PROJECT_ID=frmodel
+GCS_BUCKET_NAME=frdc-ds

.gitattributes

@@ -1 +1,3 @@
-Dockerfile text=auto eol=lf
+Dockerfile text=auto eol=lf
+* text=auto
+*.ipynb -linguist-detectable

.github/workflows/basic-tests.yml

@@ -4,6 +4,9 @@ on:
   push:
     branches: [ "main" ]
   pull_request:
+    paths:
+      - "src/frdc/**"
+      - "poetry.lock"
 
 jobs:
   build:
@@ -38,7 +41,6 @@ jobs:
           python -m pip install flake8 pytest poetry          
           poetry export --with dev --without-hashes -o requirements.txt
           pip install -r requirements.txt
-          pip install torch torchaudio torchvision lightning
 
       - name: Lint with flake8
         run: |
@@ -47,6 +49,10 @@ jobs:
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 src/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
+      - name: Copy over .env
+        run: |
+          cp .env.example .env
+
       - name: Test with pytest
         run: |
           pytest

.github/workflows/build-ls-image.yml

@@ -0,0 +1,47 @@
+name: Publish Custom Label Studio Image
+on:
+  push:
+    branches: [ 'terraform' ]
+    paths:
+      - 'src/terraform/Dockerfile'
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/[email protected]
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        uses: docker/[email protected]
+        with:
+          context: ./src/terraform
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/model-tests.yml

@@ -53,7 +53,6 @@ jobs:
           python3 -m pip install flake8 pytest poetry
           poetry export --with dev --without-hashes -o requirements.txt
           pip3 install -r requirements.txt
-          pip3 install torch torchvision torchaudio
 
       - name: Check CUDA is available
         run: nvidia-smi
@@ -81,11 +80,16 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
+      - name: Copy over .env
+        run: |
+          cp .env.example .env
+          
       - name: Run Model Training
         working-directory: ${{ github.workspace }}/tests
         run: |
           git config --global --add safe.directory /__w/FRDC-ML/FRDC-ML
-          python3 -m model_tests.chestnut_dec_may.train
+          python3 -m model_tests.chestnut_dec_may.train_fixmatch
+          python3 -m model_tests.chestnut_dec_may.train_mixmatch
 
       - name: Comment results via CML
         run: |

.gitignore

@@ -161,6 +161,43 @@ cython_debug/
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 .idea/
 
+# Terraform Ignores
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+
+
 # Ignores the raw .tif files
 rsc/**/*.tif
 

Writerside/d.tree

@@ -8,6 +8,7 @@
                   start-page="Overview.md">
 
     <toc-element topic="Overview.md"/>
+    <toc-element topic="ML-Architecture.md"/>
     <toc-element topic="Getting-Started.md">
         <toc-element topic="Get-Started-with-Dev-Containers.md"/>
     </toc-element>
@@ -28,6 +29,5 @@
         <toc-element topic="preprocessing.extract_segments.md"/>
         <toc-element topic="preprocessing.morphology.md"/>
         <toc-element topic="preprocessing.glcm_padded.md"/>
-        <toc-element topic="train.frdc_lightning.md"/>
     </toc-element>
 </instance-profile>

Writerside/topics/Get-Started-with-Dev-Containers.md

@@ -47,3 +47,8 @@ steps such as:
 - Google Cloud Application Default Credentials
 - Weight & Bias API Key
 - Label Studio API Key
+
+> You can set the API Keys in the `.env` file in the root of the project.
+> Be careful not to commit the `.env` file to the repository, which should
+> have been ignored by default.
+{style='note'}