FRML-93 Add Dev Container Spec

.devcontainer/devcontainer.json

@@ -0,0 +1,16 @@
+{
+  "name": "frdc",
+  "build": {
+    "dockerfile": "../Dockerfile",
+  },
+  "containerEnv": {
+    "LABEL_STUDIO_HOST": "host.docker.internal",
+    "LABEL_STUDIO_API_KEY": "${localEnv:LABEL_STUDIO_API_KEY}",
+  },
+  "runArgs": [
+    "--gpus=all",
+  ],
+  "hostRequirements": {
+    "gpu": true,
+  }
+}

.gitattributes

@@ -0,0 +1 @@
+Dockerfile text=auto eol=lf

Dockerfile

@@ -0,0 +1,20 @@
+FROM pytorch/pytorch:2.1.2-cuda12.1-cudnn8-runtime as torch
+WORKDIR /devcontainer
+
+COPY ./pyproject.toml /devcontainer/pyproject.toml
+
+RUN apt update -y && apt upgrade -y
+RUN apt install git -y
+
+RUN pip3 install --upgrade pip && \
+    pip3 install poetry && \
+    pip3 install lightning
+
+RUN conda init bash \
+    && . ~/.bashrc \
+    && conda activate base \
+    && poetry config virtualenvs.create false \
+    && poetry install --with dev --no-interaction --no-ansi
+
+RUN apt install curl -y && curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH $PATH:/root/google-cloud-sdk/bin

README.md

@@ -54,14 +54,6 @@ To illustrate this, take a look at how
 `tests/model_tests/chestnut_dec_may/train.py` is written. It pulls in relevant
 modules from each stage and constructs a pipeline.
 
-
-> Initially, we evaluated a few ML E2E solutions, despite them offering great
-> functionality, their flexibility was
-> limited. From a dev perspective, **Active Learning** was a gray area, and we
-> foresee heavy shoehorning.
-> Ultimately, we decided that the risk was too great, thus we resort to
-> creating our own solution.
-
 ## Contributing
 
 ### Pre-commit Hooks
@@ -80,3 +72,5 @@ If you're using `pip` instead of `poetry`, run the following commands:
 pip install pre-commit
 pre-commit install
 ```
+
+Alternatively, you can use Black configured with your own IDE.

Writerside/d.tree

@@ -8,7 +8,9 @@
                   start-page="Overview.md">
 
     <toc-element topic="Overview.md"/>
-    <toc-element topic="Getting-Started.md"/>
+    <toc-element topic="Getting-Started.md">
+        <toc-element topic="Get-Started-with-Dev-Containers.md"/>
+    </toc-element>
     <toc-element toc-title="Tutorials">
         <toc-element topic="Retrieve-our-Datasets.md"/>
     </toc-element>

Writerside/topics/Get-Started-with-Dev-Containers.md

@@ -0,0 +1,49 @@
+# Get Started with Dev Containers
+
+Dev. Containers are a great way to get started with a project. They define all
+necessary dependencies and environments, so you can just start coding within
+the container.
+
+In this article, we'll only go over **additional steps** to set up with our
+project. For more information on how to use Dev Containers, please refer to
+the official documentation for each IDE. Once you've set up the Dev Container,
+come back here to finish the setup:
+
+- [VSCode](https://code.visualstudio.com/docs/remote/containers).
+- [IntelliJ](https://www.jetbrains.com/help/idea/connect-to-devcontainer.html)
+
+> If you see the error `Error response from daemon: ... <!DOCTYPE`, you forgot
+> the `.git` at the end of the repo URL.
+{style='warning'}
+
+## Python Environment
+
+> Do not create a new environment
+{style='warning'}
+
+The dev environment is already created and is managed by Anaconda 
+`/opt/conda/bin/conda`. 
+To activate the environment, run the following command:
+
+```bash
+conda activate base
+```
+
+> Refer to your respective IDE's documentation on how to activate the
+> environment.
+
+## Mark as Sources Root (Add to PYTHONPATH)
+
+For `import` statements to work, you need to mark the `src` folder as the
+sources root. Optionally, also mark the `tests` folder as the tests root.
+
+> Refer to your respective IDE's documentation on how to mark folders as
+> sources root. (Also known as adding to the `PYTHONPATH`)
+
+## Additional Setup
+
+Refer to the [Getting Started](Getting-Started.md) guide for additional setup
+steps such as:
+- Google Cloud Application Default Credentials
+- Weight & Bias API Key
+- Label Studio API Key

Writerside/topics/Getting-Started.md

@@ -1,5 +1,7 @@
 # Getting Started
 
+> Want to use a Dev Container? See [Get Started with Dev Containers](Get-Started-with-Dev-Containers.md)
+
 <procedure title="Installing the Dev. Environment" id="install">
     <step>Ensure that you have the right version of Python.
         The required Python version can be seen in <code>pyproject.toml</code>
@@ -10,7 +12,7 @@
     </step>
     <step>Start by cloning our repository.
         <code-block lang="shell">
-          git clone https://github.com/Forest-Recovery-Digital-Companion/FRDC-ML.git
+          git clone https://github.com/FR-DC/FRDC-ML.git
         </code-block>
     </step>
     <step>Then, create a Python Virtual Env <code>pyvenv</code>
@@ -60,6 +62,7 @@
     </step>
 </procedure>
 
+
 <procedure title="Setting Up Google Cloud" id="gcloud">
     <step>
         We use Google Cloud to store our datasets. To set up Google Cloud,
@@ -86,6 +89,48 @@
     </step>
 </procedure>
 
+<procedure title="Setting Up Label Studio" id="ls">
+    <tip>This is only necessary if any task requires Label Studio annotations</tip>
+    <step>
+        We use Label Studio to annotate our datasets.
+        We won't go through how to install Label Studio, for contributors, it
+        should be up on <code>localhost:8080</code>.
+    </step>
+    <step>
+        Then, retrieve your own API key from Label Studio.
+        <a href="http://localhost:8080/user/account"> Go to your account page </a>
+        and copy the API key. <br/></step>
+    <step> Set your API key as an environment variable.
+        <tabs>
+<tab title="Windows">
+        In Windows, go to "Edit environment variables for
+        your account" and add this as a new environment variable with name
+        <code>LABEL_STUDIO_API_KEY</code>.
+</tab>
+<tab title="Linux">
+        Export it as an environment variable.
+        <code-block lang="shell">export LABEL_STUDIO_API_KEY=...</code-block>
+</tab>
+</tabs>
+    </step>
+</procedure>
+
+<procedure title="Setting Up Weight and Biases" id="wandb">
+    <step>
+        We use W&B to track our experiments. To set up W&B,
+        <a href="https://docs.wandb.ai/quickstart">
+          install the W&B CLI
+        </a>
+    </step>
+    <step>
+        Then, 
+        <a href="https://docs.wandb.ai/quickstart">
+          authenticate your account
+        </a>.
+        <code-block lang="shell">wandb login</code-block>
+    </step>
+</procedure>
+
 <procedure title="Pre-commit Hooks" collapsible="true">
     <note>This is optional but recommended.
     Pre-commit hooks are a way to ensure that your code is formatted correctly.
@@ -98,30 +143,45 @@
     </step>
 </procedure>
 
-<procedure title="Running the Tests" collapsible="true" id="tests">
+<procedure title="Running the Tests" id="tests">
     <step>
         Run the tests to make sure everything is working
         <code-block lang="shell">
             pytest
         </code-block>
     </step>
-<step>
-    In case of errors:
-    <deflist>
-        <def title="google.auth.exceptions.DefaultCredentialsError">
-            If you get this error, it means that you haven't authenticated your
-            Google Cloud account.
-            See <a anchor="gcloud">Setting Up Google Cloud</a>
-        </def>
-        <def title="ModuleNotFoundError" collapsible="true">
-            If you get this error, it means that you haven't installed the
-            dependencies.
-            See <a anchor="install">Installing the Dev. Environment</a>
-        </def>
-    </deflist>
-</step>
 </procedure>
 
+## Troubleshooting
+
+### ModuleNotFoundError
+
+It's likely that your `src` and `tests` directories are not in `PYTHONPATH`.
+To fix this, run the following command:
+
+```shell
+export PYTHONPATH=$PYTHONPATH:./src:./tests
+```
+
+Or, set it in your IDE, for example, IntelliJ allows setting directories as
+**Source Roots**.
+
+### google.auth.exceptions.DefaultCredentialsError
+
+It's likely that you haven't authenticated your Google Cloud account.
+See [Setting Up Google Cloud](#gcloud)
+
+### Couldn't connect to Label Studio
+
+Label Studio must be running locally, exposed on `localhost:8080`. Furthermore,
+you need to specify the `LABEL_STUDIO_API_KEY` environment variable. See 
+[Setting Up Label Studio](#ls)
+
+### Cannot login to W&B
+
+You need to authenticate your W&B account. See [Setting Up Weight and Biases](#wandb)
+If you're facing difficulties, set the `WANDB_MODE` environment variable to `offline`
+to disable W&B.
 
 ## Our Repository Structure
 
@@ -132,15 +192,13 @@ help you understand where to put your code.
 graph LR
     FRDC -- " Core Dependencies " --> src/frdc/
     FRDC -- " Resources " --> rsc/
-    FRDC -- " Pipeline " --> pipeline/
     FRDC -- " Tests " --> tests/
     FRDC -- " Repo Dependencies " --> pyproject.toml,poetry.lock
     src/frdc/ -- " Dataset Loaders " --> ./load/
     src/frdc/ -- " Preprocessing Fn. " --> ./preprocess/
     src/frdc/ -- " Train Deps " --> ./train/
     src/frdc/ -- " Model Architectures " --> ./models/
     rsc/ -- " Datasets ... " --> ./dataset_name/
-    pipeline/ -- " Model Training Pipeline " --> ./model_tests/
 ```
 
 src/frdc/
@@ -149,19 +207,16 @@ src/frdc/
 rsc/
 : Resources. These are usually cached datasets
 
-pipeline/
-: Pipeline code. These are the full ML tests of our pipeline.
-
 tests/
-: PyTest tests. These are unit tests & integration tests.
+: PyTest tests. These are unit, integration, and model tests.
 
 ### Unit, Integration, and Pipeline Tests
 
 We have 3 types of tests:
 
 - Unit Tests are usually small, single function tests.
 - Integration Tests are larger tests that tests a mock pipeline.
-- Pipeline Tests are the true production pipeline tests that will generate a
+- Model Tests are the true production pipeline tests that will generate a
   model.
 
 ### Where Should I contribute?
@@ -176,9 +231,9 @@ at the <code>src/frdc/</code> directory.
 By adding a new component, you'll need to add a new test. Take a look at the
 <code>tests/</code> directory.
 </def>
-<def title="Changing the pipeline">
+<def title="Changing the model pipeline">
 If you're a ML Researcher, you'll probably be changing the pipeline. Take a
-look at the <code>pipeline/</code> directory.
+look at the <code>tests/model_tests/</code> directory.
 </def>
 <def title="Adding a dependency">
 If you're adding a new dependency, use <code>poetry add PACKAGE</code> and

Writerside/writerside.cfg

@@ -4,5 +4,5 @@
 <ihp version="2.0">
     <topics dir="topics" web-path="topics"/>
     <images dir="images" web-path="images"/>
-    <instance src="d.tree" web-path="/d/" version="0.0.7"/>
+    <instance src="d.tree" web-path="/d/" version="0.0.8"/>
 </ihp>

docs/HelpTOC.json

@@ -1 +1 @@
-{"entities":{"pages":{"Overview":{"id":"Overview","title":"Overview","url":"overview.html","level":0,"tabIndex":0},"Getting-Started":{"id":"Getting-Started","title":"Getting Started","url":"getting-started.html","level":0,"tabIndex":1},"ae6f1f90_3454":{"id":"ae6f1f90_3454","title":"Tutorials","level":0,"pages":["Retrieve-our-Datasets"],"tabIndex":2},"Retrieve-our-Datasets":{"id":"Retrieve-our-Datasets","title":"Retrieve our Datasets","url":"retrieve-our-datasets.html","level":1,"parentId":"ae6f1f90_3454","tabIndex":0},"mix-match":{"id":"mix-match","title":"MixMatch","url":"mix-match.html","level":0,"pages":["mix-match-module","custom-k-aug-dataloaders"],"tabIndex":3},"mix-match-module":{"id":"mix-match-module","title":"MixMatch Module","url":"mix-match-module.html","level":1,"parentId":"mix-match","tabIndex":0},"custom-k-aug-dataloaders":{"id":"custom-k-aug-dataloaders","title":"Custom K-Aug Dataloaders","url":"custom-k-aug-dataloaders.html","level":1,"parentId":"mix-match","tabIndex":1},"ae6f1f90_3459":{"id":"ae6f1f90_3459","title":"Model Tests","level":0,"pages":["Model-Test-Chestnut-May-Dec"],"tabIndex":4},"Model-Test-Chestnut-May-Dec":{"id":"Model-Test-Chestnut-May-Dec","title":"Model Test Chestnut May-Dec","url":"model-test-chestnut-may-dec.html","level":1,"parentId":"ae6f1f90_3459","tabIndex":0},"ae6f1f90_3461":{"id":"ae6f1f90_3461","title":"API","level":0,"pages":["load.dataset","load.gcs","preprocessing.scale","preprocessing.extract_segments","preprocessing.morphology","preprocessing.glcm_padded","train.frdc_lightning"],"tabIndex":5},"load.dataset":{"id":"load.dataset","title":"load.dataset","url":"load-dataset.html","level":1,"parentId":"ae6f1f90_3461","tabIndex":0},"load.gcs":{"id":"load.gcs","title":"load.gcs","url":"load-gcs.html","level":1,"parentId":"ae6f1f90_3461","tabIndex":1},"preprocessing.scale":{"id":"preprocessing.scale","title":"preprocessing.scale","url":"preprocessing-scale.html","level":1,"parentId":"ae6f1f90_3461","tabIndex":2},"preprocessing.extract_segments":{"id":"preprocessing.extract_segments","title":"preprocessing.extract_segments","url":"preprocessing-extract-segments.html","level":1,"parentId":"ae6f1f90_3461","tabIndex":3},"preprocessing.morphology":{"id":"preprocessing.morphology","title":"preprocessing.morphology","url":"preprocessing-morphology.html","level":1,"parentId":"ae6f1f90_3461","tabIndex":4},"preprocessing.glcm_padded":{"id":"preprocessing.glcm_padded","title":"preprocessing.glcm_padded","url":"preprocessing-glcm-padded.html","level":1,"parentId":"ae6f1f90_3461","tabIndex":5},"train.frdc_lightning":{"id":"train.frdc_lightning","title":"train.frdc_datamodule \u0026 frdc_module","url":"train-frdc-lightning.html","level":1,"parentId":"ae6f1f90_3461","tabIndex":6}}},"topLevelIds":["Overview","Getting-Started","ae6f1f90_3454","mix-match","ae6f1f90_3459","ae6f1f90_3461"]}
+{"entities":{"pages":{"Overview":{"id":"Overview","title":"Overview","url":"overview.html","level":0,"tabIndex":0},"Getting-Started":{"id":"Getting-Started","title":"Getting Started","url":"getting-started.html","level":0,"pages":["Get-Started-with-Dev-Containers"],"tabIndex":1},"Get-Started-with-Dev-Containers":{"id":"Get-Started-with-Dev-Containers","title":"Get Started with Dev Containers","url":"get-started-with-dev-containers.html","level":1,"parentId":"Getting-Started","tabIndex":0},"e8e19623_66291":{"id":"e8e19623_66291","title":"Tutorials","level":0,"pages":["Retrieve-our-Datasets"],"tabIndex":2},"Retrieve-our-Datasets":{"id":"Retrieve-our-Datasets","title":"Retrieve our Datasets","url":"retrieve-our-datasets.html","level":1,"parentId":"e8e19623_66291","tabIndex":0},"mix-match":{"id":"mix-match","title":"MixMatch","url":"mix-match.html","level":0,"pages":["mix-match-module","custom-k-aug-dataloaders"],"tabIndex":3},"mix-match-module":{"id":"mix-match-module","title":"MixMatch Module","url":"mix-match-module.html","level":1,"parentId":"mix-match","tabIndex":0},"custom-k-aug-dataloaders":{"id":"custom-k-aug-dataloaders","title":"Custom K-Aug Dataloaders","url":"custom-k-aug-dataloaders.html","level":1,"parentId":"mix-match","tabIndex":1},"e8e19623_66296":{"id":"e8e19623_66296","title":"Model Tests","level":0,"pages":["Model-Test-Chestnut-May-Dec"],"tabIndex":4},"Model-Test-Chestnut-May-Dec":{"id":"Model-Test-Chestnut-May-Dec","title":"Model Test Chestnut May-Dec","url":"model-test-chestnut-may-dec.html","level":1,"parentId":"e8e19623_66296","tabIndex":0},"e8e19623_66298":{"id":"e8e19623_66298","title":"API","level":0,"pages":["load.dataset","load.gcs","preprocessing.scale","preprocessing.extract_segments","preprocessing.morphology","preprocessing.glcm_padded","train.frdc_lightning"],"tabIndex":5},"load.dataset":{"id":"load.dataset","title":"load.dataset","url":"load-dataset.html","level":1,"parentId":"e8e19623_66298","tabIndex":0},"load.gcs":{"id":"load.gcs","title":"load.gcs","url":"load-gcs.html","level":1,"parentId":"e8e19623_66298","tabIndex":1},"preprocessing.scale":{"id":"preprocessing.scale","title":"preprocessing.scale","url":"preprocessing-scale.html","level":1,"parentId":"e8e19623_66298","tabIndex":2},"preprocessing.extract_segments":{"id":"preprocessing.extract_segments","title":"preprocessing.extract_segments","url":"preprocessing-extract-segments.html","level":1,"parentId":"e8e19623_66298","tabIndex":3},"preprocessing.morphology":{"id":"preprocessing.morphology","title":"preprocessing.morphology","url":"preprocessing-morphology.html","level":1,"parentId":"e8e19623_66298","tabIndex":4},"preprocessing.glcm_padded":{"id":"preprocessing.glcm_padded","title":"preprocessing.glcm_padded","url":"preprocessing-glcm-padded.html","level":1,"parentId":"e8e19623_66298","tabIndex":5},"train.frdc_lightning":{"id":"train.frdc_lightning","title":"train.frdc_datamodule \u0026 frdc_module","url":"train-frdc-lightning.html","level":1,"parentId":"e8e19623_66298","tabIndex":6}}},"topLevelIds":["Overview","Getting-Started","e8e19623_66291","mix-match","e8e19623_66296","e8e19623_66298"]}