Merge pull request #4 from CupOfGeo/tf-docs

Tf docs

.devcontainer/devcontainer.json

@@ -29,7 +29,6 @@
 			]
 		}
 	}
-
 	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
 	// "remoteUser": "root"
 }

.github/workflows/cloudrun-deploy.yaml

@@ -0,0 +1,120 @@
+# This workflow build and push a Docker container to Google Artifact Registry and deploy it on Cloud Run when a commit is pushed to the "main" branch
+#
+# Overview:
+#
+# 1. Authenticate to Google Cloud
+# 2. Authenticate Docker to Artifact Registry
+# 3. Build a docker container
+# 4. Publish it to Google Artifact Registry
+# 5. Deploy it to Cloud Run
+#
+# To configure this workflow:
+#
+# 1. Ensure the required Google Cloud APIs are enabled:
+#
+#    Cloud Run            run.googleapis.com
+#    Artifact Registry    artifactregistry.googleapis.com
+#
+# 2. Create and configure Workload Identity Federation for GitHub (https://github.com/google-github-actions/auth#setting-up-workload-identity-federation)
+#
+# 3. Ensure the required IAM permissions are granted
+#
+#    Cloud Run
+#      roles/run.admin
+#      roles/iam.serviceAccountUser     (to act as the Cloud Run runtime service account)
+#
+#    Artifact Registry
+#      roles/artifactregistry.admin     (project or repository level)
+#
+#    NOTE: You should always follow the principle of least privilege when assigning IAM roles
+#
+# 4. Create GitHub secrets for WIF_PROVIDER and WIF_SERVICE_ACCOUNT
+#
+# 5. Change the values for the GAR_LOCATION, SERVICE and REGION environment variables (below).
+#
+# NOTE: To use Google Container Registry instead, replace ${{ env.GAR_LOCATION }}-docker.pkg.dev with gcr.io
+#
+# For more support on how to run this workflow, please visit https://github.com/marketplace/actions/deploy-to-cloud-run
+#
+# Further reading:
+#   Cloud Run IAM permissions                 - https://cloud.google.com/run/docs/deploying
+#   Artifact Registry IAM permissions         - https://cloud.google.com/artifact-registry/docs/access-control#roles
+#   Container Registry vs Artifact Registry   - https://cloud.google.com/blog/products/application-development/understanding-artifact-registry-vs-container-registry
+#   Principle of least privilege              - https://cloud.google.com/blog/products/identity-security/dont-get-pwned-practicing-the-principle-of-least-privilege
+name: Build and Deploy to Cloud Run
+
+on:
+  push:
+    branches: [ main ]
+
+env:
+  PROJECT_ID: "geo-attractors"
+  GAR_LOCATION: "us-central1"
+  REGION: 'us-central1'
+  REPO: 'attractors'
+
+jobs:
+  deploy:
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Google Auth
+        id: auth
+        uses: 'google-github-actions/auth@v1'
+        with:
+          token_format: 'access_token'
+          workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' # e.g. - projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider
+          service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' # e.g. - [email protected]
+
+      # Authenticate Docker to Google Cloud Artifact Registry
+      - name: Login to GAR
+        uses: docker/login-action@v3
+        with:
+          registry: '${{ env.GAR_LOCATION }}-docker.pkg.dev'
+          username: 'oauth2accesstoken'
+          password: ${{ steps.auth.outputs.access_token }}
+
+      - name: Build and Push Backend
+        run: |-
+          cd backend/
+          docker build --target BUILD -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest" ./
+          docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest"
+
+      - name: Build and Push Frontend
+        run: |-
+          cd frontend/
+          docker build --target BUILD -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest" ./
+          docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest"
+
+      # END - Docker auth and build
+
+      - name: Deploy Backend to Cloud Run
+        id: deploy
+        uses: google-github-actions/deploy-cloudrun@v2
+        with:
+          service: attractors-backend-service
+          region: ${{ env.REGION }}
+          # NOTE: If using a pre-built image, update the image name here
+          image: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest
+
+      # If required, use the Cloud Run url output in later steps
+      - name: Show Backend URL
+        run: echo ${{ steps.deploy.outputs.url }}
+
+      - name: Deploy Frontend to Cloud Run
+        id: deploy-frontend
+        uses: google-github-actions/deploy-cloudrun@v2
+        with:
+          service: attractors-frontend-service
+          region: ${{ env.REGION }}
+          # NOTE: If using a pre-built image, update the image name here
+          image: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest
+
+      - name: Show Frontend URL
+        run: echo ${{ steps.deploy-frontend.outputs.url }}

.pre-commit-config.yaml

@@ -54,3 +54,9 @@ repos:
     rev: v1.10.0
     hooks:
       - id: python-check-blanket-noqa
+
+  - repo: https://github.com/antonbabenko/pre-commit-terraform
+    rev: v1.86.0
+    hooks:
+      - id: terraform_tflint
+      - id: terraform_fmt

.vscode/launch.json

@@ -5,10 +5,19 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "FastAPI",
+            "name": "Dash Frontend",
             "type": "python",
             "request": "launch",
             "module": "src",
+            "justMyCode": true,
+            "cwd": "${workspaceFolder}/frontend",
+            "python": "${workspaceFolder}/frontend/.venv-front/bin/python"
+        },
+        {
+            "name": "FastAPI",
+            "type": "python",
+            "request": "launch",
+            "module": "backend/src",
             "justMyCode": true
         }
     ]

README.md

@@ -2,39 +2,45 @@
 Hey so a long time ago i found this https://examples.holoviz.org/attractors/attractors.html project and thought it looked really cool so i played around with it and wanted to turn it into a rest api.
 
 It was a super cool demo learned about jit and datashaders
+TODO thank the guy that gave the talk scaling open source science.
 
 It going to make me pretty pictures and the server will be a good test service for other things.
 
+# dev containers and CodeSpaces
 going to learn to use the .devcontainers
 
 so im now in a clean 3.11 dev container :) I have a bash terminal
 
 Cool so i updated my pre-commit hooks as well
 
-Im running without a venv bc im already in a clean room
-
-# Redis
-Idea to add some sort of caching for the attractors so if i want to recolor it i can pull the initial conditions from a cache
+Im running without a venv bc im already in a clean room. This is great now I can install all my dev dependencies and then make a clean .venv with only whats needed to deploy.
 
+## Issues
+- How can i set the default shell to zsh? it is an option
+- Can't build or run Dockerfile in the container.
+possibly I could pass the local machines docker into the container
 ```bash
-docker pull redis
-# -p and --name must come before -d
-docker run -p 6379:6379 --name my-redis -d redis
-docker exec -it my-redis redis-cli
-
-set my-key "hello redis"
-get my-key
+ls -la /var/run/docker.sock
+docker run -v /var/run/docker.sock:/var/run/docker.sock -it your-dev-container-image
 ```
 
-```bash
-pip install redis
-pip install aioredis
+- I'm running redis on port 6379 but i cant connect to it bc im in a container
+    - ok so i had to do this
+    ```bash
+    docker network create my_network
+    # re run a new redis instance with the --network
+    docker run --name my_redis_container --network my_network -p 6379:6379 -d redis
+    # i had to also add my_network to the build args of this in devcontainer.json
+    ```
+- super strange git issue
 ```
+git add src/
+fatal: detected dubious ownership in repository at '/workspaces/attractorsIII'
+To add an exception for this directory, call:
 
-Redis is cool but its overkill I can just use `from cachetools import TTLCache`
-No thats not async enough im going to use `from aiocache import caches`
+        git config --global --add safe.directory /workspaces/attractorsIII
+```
 
-Im now seeing in jit that it can do caching. But that wouldn't be helpful as my function does one iteration at a time and i just cache the resulting 10000000 iterations with the initial conditions. I also compress it with gzip so which is nice too.
 
 # Deploying
 - I created a new GCP project geo-attractors.
@@ -44,7 +50,7 @@ Im now seeing in jit that it can do caching. But that wouldn't be helpful as my
 
 - starting with cloud run made some terraform and some startup scripts
 had to send it to gcloud to build 1. on the m1 mac the build is strange and 2. can't build in the docker container
-`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-fastapi`
+`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-backend"`
 this will be solved when its just automatic with the ci/cd pipeline thats totally coming soon
 
 
@@ -55,32 +61,25 @@ this will be solved when its just automatic with the ci/cd pipeline thats totall
     - added backend to main.tf then did a `terraform init`
 
 
-# Issues
-- Learn how to write tests
-- Terminal auto complete
-- Can't build or run Dockerfile in the container.
-possibly I could pass the local machines docker into the container
-```bash
-ls -la /var/run/docker.sock
-docker run -v /var/run/docker.sock:/var/run/docker.sock -it your-dev-container-image
+# two services one repo
+each service will have its own venv and requirements.txt. But the requirements.txt in the root of this project will have all the dev dependencies need to run pre-commit hooks and any other project dependencies.
+
+```mermaid
+graph LR
+    A[Dash Frontend] <--> B[Backend FastAPI]
 ```
 
-- I'm running redis on port 6379 but i cant connect to it bc im in a container
-    - ok so i had to do this
-    ```bash
-    docker network create my_network
-    # re run a new redis instance with the --network
-    docker run --name my_redis_container --network my_network -p 6379:6379 -d redis
-    # i had to also add my_network to the build args of this in devcontainer.json
-    ```
 
-- Brew 🍺 would be nice
-- super strange git issue
-```
-git add src/
-fatal: detected dubious ownership in repository at '/workspaces/attractorsIII'
-To add an exception for this directory, call:
+# GH workload federation stuff
 
-        git config --global --add safe.directory /workspaces/attractorsIII
+[Here is the module I used](https://github.com/terraform-google-modules/terraform-google-github-actions-runners/blob/master/modules/gh-oidc/README.md
+)
+
+- need to enable IAM Service Account Credentials API
+
+- two variables need to be set you get them after applying the terraform and then running `terraform output` and then use the value from the outputs
+`gh_federation_provider_name and gh_federation_sa_email`
+```bash
+gh secret set WIF_PROVIDER -b"gh_federation_provider_name"
+gh secret set WIF_SERVICE_ACCOUNT -b"gh_federation_sa_email"
 ```
-- doesn't open properly in code-space got an error on creations will check later at the airport on my ipad. not sure if it was the my network or i had an extra comma that broke it probably the latter as the error log didn't say much about the build arg its very slow probably need to give it more compute. could it be that its using my wifi to download packages I don't think so.

backend/.dockerignore

@@ -1,5 +1,5 @@
 **/__pycache__
-**/.venv
+**/.venv*
 **/.classpath
 **/.dockerignore
 **/.env
@@ -17,7 +17,6 @@
 **/charts
 **/docker-compose*
 **/compose*
-**/Dockerfile*
 **/node_modules
 **/npm-debug.log
 **/obj

backend/Dockerfile

@@ -1,5 +1,5 @@
-### PROD ###
-FROM python:3.11-slim-buster as prod
+### BUILD ###
+FROM python:3.11-slim-buster as BUILD
 RUN apt-get update
 # Set the working directory to /app
 WORKDIR /app

backend/README.md

@@ -0,0 +1,3 @@
+This backend uses FastAPI.
+
+`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-backend" --ignore-file .dockerignore`

backend/requirements.txt

@@ -1,16 +1,49 @@
+aiocache==0.12.2
 annotated-types==0.6.0
 anyio==4.2.0
+certifi==2023.11.17
+charset-normalizer==3.3.2
 click==8.1.7
+cloudpickle==3.0.0
+colorcet==3.0.1
+dask==2023.12.1
+datashader==0.16.0
 fastapi==0.108.0
+fsspec==2023.12.2
 h11==0.14.0
 idna==3.6
+importlib-metadata==7.0.1
+llvmlite==0.41.1
+locket==1.0.0
+loguru==0.7.2
+multipledispatch==1.0.0
+numba==0.58.1
+numpy==1.26.3
+orjson==3.9.10
+packaging==23.2
+pandas==2.1.4
+param==2.0.1
+partd==1.4.1
+pillow==10.2.0
 prometheus-client==0.19.0
 prometheus-fastapi-instrumentator==6.1.0
+pyct==0.5.0
 pydantic==2.5.3
 pydantic-settings==2.1.0
 pydantic_core==2.14.6
+python-dateutil==2.8.2
 python-dotenv==1.0.0
+pytz==2023.3.post1
+PyYAML==6.0.1
+requests==2.31.0
+scipy==1.11.4
+six==1.16.0
 sniffio==1.3.0
 starlette==0.32.0.post1
+toolz==0.12.0
 typing_extensions==4.9.0
+tzdata==2023.4
+urllib3==2.1.0
 uvicorn==0.25.0
+xarray==2023.12.0
+zipp==3.17.0

backend/src/api/attractors/attractor_service.py

@@ -36,8 +36,9 @@ def trajectory(self, fn, x0, y0, a, b=0, c=0, d=0, e=0, f=0, n=10000000):
     def gen_random(self, func, desired_empty=10000):
         # finds some nice  initial conditions
         non_empty = 0
+        tries = 0
         # how many non empty pixels
-        while non_empty < desired_empty:
+        while non_empty < desired_empty or tries <= 10:
             initial_conditions = np.c_[
                 np.zeros((1, 2)), np.random.random((1, 6)) * 4 - 2
             ][0]
@@ -48,6 +49,7 @@ def gen_random(self, func, desired_empty=10000):
             agg = cvs.points(df, "x", "y")
             non_empty = np.count_nonzero(np.array(agg))
             logger.info(f"non_empty: {non_empty}")
+            tries += 1
         return initial_conditions
 
     def make_dataframe(

frontend/.dockerignore

@@ -1,5 +1,5 @@
 **/__pycache__
-**/.venv
+**/.venv*
 **/.classpath
 **/.dockerignore
 **/.env
@@ -17,7 +17,6 @@
 **/charts
 **/docker-compose*
 **/compose*
-**/Dockerfile*
 **/node_modules
 **/npm-debug.log
 **/obj