diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d48136f..a6a7937 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -29,7 +29,6 @@ ] } } - // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "root" } diff --git a/.github/workflows/cloudrun-deploy.yaml b/.github/workflows/cloudrun-deploy.yaml new file mode 100644 index 0000000..f40ae87 --- /dev/null +++ b/.github/workflows/cloudrun-deploy.yaml @@ -0,0 +1,120 @@ +# This workflow build and push a Docker container to Google Artifact Registry and deploy it on Cloud Run when a commit is pushed to the "main" branch +# +# Overview: +# +# 1. Authenticate to Google Cloud +# 2. Authenticate Docker to Artifact Registry +# 3. Build a docker container +# 4. Publish it to Google Artifact Registry +# 5. Deploy it to Cloud Run +# +# To configure this workflow: +# +# 1. Ensure the required Google Cloud APIs are enabled: +# +# Cloud Run run.googleapis.com +# Artifact Registry artifactregistry.googleapis.com +# +# 2. Create and configure Workload Identity Federation for GitHub (https://github.com/google-github-actions/auth#setting-up-workload-identity-federation) +# +# 3. Ensure the required IAM permissions are granted +# +# Cloud Run +# roles/run.admin +# roles/iam.serviceAccountUser (to act as the Cloud Run runtime service account) +# +# Artifact Registry +# roles/artifactregistry.admin (project or repository level) +# +# NOTE: You should always follow the principle of least privilege when assigning IAM roles +# +# 4. Create GitHub secrets for WIF_PROVIDER and WIF_SERVICE_ACCOUNT +# +# 5. Change the values for the GAR_LOCATION, SERVICE and REGION environment variables (below). +# +# NOTE: To use Google Container Registry instead, replace ${{ env.GAR_LOCATION }}-docker.pkg.dev with gcr.io +# +# For more support on how to run this workflow, please visit https://github.com/marketplace/actions/deploy-to-cloud-run +# +# Further reading: +# Cloud Run IAM permissions - https://cloud.google.com/run/docs/deploying +# Artifact Registry IAM permissions - https://cloud.google.com/artifact-registry/docs/access-control#roles +# Container Registry vs Artifact Registry - https://cloud.google.com/blog/products/application-development/understanding-artifact-registry-vs-container-registry +# Principle of least privilege - https://cloud.google.com/blog/products/identity-security/dont-get-pwned-practicing-the-principle-of-least-privilege +name: Build and Deploy to Cloud Run + +on: + push: + branches: [ main ] + +env: + PROJECT_ID: "geo-attractors" + GAR_LOCATION: "us-central1" + REGION: 'us-central1' + REPO: 'attractors' + +jobs: + deploy: + permissions: + contents: 'read' + id-token: 'write' + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v1' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' # e.g. - projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' # e.g. - my-service-account@my-project.iam.gserviceaccount.com + + # Authenticate Docker to Google Cloud Artifact Registry + - name: Login to GAR + uses: docker/login-action@v3 + with: + registry: '${{ env.GAR_LOCATION }}-docker.pkg.dev' + username: 'oauth2accesstoken' + password: ${{ steps.auth.outputs.access_token }} + + - name: Build and Push Backend + run: |- + cd backend/ + docker build --target BUILD -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest" ./ + docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest" + + - name: Build and Push Frontend + run: |- + cd frontend/ + docker build --target BUILD -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest" ./ + docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest" + + # END - Docker auth and build + + - name: Deploy Backend to Cloud Run + id: deploy + uses: google-github-actions/deploy-cloudrun@v2 + with: + service: attractors-backend-service + region: ${{ env.REGION }} + # NOTE: If using a pre-built image, update the image name here + image: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest + + # If required, use the Cloud Run url output in later steps + - name: Show Backend URL + run: echo ${{ steps.deploy.outputs.url }} + + - name: Deploy Frontend to Cloud Run + id: deploy-frontend + uses: google-github-actions/deploy-cloudrun@v2 + with: + service: attractors-frontend-service + region: ${{ env.REGION }} + # NOTE: If using a pre-built image, update the image name here + image: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest + + - name: Show Frontend URL + run: echo ${{ steps.deploy-frontend.outputs.url }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 274722a..9ff91ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,3 +54,9 @@ repos: rev: v1.10.0 hooks: - id: python-check-blanket-noqa + + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.86.0 + hooks: + - id: terraform_tflint + - id: terraform_fmt diff --git a/.vscode/launch.json b/.vscode/launch.json index 4e28f43..23697d5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,10 +5,19 @@ "version": "0.2.0", "configurations": [ { - "name": "FastAPI", + "name": "Dash Frontend", "type": "python", "request": "launch", "module": "src", + "justMyCode": true, + "cwd": "${workspaceFolder}/frontend", + "python": "${workspaceFolder}/frontend/.venv-front/bin/python" + }, + { + "name": "FastAPI", + "type": "python", + "request": "launch", + "module": "backend/src", "justMyCode": true } ] diff --git a/README.md b/README.md index 1650481..b6a5baf 100644 --- a/README.md +++ b/README.md @@ -2,39 +2,45 @@ Hey so a long time ago i found this https://examples.holoviz.org/attractors/attractors.html project and thought it looked really cool so i played around with it and wanted to turn it into a rest api. It was a super cool demo learned about jit and datashaders +TODO thank the guy that gave the talk scaling open source science. It going to make me pretty pictures and the server will be a good test service for other things. +# dev containers and CodeSpaces going to learn to use the .devcontainers so im now in a clean 3.11 dev container :) I have a bash terminal Cool so i updated my pre-commit hooks as well -Im running without a venv bc im already in a clean room - -# Redis -Idea to add some sort of caching for the attractors so if i want to recolor it i can pull the initial conditions from a cache +Im running without a venv bc im already in a clean room. This is great now I can install all my dev dependencies and then make a clean .venv with only whats needed to deploy. +## Issues +- How can i set the default shell to zsh? it is an option +- Can't build or run Dockerfile in the container. +possibly I could pass the local machines docker into the container ```bash -docker pull redis -# -p and --name must come before -d -docker run -p 6379:6379 --name my-redis -d redis -docker exec -it my-redis redis-cli - -set my-key "hello redis" -get my-key +ls -la /var/run/docker.sock +docker run -v /var/run/docker.sock:/var/run/docker.sock -it your-dev-container-image ``` -```bash -pip install redis -pip install aioredis +- I'm running redis on port 6379 but i cant connect to it bc im in a container + - ok so i had to do this + ```bash + docker network create my_network + # re run a new redis instance with the --network + docker run --name my_redis_container --network my_network -p 6379:6379 -d redis + # i had to also add my_network to the build args of this in devcontainer.json + ``` +- super strange git issue ``` +git add src/ +fatal: detected dubious ownership in repository at '/workspaces/attractorsIII' +To add an exception for this directory, call: -Redis is cool but its overkill I can just use `from cachetools import TTLCache` -No thats not async enough im going to use `from aiocache import caches` + git config --global --add safe.directory /workspaces/attractorsIII +``` -Im now seeing in jit that it can do caching. But that wouldn't be helpful as my function does one iteration at a time and i just cache the resulting 10000000 iterations with the initial conditions. I also compress it with gzip so which is nice too. # Deploying - I created a new GCP project geo-attractors. @@ -44,7 +50,7 @@ Im now seeing in jit that it can do caching. But that wouldn't be helpful as my - starting with cloud run made some terraform and some startup scripts had to send it to gcloud to build 1. on the m1 mac the build is strange and 2. can't build in the docker container -`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-fastapi` +`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-backend"` this will be solved when its just automatic with the ci/cd pipeline thats totally coming soon @@ -55,32 +61,25 @@ this will be solved when its just automatic with the ci/cd pipeline thats totall - added backend to main.tf then did a `terraform init` -# Issues -- Learn how to write tests -- Terminal auto complete -- Can't build or run Dockerfile in the container. -possibly I could pass the local machines docker into the container -```bash -ls -la /var/run/docker.sock -docker run -v /var/run/docker.sock:/var/run/docker.sock -it your-dev-container-image +# two services one repo +each service will have its own venv and requirements.txt. But the requirements.txt in the root of this project will have all the dev dependencies need to run pre-commit hooks and any other project dependencies. + +```mermaid +graph LR + A[Dash Frontend] <--> B[Backend FastAPI] ``` -- I'm running redis on port 6379 but i cant connect to it bc im in a container - - ok so i had to do this - ```bash - docker network create my_network - # re run a new redis instance with the --network - docker run --name my_redis_container --network my_network -p 6379:6379 -d redis - # i had to also add my_network to the build args of this in devcontainer.json - ``` -- Brew 🍺 would be nice -- super strange git issue -``` -git add src/ -fatal: detected dubious ownership in repository at '/workspaces/attractorsIII' -To add an exception for this directory, call: +# GH workload federation stuff - git config --global --add safe.directory /workspaces/attractorsIII +[Here is the module I used](https://github.com/terraform-google-modules/terraform-google-github-actions-runners/blob/master/modules/gh-oidc/README.md +) + +- need to enable IAM Service Account Credentials API + +- two variables need to be set you get them after applying the terraform and then running `terraform output` and then use the value from the outputs +`gh_federation_provider_name and gh_federation_sa_email` +```bash +gh secret set WIF_PROVIDER -b"gh_federation_provider_name" +gh secret set WIF_SERVICE_ACCOUNT -b"gh_federation_sa_email" ``` -- doesn't open properly in code-space got an error on creations will check later at the airport on my ipad. not sure if it was the my network or i had an extra comma that broke it probably the latter as the error log didn't say much about the build arg its very slow probably need to give it more compute. could it be that its using my wifi to download packages I don't think so. diff --git a/backend/.dockerignore b/backend/.dockerignore index 0b1e1e7..497a1db 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -1,5 +1,5 @@ **/__pycache__ -**/.venv +**/.venv* **/.classpath **/.dockerignore **/.env @@ -17,7 +17,6 @@ **/charts **/docker-compose* **/compose* -**/Dockerfile* **/node_modules **/npm-debug.log **/obj diff --git a/backend/Dockerfile b/backend/Dockerfile index cd07ff4..bcc0f70 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -### PROD ### -FROM python:3.11-slim-buster as prod +### BUILD ### +FROM python:3.11-slim-buster as BUILD RUN apt-get update # Set the working directory to /app WORKDIR /app diff --git a/backend/README.md b/backend/README.md index e69de29..c1bde1a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -0,0 +1,3 @@ +This backend uses FastAPI. + +`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-backend" --ignore-file .dockerignore` diff --git a/backend/requirements.txt b/backend/requirements.txt index 687d13a..377bd89 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,16 +1,49 @@ +aiocache==0.12.2 annotated-types==0.6.0 anyio==4.2.0 +certifi==2023.11.17 +charset-normalizer==3.3.2 click==8.1.7 +cloudpickle==3.0.0 +colorcet==3.0.1 +dask==2023.12.1 +datashader==0.16.0 fastapi==0.108.0 +fsspec==2023.12.2 h11==0.14.0 idna==3.6 +importlib-metadata==7.0.1 +llvmlite==0.41.1 +locket==1.0.0 +loguru==0.7.2 +multipledispatch==1.0.0 +numba==0.58.1 +numpy==1.26.3 +orjson==3.9.10 +packaging==23.2 +pandas==2.1.4 +param==2.0.1 +partd==1.4.1 +pillow==10.2.0 prometheus-client==0.19.0 prometheus-fastapi-instrumentator==6.1.0 +pyct==0.5.0 pydantic==2.5.3 pydantic-settings==2.1.0 pydantic_core==2.14.6 +python-dateutil==2.8.2 python-dotenv==1.0.0 +pytz==2023.3.post1 +PyYAML==6.0.1 +requests==2.31.0 +scipy==1.11.4 +six==1.16.0 sniffio==1.3.0 starlette==0.32.0.post1 +toolz==0.12.0 typing_extensions==4.9.0 +tzdata==2023.4 +urllib3==2.1.0 uvicorn==0.25.0 +xarray==2023.12.0 +zipp==3.17.0 diff --git a/backend/src/api/attractors/attractor_service.py b/backend/src/api/attractors/attractor_service.py index 0a6e82a..8520d17 100644 --- a/backend/src/api/attractors/attractor_service.py +++ b/backend/src/api/attractors/attractor_service.py @@ -36,8 +36,9 @@ def trajectory(self, fn, x0, y0, a, b=0, c=0, d=0, e=0, f=0, n=10000000): def gen_random(self, func, desired_empty=10000): # finds some nice initial conditions non_empty = 0 + tries = 0 # how many non empty pixels - while non_empty < desired_empty: + while non_empty < desired_empty or tries <= 10: initial_conditions = np.c_[ np.zeros((1, 2)), np.random.random((1, 6)) * 4 - 2 ][0] @@ -48,6 +49,7 @@ def gen_random(self, func, desired_empty=10000): agg = cvs.points(df, "x", "y") non_empty = np.count_nonzero(np.array(agg)) logger.info(f"non_empty: {non_empty}") + tries += 1 return initial_conditions def make_dataframe( diff --git a/frontend/.dockerignore b/frontend/.dockerignore index 0b1e1e7..497a1db 100644 --- a/frontend/.dockerignore +++ b/frontend/.dockerignore @@ -1,5 +1,5 @@ **/__pycache__ -**/.venv +**/.venv* **/.classpath **/.dockerignore **/.env @@ -17,7 +17,6 @@ **/charts **/docker-compose* **/compose* -**/Dockerfile* **/node_modules **/npm-debug.log **/obj diff --git a/frontend/Dockerfile b/frontend/Dockerfile index cd07ff4..bcc0f70 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,5 +1,5 @@ -### PROD ### -FROM python:3.11-slim-buster as prod +### BUILD ### +FROM python:3.11-slim-buster as BUILD RUN apt-get update # Set the working directory to /app WORKDIR /app diff --git a/frontend/README.md b/frontend/README.md index 05020db..ed468cf 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -11,4 +11,4 @@ TODO So dash uses flask for its server backend so it uses gunicorn which is a WSGI (Web server gateway interface) and wont work with uvicorn a ASGI (async server gateway interface) -gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-frontend" +gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-frontend" --ignore-file .dockerignore diff --git a/frontend/src/application.py b/frontend/src/application.py index 1a99932..2e16c5d 100644 --- a/frontend/src/application.py +++ b/frontend/src/application.py @@ -1,7 +1,6 @@ import dash_bootstrap_components as dbc -from dash import Dash, html - import src.frontend as frontend +from dash import Dash, html from src.backend_check import backend_check, backend_check_layout from src.components import layout diff --git a/frontend/src/backend_check.py b/frontend/src/backend_check.py index 7553295..8a583fd 100644 --- a/frontend/src/backend_check.py +++ b/frontend/src/backend_check.py @@ -2,7 +2,6 @@ from dash import dcc, html from dash.dependencies import Input, Output from loguru import logger - from src.settings import settings backend_check_layout = html.Div( diff --git a/frontend/src/components.py b/frontend/src/components.py index 886d262..b2d9d44 100644 --- a/frontend/src/components.py +++ b/frontend/src/components.py @@ -17,8 +17,14 @@ id="function-dropdown", options=[ {"label": "Clifford", "value": "Clifford"}, - {"label": "De Jong", "value": "de_jong"}, - {"label": "Bedhead", "value": "bedhead"}, + {"label": "De Jong", "value": "De Jong"}, + {"label": "Svensson", "value": "Svensson"}, + {"label": "Fractal Dream", "value": "Fractal Dream"}, + {"label": "Bedhead", "value": "Bedhead"}, + {"label": "HopaLong", "value": "Hopalong"}, + {"label": "HopaLong2", "value": "Hopalong2"}, + {"label": "Gumowski Mira", "value": "Gumowski Mira"}, + {"label": "Symmetric Icon", "values": "Symmetric Icon"}, ], value="Clifford", ) diff --git a/frontend/src/frontend.py b/frontend/src/frontend.py index af88c40..699d442 100644 --- a/frontend/src/frontend.py +++ b/frontend/src/frontend.py @@ -5,7 +5,6 @@ from dash.dependencies import Input, Output, State from dash.exceptions import PreventUpdate from loguru import logger - from src.client_mem_model import ClientMemModel from src.settings import settings diff --git a/frontend/src/logging.py b/frontend/src/logging.py index bf02c09..b16a221 100644 --- a/frontend/src/logging.py +++ b/frontend/src/logging.py @@ -3,7 +3,6 @@ from typing import Union from loguru import logger - from src.settings import settings diff --git a/frontend/src/settings.py b/frontend/src/settings.py index bff85a3..2c931cf 100644 --- a/frontend/src/settings.py +++ b/frontend/src/settings.py @@ -2,6 +2,7 @@ from pathlib import Path from tempfile import gettempdir +from pydantic import validator from pydantic_settings import BaseSettings from yarl import URL @@ -47,7 +48,12 @@ class Settings(BaseSettings): log_level: LogLevel = LogLevel.INFO - backend_url: URL = URL("https://attractors-service-c6dyl3tniq-uc.a.run.app") + backend_url: URL = URL("https://attractors-backend-service-c6dyl3tniq-uc.a.run.app") + + @validator("backend_url", pre=True) + def parse_backend_url(cls, value: str) -> URL: + """Parse backend url.""" + return URL(value) class Config: env_file = ".env" diff --git a/geo-notes.md b/geo-notes.md new file mode 100644 index 0000000..18b7b13 --- /dev/null +++ b/geo-notes.md @@ -0,0 +1,22 @@ +# Redis +Idea to add some sort of caching for the attractors so if i want to recolor it i can pull the initial conditions from a cache + +```bash +docker pull redis +# -p and --name must come before -d +docker run -p 6379:6379 --name my-redis -d redis +docker exec -it my-redis redis-cli + +set my-key "hello redis" +get my-key +``` + +```bash +pip install redis +pip install aioredis +``` + +Redis is cool but its overkill I can just use `from cachetools import TTLCache` +No thats not async enough im going to use `from aiocache import caches` + +Im now seeing in jit that it can do caching. But that wouldn't be helpful as my function does one iteration at a time and i just cache the resulting 10000000 iterations with the initial conditions. I also compress it with gzip so which is nice too. diff --git a/scripts/alias.sh b/scripts/alias.sh new file mode 100644 index 0000000..e69de29 diff --git a/scripts/devstart.sh b/scripts/devstart.sh index 52ae3a4..61da0dc 100755 --- a/scripts/devstart.sh +++ b/scripts/devstart.sh @@ -26,3 +26,10 @@ sudo apt update && sudo apt install -y terraform # && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ # && sudo apt update \ # && sudo apt install gh -y + +# Brew ---Before uncommenting this will only go to bashrc --- +# /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +# (echo; echo 'eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"') >> /home/vscode/.bashrc +# eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + +# brew install tflint diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 38c94fa..b18059a 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -20,3 +20,23 @@ provider "registry.terraform.io/hashicorp/google" { "zh:fb82f6b5d1f992243ab8fe417659cdf9831202cf1e16fe7593d3967888b035cc", ] } + +provider "registry.terraform.io/hashicorp/google-beta" { + version = "5.11.0" + constraints = ">= 3.64.0, < 6.0.0" + hashes = [ + "h1:yoAwpGJ3gxIoctH5U4yEcBfE2IJowny7ul1EWkRusww=", + "zh:0efa82e6fe2c83bd5280c3009db1c3acc9cdad3c9419b6ec721fbefc9f832449", + "zh:371df01e4f38b828195d115c9a8bebddebec4d34e9ef74cf3a79161da08e44b2", + "zh:5089967c420c5e4a4ba0d4c8c6ca344c7bb2476ec928f8319856260eacded369", + "zh:798a65c79386d356d6a097de680f4ece8982daae1cb0e10d6c53b383efef45f0", + "zh:90178911ac0e624c69a54a992fb3425ef09fdfb3e34b496ad7b6e168e80d4e0c", + "zh:b59c60f8479b8f0c8e91a93a4e707ce6d17c8e50e2f5afaf1d9a03c03cfedbf8", + "zh:c7f946282d80223ab3a6b284c22e4b53ffcd7b1a02449bb95a350007f30c87dc", + "zh:cd60e76987c2fdce2c84219eaff9390cd135f88aa9a27bc4d79a8fd4a8d09622", + "zh:de06bfa0393206c0253ebdea70821cb3b08ef87d5d4844be3ae463abfb4e1884", + "zh:de494bad600cca78986ce63d1018f5dbc1a1fcc2d4c41c94c15d5346f2b0dd1e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f97a8b6e83e0083dcb42a87e8e418ab33f12d641f9cdfdc92d154ba7fd7398fb", + ] +} diff --git a/terraform/cloudrun/main.tf b/terraform/cloudrun/main.tf new file mode 100644 index 0000000..87a6bed --- /dev/null +++ b/terraform/cloudrun/main.tf @@ -0,0 +1,46 @@ +data "google_project" "current" {} + +resource "google_cloud_run_service" "cloudrun_service" { + name = var.service_name + location = var.ar_repo_location + + template { + spec { + containers { + image = "${var.ar_repo_location}-docker.pkg.dev/${data.google_project.current.name}/${var.ar_repo_name}/${var.container_image}:latest" + resources { + limits = { + cpu = "1000m" + memory = "1Gi" + } + } + env { + name = "APP_BACKEND_URL" + value = var.backend_url_env + } + ports { + container_port = 8080 + } + } + timeout_seconds = 300 + } + } + + traffic { + percent = 100 + latest_revision = true + } +} + +resource "google_cloud_run_service_iam_member" "public" { + count = var.is_public ? 1 : 0 + service = google_cloud_run_service.cloudrun_service.name + location = google_cloud_run_service.cloudrun_service.location + role = "roles/run.invoker" + member = "allUsers" +} + +output "cloudrun_service_url" { + description = "The URL of the cloud run service" + value = google_cloud_run_service.cloudrun_service.status[0].url +} diff --git a/terraform/cloudrun/variables.tf b/terraform/cloudrun/variables.tf new file mode 100644 index 0000000..16bd7ed --- /dev/null +++ b/terraform/cloudrun/variables.tf @@ -0,0 +1,36 @@ +variable "location" { + type = string + description = "GCP region" + default = "us-central1" +} + +variable "ar_repo_location" { + description = "The location of the Artifact Registry repository and the cloudrun service" + type = string +} + +variable "ar_repo_name" { + description = "The name of the Artifact Registry repository" + type = string +} + +variable "container_image" { + type = string + description = "image in GAR" +} + +variable "service_name" { + type = string + description = "cloudrun service name" +} + +variable "is_public" { + type = bool + description = "Should the service be public" +} + +variable "backend_url_env" { + type = string + description = "The URL of the backend service" + default = "None" +} diff --git a/terraform/frontend/main.tf b/terraform/frontend/main.tf deleted file mode 100644 index 12b2c11..0000000 --- a/terraform/frontend/main.tf +++ /dev/null @@ -1,42 +0,0 @@ -variable "ar_repo_name" { - description = "The name of the Artifact Registry repository" - type = string -} -resource "google_cloud_run_service" "cloudrun_frontend_service" { - name = "attractors-frontend-service" - location = "us-central1" - - template { - spec { - containers { - image = "us-central1-docker.pkg.dev/geo-attractors/${var.ar_repo_name}/attractors-frontend:latest" - resources { - limits = { - cpu = "1000m" - memory = "1Gi" - } - } - env { - name = "MY_ENV_VAR" - value = "my-value" - } - ports { - container_port = 8080 - } - } - timeout_seconds = 300 - } - } - - traffic { - percent = 100 - latest_revision = true - } -} - -resource "google_cloud_run_service_iam_member" "public" { - service = google_cloud_run_service.cloudrun_frontend_service.name - location = google_cloud_run_service.cloudrun_frontend_service.location - role = "roles/run.invoker" - member = "allUsers" -} diff --git a/terraform/gh-id-federation/main.tf b/terraform/gh-id-federation/main.tf new file mode 100644 index 0000000..b27f5f5 --- /dev/null +++ b/terraform/gh-id-federation/main.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +resource "google_service_account" "sa" { + project = var.project_id + account_id = "gh-federation-sa" +} + +resource "google_project_iam_member" "artifact_registry_writer" { + project = var.project_id + role = "roles/artifactregistry.writer" + member = "serviceAccount:${google_service_account.sa.email}" +} + +resource "google_project_iam_member" "cloud_run_admin" { + project = var.project_id + role = "roles/run.admin" + member = "serviceAccount:${google_service_account.sa.email}" +} + +resource "google_project_iam_member" "service_account_user" { + project = var.project_id + role = "roles/iam.serviceAccountUser" + member = "serviceAccount:${google_service_account.sa.email}" +} + +resource "google_project_iam_member" "project" { + project = var.project_id + role = "roles/storage.admin" + member = "serviceAccount:${google_service_account.sa.email}" +} + +module "oidc" { + source = "terraform-google-modules/github-actions-runners/google//modules/gh-oidc" + version = "~> 3.0" + + project_id = var.project_id + pool_id = "my-pool" + provider_id = "gh-provider" + sa_mapping = { + (google_service_account.sa.account_id) = { + sa_name = google_service_account.sa.name + attribute = "attribute.repository/${var.github_org}/${var.github_repo}" + } + } +} diff --git a/terraform/gh-id-federation/outputs.tf b/terraform/gh-id-federation/outputs.tf new file mode 100644 index 0000000..ac46dd9 --- /dev/null +++ b/terraform/gh-id-federation/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "pool_name" { + description = "Pool name" + value = module.oidc.pool_name +} + +output "provider_name" { + description = "Provider name" + value = module.oidc.provider_name +} + +output "sa_email" { + description = "Example SA email" + value = google_service_account.sa.email +} diff --git a/terraform/gh-id-federation/variables.tf b/terraform/gh-id-federation/variables.tf new file mode 100644 index 0000000..acd7b7b --- /dev/null +++ b/terraform/gh-id-federation/variables.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The project id to create WIF pool and example SA" +} + +variable "github_repo" { + type = string + description = "The name of the repo" +} + +variable "github_org" { + type = string + description = "The org or user the github repo is under" +} diff --git a/terraform/main.tf b/terraform/main.tf index bf4d387..81681df 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,8 +1,25 @@ +terraform { + required_version = ">= 1.6.6" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.10" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 5.11" + } + } +} + provider "google" { project = "geo-attractors" region = "us-central1" } +data "google_project" "current" { + project_id = "geo-attractors" +} terraform { backend "gcs" { @@ -11,10 +28,6 @@ terraform { } } -data "google_project" "current" { - project_id = "geo-attractors" -} - resource "google_artifact_registry_repository" "my_ar_repo" { location = "us-central1" repository_id = "attractors" @@ -22,51 +35,45 @@ resource "google_artifact_registry_repository" "my_ar_repo" { } resource "google_project_iam_member" "artifact_registry_reader" { - project = "geo-attractors" + project = data.google_project.current.name role = "roles/artifactregistry.reader" member = "serviceAccount:service-${data.google_project.current.number}@serverless-robot-prod.iam.gserviceaccount.com" } -resource "google_cloud_run_service" "my_cloudrun_service" { - name = "attractors-service" - location = "us-central1" - template { - spec { - containers { - image = "us-central1-docker.pkg.dev/geo-attractors/${google_artifact_registry_repository.my_ar_repo.name}/attractors-fastapi:latest" - resources { - limits = { - cpu = "1000m" - memory = "2Gi" - } - } - env { - name = "MY_ENV_VAR" - value = "my-value" - } - ports { - container_port = 8080 - } - } - timeout_seconds = 300 - } - } +module "backend" { + source = "./cloudrun" + service_name = "attractors-backend-service" + container_image = "attractors-backend" + ar_repo_name = google_artifact_registry_repository.my_ar_repo.name + ar_repo_location = google_artifact_registry_repository.my_ar_repo.location + is_public = true +} - traffic { - percent = 100 - latest_revision = true - } +module "frontend" { + source = "./cloudrun" + service_name = "attractors-frontend-service" + container_image = "attractors-frontend" + backend_url_env = module.backend.cloudrun_service_url + ar_repo_name = google_artifact_registry_repository.my_ar_repo.name + ar_repo_location = google_artifact_registry_repository.my_ar_repo.location + is_public = true } -resource "google_cloud_run_service_iam_member" "public" { - service = google_cloud_run_service.my_cloudrun_service.name - location = google_cloud_run_service.my_cloudrun_service.location - role = "roles/run.invoker" - member = "allUsers" +module "gh-federation" { + source = "./gh-id-federation" + project_id = data.google_project.current.name + github_org = "CupOfGeo" + github_repo = "Attractors" } -module "frontend" { - source = "./frontend" - ar_repo_name = google_artifact_registry_repository.my_ar_repo.name + +output "gh_federation_provider_name" { + description = "Provider name from gh-federation module" + value = module.gh-federation.provider_name +} + +output "gh_federation_sa_email" { + description = "Service account email from gh-federation module" + value = module.gh-federation.sa_email }