From 047b7da13a886c78fce07323cb76e2c4761cf2f3 Mon Sep 17 00:00:00 2001 From: Geo Date: Mon, 8 Jan 2024 03:11:15 +0000 Subject: [PATCH 01/17] templated public cloud run --- .devcontainer/devcontainer.json | 6 +- backend/README.md | 431 ++++++++++++++++++++++++++++++++ terraform/README.md | 34 +++ terraform/cloudrun/README.md | 40 +++ terraform/cloudrun/main.tf | 46 ++++ terraform/cloudrun/variables.tf | 30 +++ terraform/frontend/main.tf | 42 ---- terraform/main.tf | 60 ++--- 8 files changed, 603 insertions(+), 86 deletions(-) create mode 100644 terraform/README.md create mode 100644 terraform/cloudrun/README.md create mode 100644 terraform/cloudrun/main.tf create mode 100644 terraform/cloudrun/variables.tf delete mode 100644 terraform/frontend/main.tf diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d48136f..e9f3df9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -26,7 +26,11 @@ "vscode": { "extensions": [ "streetsidesoftware.code-spell-checker" - ] + ], + "settings": { + "terminal.integrated.shell.linux": "/bin/zsh" + } + } } diff --git a/backend/README.md b/backend/README.md index e69de29..97dd257 100644 --- a/backend/README.md +++ b/backend/README.md @@ -0,0 +1,431 @@ +# terraform-docs + +[![Build Status](https://github.com/terraform-docs/terraform-docs/workflows/ci/badge.svg)](https://github.com/terraform-docs/terraform-docs/actions) [![GoDoc](https://pkg.go.dev/badge/github.com/terraform-docs/terraform-docs)](https://pkg.go.dev/github.com/terraform-docs/terraform-docs) [![Go Report Card](https://goreportcard.com/badge/github.com/terraform-docs/terraform-docs)](https://goreportcard.com/report/github.com/terraform-docs/terraform-docs) [![Codecov Report](https://codecov.io/gh/terraform-docs/terraform-docs/branch/master/graph/badge.svg)](https://codecov.io/gh/terraform-docs/terraform-docs) [![License](https://img.shields.io/github/license/terraform-docs/terraform-docs)](https://github.com/terraform-docs/terraform-docs/blob/master/LICENSE) [![Latest release](https://img.shields.io/github/v/release/terraform-docs/terraform-docs)](https://github.com/terraform-docs/terraform-docs/releases) + +![terraform-docs-teaser](./images/terraform-docs-teaser.png) + +## What is terraform-docs + +A utility to generate documentation from Terraform modules in various output formats. + +## Installation + +macOS users can install using [Homebrew]: + +```bash +brew install terraform-docs +``` + +or + +```bash +brew install terraform-docs/tap/terraform-docs +``` + +Windows users can install using [Scoop]: + +```bash +scoop bucket add terraform-docs https://github.com/terraform-docs/scoop-bucket +scoop install terraform-docs +``` + +or [Chocolatey]: + +```bash +choco install terraform-docs +``` + +Stable binaries are also available on the [releases] page. To install, download the +binary for your platform from "Assets" and place this into your `$PATH`: + +```bash +curl -Lo ./terraform-docs.tar.gz https://github.com/terraform-docs/terraform-docs/releases/download/v0.17.0/terraform-docs-v0.17.0-$(uname)-amd64.tar.gz +tar -xzf terraform-docs.tar.gz +chmod +x terraform-docs +mv terraform-docs /usr/local/bin/terraform-docs +``` + +**NOTE:** Windows releases are in `ZIP` format. + +The latest version can be installed using `go install` or `go get`: + +```bash +# go1.17+ +go install github.com/terraform-docs/terraform-docs@v0.17.0 +``` + +```bash +# go1.16 +GO111MODULE="on" go get github.com/terraform-docs/terraform-docs@v0.17.0 +``` + +**NOTE:** please use the latest Go to do this, minimum `go1.16` is required. + +This will put `terraform-docs` in `$(go env GOPATH)/bin`. If you encounter the error +`terraform-docs: command not found` after installation then you may need to either add +that directory to your `$PATH` as shown [here] or do a manual installation by cloning +the repo and run `make build` from the repository which will put `terraform-docs` in: + +```bash +$(go env GOPATH)/src/github.com/terraform-docs/terraform-docs/bin/$(uname | tr '[:upper:]' '[:lower:]')-amd64/terraform-docs +``` + +## Usage + +### Running the binary directly + +To run and generate documentation into README within a directory: + +```bash +terraform-docs markdown table --output-file README.md --output-mode inject /path/to/module +``` + +Check [`output`] configuration for more details and examples. + +### Using docker + +terraform-docs can be run as a container by mounting a directory with `.tf` +files in it and run the following command: + +```bash +docker run --rm --volume "$(pwd):/terraform-docs" -u $(id -u) quay.io/terraform-docs/terraform-docs:0.17.0 markdown /terraform-docs +``` + +If `output.file` is not enabled for this module, generated output can be redirected +back to a file: + +```bash +docker run --rm --volume "$(pwd):/terraform-docs" -u $(id -u) quay.io/terraform-docs/terraform-docs:0.17.0 markdown /terraform-docs > doc.md +``` + +**NOTE:** Docker tag `latest` refers to _latest_ stable released version and `edge` +refers to HEAD of `master` at any given point in time. + +### Using GitHub Actions + +To use terraform-docs GitHub Action, configure a YAML workflow file (e.g. +`.github/workflows/documentation.yml`) with the following: + +```yaml +name: Generate terraform docs +on: + - pull_request + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + + - name: Render terraform docs and push changes back to PR + uses: terraform-docs/gh-actions@main + with: + working-dir: . + output-file: README.md + output-method: inject + git-push: "true" +``` + +Read more about [terraform-docs GitHub Action] and its configuration and +examples. + +### pre-commit hook + +With pre-commit, you can ensure your Terraform module documentation is kept +up-to-date each time you make a commit. + +First [install pre-commit] and then create or update a `.pre-commit-config.yaml` +in the root of your Git repo with at least the following content: + +```yaml +repos: + - repo: https://github.com/terraform-docs/terraform-docs + rev: "v0.17.0" + hooks: + - id: terraform-docs-go + args: ["markdown", "table", "--output-file", "README.md", "./mymodule/path"] +``` + +Then run: + +```bash +pre-commit install +pre-commit install-hooks +``` + +Further changes to your module's `.tf` files will cause an update to documentation +when you make a commit. + +## Configuration + +terraform-docs can be configured with a yaml file. The default name of this file is +`.terraform-docs.yml` and the path order for locating it is: + +1. root of module directory +1. `.config/` folder at root of module directory +1. current directory +1. `.config/` folder at current directory +1. `$HOME/.tfdocs.d/` + +```yaml +formatter: "" # this is required + +version: "" + +header-from: main.tf +footer-from: "" + +recursive: + enabled: false + path: modules + +sections: + hide: [] + show: [] + +content: "" + +output: + file: "" + mode: inject + template: |- + + {{ .Content }} + + +output-values: + enabled: false + from: "" + +sort: + enabled: true + by: name + +settings: + anchor: true + color: true + default: true + description: false + escape: true + hide-empty: false + html: true + indent: 2 + lockfile: true + read-comments: true + required: true + sensitive: true + type: true +``` + +## Content Template + +Generated content can be customized further away with `content` in configuration. +If the `content` is empty the default order of sections is used. + +Compatible formatters for customized content are `asciidoc` and `markdown`. `content` +will be ignored for other formatters. + +`content` is a Go template with following additional variables: + +- `{{ .Header }}` +- `{{ .Footer }}` +- `{{ .Inputs }}` +- `{{ .Modules }}` +- `{{ .Outputs }}` +- `{{ .Providers }}` +- `{{ .Requirements }}` +- `{{ .Resources }}` + +and following functions: + +- `{{ include "relative/path/to/file" }}` + +These variables are the generated output of individual sections in the selected +formatter. For example `{{ .Inputs }}` is Markdown Table representation of _inputs_ +when formatter is set to `markdown table`. + +Note that sections visibility (i.e. `sections.show` and `sections.hide`) takes +precedence over the `content`. + +Additionally there's also one extra special variable avaialble to the `content`: + +- `{{ .Module }}` + +As opposed to the other variables mentioned above, which are generated sections +based on a selected formatter, the `{{ .Module }}` variable is just a `struct` +representing a [Terraform module]. + +````yaml +content: |- + Any arbitrary text can be placed anywhere in the content + + {{ .Header }} + + and even in between sections + + {{ .Providers }} + + and they don't even need to be in the default order + + {{ .Outputs }} + + include any relative files + + {{ include "relative/path/to/file" }} + + {{ .Inputs }} + + # Examples + + ```hcl + {{ include "examples/foo/main.tf" }} + ``` + + ## Resources + + {{ range .Module.Resources }} + - {{ .GetMode }}.{{ .Spec }} ({{ .Position.Filename }}#{{ .Position.Line }}) + {{- end }} +```` + +## Build on top of terraform-docs + +terraform-docs primary use-case is to be utilized as a standalone binary, but +some parts of it is also available publicly and can be imported in your project +as a library. + +```go +import ( + "github.com/terraform-docs/terraform-docs/format" + "github.com/terraform-docs/terraform-docs/print" + "github.com/terraform-docs/terraform-docs/terraform" +) + +// buildTerraformDocs for module root `path` and provided content `tmpl`. +func buildTerraformDocs(path string, tmpl string) (string, error) { + config := print.DefaultConfig() + config.ModuleRoot = path // module root path (can be relative or absolute) + + module, err := terraform.LoadWithOptions(config) + if err != nil { + return "", err + } + + // Generate in Markdown Table format + formatter := format.NewMarkdownTable(config) + + if err := formatter.Generate(module); err != nil { + return "", err + } + + // // Note: if you don't intend to provide additional template for the generated + // // content, or the target format doesn't provide templating (e.g. json, yaml, + // // xml, or toml) you can use `Content()` function instead of `Render()`. + // // `Content()` returns all the sections combined with predefined order. + // return formatter.Content(), nil + + return formatter.Render(tmpl) +} +``` + +## Plugin + +Generated output can be heavily customized with [`content`], but if using that +is not enough for your use-case, you can write your own plugin. + +In order to install a plugin the following steps are needed: + +- download the plugin and place it in `~/.tfdocs.d/plugins` (or `./.tfdocs.d/plugins`) +- make sure the plugin file name is `tfdocs-format-` +- modify [`formatter`] of `.terraform-docs.yml` file to be `` + +**Important notes:** + +- if the plugin file name is different than the example above, terraform-docs won't +be able to to pick it up nor register it properly +- you can only use plugin thorough `.terraform-docs.yml` file and it cannot be used +with CLI arguments + +To create a new plugin create a new repository called `tfdocs-format-` with +following `main.go`: + +```go +package main + +import ( + _ "embed" //nolint + + "github.com/terraform-docs/terraform-docs/plugin" + "github.com/terraform-docs/terraform-docs/print" + "github.com/terraform-docs/terraform-docs/template" + "github.com/terraform-docs/terraform-docs/terraform" +) + +func main() { + plugin.Serve(&plugin.ServeOpts{ + Name: "", + Version: "0.1.0", + Printer: printerFunc, + }) +} + +//go:embed sections.tmpl +var tplCustom []byte + +// printerFunc the function being executed by the plugin client. +func printerFunc(config *print.Config, module *terraform.Module) (string, error) { + tpl := template.New(config, + &template.Item{Name: "custom", Text: string(tplCustom)}, + ) + + rendered, err := tpl.Render("custom", module) + if err != nil { + return "", err + } + + return rendered, nil +} +``` + +Please refer to [tfdocs-format-template] for more details. You can create a new +repository from it by clicking on `Use this template` button. + +## Documentation + +- **Users** + - Read the [User Guide] to learn how to use terraform-docs + - Read the [Formats Guide] to learn about different output formats of terraform-docs + - Refer to [Config File Reference] for all the available configuration options +- **Developers** + - Read [Contributing Guide] before submitting a pull request + +Visit [our website] for all documentation. + +## Community + +- Discuss terraform-docs on [Slack] + +## License + +MIT License - Copyright (c) 2021 The terraform-docs Authors. + +[Chocolatey]: https://www.chocolatey.org +[Config File Reference]: https://terraform-docs.io/user-guide/configuration/ +[`content`]: https://terraform-docs.io/user-guide/configuration/content/ +[Contributing Guide]: CONTRIBUTING.md +[Formats Guide]: https://terraform-docs.io/reference/terraform-docs/ +[`formatter`]: https://terraform-docs.io/user-guide/configuration/formatter/ +[here]: https://golang.org/doc/code.html#GOPATH +[Homebrew]: https://brew.sh +[install pre-commit]: https://pre-commit.com/#install +[`output`]: https://terraform-docs.io/user-guide/configuration/output/ +[releases]: https://github.com/terraform-docs/terraform-docs/releases +[Scoop]: https://scoop.sh/ +[Slack]: https://slack.terraform-docs.io/ +[terraform-docs GitHub Action]: https://github.com/terraform-docs/gh-actions +[Terraform module]: https://pkg.go.dev/github.com/terraform-docs/terraform-docs/terraform#Module +[tfdocs-format-template]: https://github.com/terraform-docs/tfdocs-format-template +[our website]: https://terraform-docs.io/ +[User Guide]: https://terraform-docs.io/user-guide/introduction/ diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..7f95f89 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,34 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | 5.10.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [backend](#module\_backend) | ./cloudrun | n/a | +| [frontend](#module\_frontend) | ./cloudrun | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_artifact_registry_repository.my_ar_repo](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository) | resource | +| [google_project_iam_member.artifact_registry_reader](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | +| [google_project.current](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +No inputs. + +## Outputs + +No outputs. + diff --git a/terraform/cloudrun/README.md b/terraform/cloudrun/README.md new file mode 100644 index 0000000..5f4b7d8 --- /dev/null +++ b/terraform/cloudrun/README.md @@ -0,0 +1,40 @@ + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_cloud_run_service.cloudrun_service](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | +| [google_cloud_run_service_iam_member.public](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service_iam_member) | resource | +| [google_project.current](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ar\_repo\_location](#input\_ar\_repo\_location) | The location of the Artifact Registry repository and the cloudrun service | `string` | n/a | yes | +| [ar\_repo\_name](#input\_ar\_repo\_name) | The name of the Artifact Registry repository | `string` | n/a | yes | +| [container\_image](#input\_container\_image) | image in GAR | `string` | n/a | yes | +| [is\_public](#input\_is\_public) | Should the service be public | `bool` | n/a | yes | +| [location](#input\_location) | GCP region | `string` | `"us-central1"` | no | +| [service\_name](#input\_service\_name) | cloudrun service name | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [cloudrun\_service\_url](#output\_cloudrun\_service\_url) | The URL of the cloud run service | + diff --git a/terraform/cloudrun/main.tf b/terraform/cloudrun/main.tf new file mode 100644 index 0000000..97b8ebd --- /dev/null +++ b/terraform/cloudrun/main.tf @@ -0,0 +1,46 @@ +data "google_project" "current" {} + +resource "google_cloud_run_service" "cloudrun_service" { + name = var.service_name + location = var.ar_repo_location + + template { + spec { + containers { + image = "${var.ar_repo_location}-docker.pkg.dev/${data.google_project.current.name}/${var.ar_repo_name}/${var.container_image}:latest" + resources { + limits = { + cpu = "1000m" + memory = "1Gi" + } + } + env { + name = "MY_ENV_VAR" + value = "my-value" + } + ports { + container_port = 8080 + } + } + timeout_seconds = 300 + } + } + + traffic { + percent = 100 + latest_revision = true + } +} + +resource "google_cloud_run_service_iam_member" "public" { + count = var.is_public ? 1 : 0 + service = google_cloud_run_service.cloudrun_service.name + location = google_cloud_run_service.cloudrun_service.location + role = "roles/run.invoker" + member = "allUsers" +} + +output "cloudrun_service_url" { + description = "The URL of the cloud run service" + value = google_cloud_run_service.cloudrun_service.status[0].url +} diff --git a/terraform/cloudrun/variables.tf b/terraform/cloudrun/variables.tf new file mode 100644 index 0000000..2e38bc0 --- /dev/null +++ b/terraform/cloudrun/variables.tf @@ -0,0 +1,30 @@ +variable "location" { + type = string + description = "GCP region" + default = "us-central1" +} + +variable "ar_repo_location" { + description = "The location of the Artifact Registry repository and the cloudrun service" + type = string +} + +variable "ar_repo_name" { + description = "The name of the Artifact Registry repository" + type = string +} + +variable "container_image" { + type = string + description = "image in GAR" +} + +variable "service_name" { + type = string + description = "cloudrun service name" +} + +variable "is_public" { + type = bool + description = "Should the service be public" +} diff --git a/terraform/frontend/main.tf b/terraform/frontend/main.tf deleted file mode 100644 index 12b2c11..0000000 --- a/terraform/frontend/main.tf +++ /dev/null @@ -1,42 +0,0 @@ -variable "ar_repo_name" { - description = "The name of the Artifact Registry repository" - type = string -} -resource "google_cloud_run_service" "cloudrun_frontend_service" { - name = "attractors-frontend-service" - location = "us-central1" - - template { - spec { - containers { - image = "us-central1-docker.pkg.dev/geo-attractors/${var.ar_repo_name}/attractors-frontend:latest" - resources { - limits = { - cpu = "1000m" - memory = "1Gi" - } - } - env { - name = "MY_ENV_VAR" - value = "my-value" - } - ports { - container_port = 8080 - } - } - timeout_seconds = 300 - } - } - - traffic { - percent = 100 - latest_revision = true - } -} - -resource "google_cloud_run_service_iam_member" "public" { - service = google_cloud_run_service.cloudrun_frontend_service.name - location = google_cloud_run_service.cloudrun_frontend_service.location - role = "roles/run.invoker" - member = "allUsers" -} diff --git a/terraform/main.tf b/terraform/main.tf index bf4d387..fd69a49 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -3,6 +3,9 @@ provider "google" { region = "us-central1" } +data "google_project" "current" { + project_id = "geo-attractors" +} terraform { backend "gcs" { @@ -11,10 +14,6 @@ terraform { } } -data "google_project" "current" { - project_id = "geo-attractors" -} - resource "google_artifact_registry_repository" "my_ar_repo" { location = "us-central1" repository_id = "attractors" @@ -22,51 +21,26 @@ resource "google_artifact_registry_repository" "my_ar_repo" { } resource "google_project_iam_member" "artifact_registry_reader" { - project = "geo-attractors" + project = data.google_project.current.name role = "roles/artifactregistry.reader" member = "serviceAccount:service-${data.google_project.current.number}@serverless-robot-prod.iam.gserviceaccount.com" } -resource "google_cloud_run_service" "my_cloudrun_service" { - name = "attractors-service" - location = "us-central1" - - template { - spec { - containers { - image = "us-central1-docker.pkg.dev/geo-attractors/${google_artifact_registry_repository.my_ar_repo.name}/attractors-fastapi:latest" - resources { - limits = { - cpu = "1000m" - memory = "2Gi" - } - } - env { - name = "MY_ENV_VAR" - value = "my-value" - } - ports { - container_port = 8080 - } - } - timeout_seconds = 300 - } - } - - traffic { - percent = 100 - latest_revision = true - } -} -resource "google_cloud_run_service_iam_member" "public" { - service = google_cloud_run_service.my_cloudrun_service.name - location = google_cloud_run_service.my_cloudrun_service.location - role = "roles/run.invoker" - member = "allUsers" +module "backend" { + source = "./cloudrun" + service_name = "attractors-backend-service" + container_image = "attractors-fastapi" + ar_repo_name = google_artifact_registry_repository.my_ar_repo.name + ar_repo_location = google_artifact_registry_repository.my_ar_repo.location + is_public = true } module "frontend" { - source = "./frontend" - ar_repo_name = google_artifact_registry_repository.my_ar_repo.name + source = "./cloudrun" + service_name = "attractors-frontend-service" + container_image = "attractors-frontend" + ar_repo_name = google_artifact_registry_repository.my_ar_repo.name + ar_repo_location = google_artifact_registry_repository.my_ar_repo.location + is_public = true } From 1eab8eaae8a1ad53fb3a1fdd3565c996600c1808 Mon Sep 17 00:00:00 2001 From: Geo Date: Tue, 9 Jan 2024 01:52:53 +0000 Subject: [PATCH 02/17] terraform update so backend_url get set correctly --- .vscode/launch.json | 11 +- README.md | 11 +- backend/.dockerignore | 3 +- backend/README.md | 432 +----------------- .../src/api/attractors/attractor_service.py | 4 +- frontend/.dockerignore | 3 +- frontend/README.md | 2 +- frontend/src/components.py | 10 +- frontend/src/settings.py | 8 +- scripts/devstart.sh | 5 + terraform/cloudrun/main.tf | 4 +- terraform/cloudrun/variables.tf | 6 + terraform/main.tf | 3 + 13 files changed, 58 insertions(+), 444 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 4e28f43..23697d5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,10 +5,19 @@ "version": "0.2.0", "configurations": [ { - "name": "FastAPI", + "name": "Dash Frontend", "type": "python", "request": "launch", "module": "src", + "justMyCode": true, + "cwd": "${workspaceFolder}/frontend", + "python": "${workspaceFolder}/frontend/.venv-front/bin/python" + }, + { + "name": "FastAPI", + "type": "python", + "request": "launch", + "module": "backend/src", "justMyCode": true } ] diff --git a/README.md b/README.md index 1650481..95ff022 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Im now seeing in jit that it can do caching. But that wouldn't be helpful as my - starting with cloud run made some terraform and some startup scripts had to send it to gcloud to build 1. on the m1 mac the build is strange and 2. can't build in the docker container -`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-fastapi` +`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-fastapi"` this will be solved when its just automatic with the ci/cd pipeline thats totally coming soon @@ -84,3 +84,12 @@ To add an exception for this directory, call: git config --global --add safe.directory /workspaces/attractorsIII ``` - doesn't open properly in code-space got an error on creations will check later at the airport on my ipad. not sure if it was the my network or i had an extra comma that broke it probably the latter as the error log didn't say much about the build arg its very slow probably need to give it more compute. could it be that its using my wifi to download packages I don't think so. + + +# two services one repo +each service will have its own venv and requirements.txt. But the requirements.txt in the root of this project will have all the dev dependencies need to run pre-commit hooks and any other project dependencies. + +```mermaid +graph LR + A[Dash Frontend] <--> B[Backend FastAPI] +``` diff --git a/backend/.dockerignore b/backend/.dockerignore index 0b1e1e7..497a1db 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -1,5 +1,5 @@ **/__pycache__ -**/.venv +**/.venv* **/.classpath **/.dockerignore **/.env @@ -17,7 +17,6 @@ **/charts **/docker-compose* **/compose* -**/Dockerfile* **/node_modules **/npm-debug.log **/obj diff --git a/backend/README.md b/backend/README.md index 97dd257..6de0c1c 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,431 +1 @@ -# terraform-docs - -[![Build Status](https://github.com/terraform-docs/terraform-docs/workflows/ci/badge.svg)](https://github.com/terraform-docs/terraform-docs/actions) [![GoDoc](https://pkg.go.dev/badge/github.com/terraform-docs/terraform-docs)](https://pkg.go.dev/github.com/terraform-docs/terraform-docs) [![Go Report Card](https://goreportcard.com/badge/github.com/terraform-docs/terraform-docs)](https://goreportcard.com/report/github.com/terraform-docs/terraform-docs) [![Codecov Report](https://codecov.io/gh/terraform-docs/terraform-docs/branch/master/graph/badge.svg)](https://codecov.io/gh/terraform-docs/terraform-docs) [![License](https://img.shields.io/github/license/terraform-docs/terraform-docs)](https://github.com/terraform-docs/terraform-docs/blob/master/LICENSE) [![Latest release](https://img.shields.io/github/v/release/terraform-docs/terraform-docs)](https://github.com/terraform-docs/terraform-docs/releases) - -![terraform-docs-teaser](./images/terraform-docs-teaser.png) - -## What is terraform-docs - -A utility to generate documentation from Terraform modules in various output formats. - -## Installation - -macOS users can install using [Homebrew]: - -```bash -brew install terraform-docs -``` - -or - -```bash -brew install terraform-docs/tap/terraform-docs -``` - -Windows users can install using [Scoop]: - -```bash -scoop bucket add terraform-docs https://github.com/terraform-docs/scoop-bucket -scoop install terraform-docs -``` - -or [Chocolatey]: - -```bash -choco install terraform-docs -``` - -Stable binaries are also available on the [releases] page. To install, download the -binary for your platform from "Assets" and place this into your `$PATH`: - -```bash -curl -Lo ./terraform-docs.tar.gz https://github.com/terraform-docs/terraform-docs/releases/download/v0.17.0/terraform-docs-v0.17.0-$(uname)-amd64.tar.gz -tar -xzf terraform-docs.tar.gz -chmod +x terraform-docs -mv terraform-docs /usr/local/bin/terraform-docs -``` - -**NOTE:** Windows releases are in `ZIP` format. - -The latest version can be installed using `go install` or `go get`: - -```bash -# go1.17+ -go install github.com/terraform-docs/terraform-docs@v0.17.0 -``` - -```bash -# go1.16 -GO111MODULE="on" go get github.com/terraform-docs/terraform-docs@v0.17.0 -``` - -**NOTE:** please use the latest Go to do this, minimum `go1.16` is required. - -This will put `terraform-docs` in `$(go env GOPATH)/bin`. If you encounter the error -`terraform-docs: command not found` after installation then you may need to either add -that directory to your `$PATH` as shown [here] or do a manual installation by cloning -the repo and run `make build` from the repository which will put `terraform-docs` in: - -```bash -$(go env GOPATH)/src/github.com/terraform-docs/terraform-docs/bin/$(uname | tr '[:upper:]' '[:lower:]')-amd64/terraform-docs -``` - -## Usage - -### Running the binary directly - -To run and generate documentation into README within a directory: - -```bash -terraform-docs markdown table --output-file README.md --output-mode inject /path/to/module -``` - -Check [`output`] configuration for more details and examples. - -### Using docker - -terraform-docs can be run as a container by mounting a directory with `.tf` -files in it and run the following command: - -```bash -docker run --rm --volume "$(pwd):/terraform-docs" -u $(id -u) quay.io/terraform-docs/terraform-docs:0.17.0 markdown /terraform-docs -``` - -If `output.file` is not enabled for this module, generated output can be redirected -back to a file: - -```bash -docker run --rm --volume "$(pwd):/terraform-docs" -u $(id -u) quay.io/terraform-docs/terraform-docs:0.17.0 markdown /terraform-docs > doc.md -``` - -**NOTE:** Docker tag `latest` refers to _latest_ stable released version and `edge` -refers to HEAD of `master` at any given point in time. - -### Using GitHub Actions - -To use terraform-docs GitHub Action, configure a YAML workflow file (e.g. -`.github/workflows/documentation.yml`) with the following: - -```yaml -name: Generate terraform docs -on: - - pull_request - -jobs: - docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.ref }} - - - name: Render terraform docs and push changes back to PR - uses: terraform-docs/gh-actions@main - with: - working-dir: . - output-file: README.md - output-method: inject - git-push: "true" -``` - -Read more about [terraform-docs GitHub Action] and its configuration and -examples. - -### pre-commit hook - -With pre-commit, you can ensure your Terraform module documentation is kept -up-to-date each time you make a commit. - -First [install pre-commit] and then create or update a `.pre-commit-config.yaml` -in the root of your Git repo with at least the following content: - -```yaml -repos: - - repo: https://github.com/terraform-docs/terraform-docs - rev: "v0.17.0" - hooks: - - id: terraform-docs-go - args: ["markdown", "table", "--output-file", "README.md", "./mymodule/path"] -``` - -Then run: - -```bash -pre-commit install -pre-commit install-hooks -``` - -Further changes to your module's `.tf` files will cause an update to documentation -when you make a commit. - -## Configuration - -terraform-docs can be configured with a yaml file. The default name of this file is -`.terraform-docs.yml` and the path order for locating it is: - -1. root of module directory -1. `.config/` folder at root of module directory -1. current directory -1. `.config/` folder at current directory -1. `$HOME/.tfdocs.d/` - -```yaml -formatter: "" # this is required - -version: "" - -header-from: main.tf -footer-from: "" - -recursive: - enabled: false - path: modules - -sections: - hide: [] - show: [] - -content: "" - -output: - file: "" - mode: inject - template: |- - - {{ .Content }} - - -output-values: - enabled: false - from: "" - -sort: - enabled: true - by: name - -settings: - anchor: true - color: true - default: true - description: false - escape: true - hide-empty: false - html: true - indent: 2 - lockfile: true - read-comments: true - required: true - sensitive: true - type: true -``` - -## Content Template - -Generated content can be customized further away with `content` in configuration. -If the `content` is empty the default order of sections is used. - -Compatible formatters for customized content are `asciidoc` and `markdown`. `content` -will be ignored for other formatters. - -`content` is a Go template with following additional variables: - -- `{{ .Header }}` -- `{{ .Footer }}` -- `{{ .Inputs }}` -- `{{ .Modules }}` -- `{{ .Outputs }}` -- `{{ .Providers }}` -- `{{ .Requirements }}` -- `{{ .Resources }}` - -and following functions: - -- `{{ include "relative/path/to/file" }}` - -These variables are the generated output of individual sections in the selected -formatter. For example `{{ .Inputs }}` is Markdown Table representation of _inputs_ -when formatter is set to `markdown table`. - -Note that sections visibility (i.e. `sections.show` and `sections.hide`) takes -precedence over the `content`. - -Additionally there's also one extra special variable avaialble to the `content`: - -- `{{ .Module }}` - -As opposed to the other variables mentioned above, which are generated sections -based on a selected formatter, the `{{ .Module }}` variable is just a `struct` -representing a [Terraform module]. - -````yaml -content: |- - Any arbitrary text can be placed anywhere in the content - - {{ .Header }} - - and even in between sections - - {{ .Providers }} - - and they don't even need to be in the default order - - {{ .Outputs }} - - include any relative files - - {{ include "relative/path/to/file" }} - - {{ .Inputs }} - - # Examples - - ```hcl - {{ include "examples/foo/main.tf" }} - ``` - - ## Resources - - {{ range .Module.Resources }} - - {{ .GetMode }}.{{ .Spec }} ({{ .Position.Filename }}#{{ .Position.Line }}) - {{- end }} -```` - -## Build on top of terraform-docs - -terraform-docs primary use-case is to be utilized as a standalone binary, but -some parts of it is also available publicly and can be imported in your project -as a library. - -```go -import ( - "github.com/terraform-docs/terraform-docs/format" - "github.com/terraform-docs/terraform-docs/print" - "github.com/terraform-docs/terraform-docs/terraform" -) - -// buildTerraformDocs for module root `path` and provided content `tmpl`. -func buildTerraformDocs(path string, tmpl string) (string, error) { - config := print.DefaultConfig() - config.ModuleRoot = path // module root path (can be relative or absolute) - - module, err := terraform.LoadWithOptions(config) - if err != nil { - return "", err - } - - // Generate in Markdown Table format - formatter := format.NewMarkdownTable(config) - - if err := formatter.Generate(module); err != nil { - return "", err - } - - // // Note: if you don't intend to provide additional template for the generated - // // content, or the target format doesn't provide templating (e.g. json, yaml, - // // xml, or toml) you can use `Content()` function instead of `Render()`. - // // `Content()` returns all the sections combined with predefined order. - // return formatter.Content(), nil - - return formatter.Render(tmpl) -} -``` - -## Plugin - -Generated output can be heavily customized with [`content`], but if using that -is not enough for your use-case, you can write your own plugin. - -In order to install a plugin the following steps are needed: - -- download the plugin and place it in `~/.tfdocs.d/plugins` (or `./.tfdocs.d/plugins`) -- make sure the plugin file name is `tfdocs-format-` -- modify [`formatter`] of `.terraform-docs.yml` file to be `` - -**Important notes:** - -- if the plugin file name is different than the example above, terraform-docs won't -be able to to pick it up nor register it properly -- you can only use plugin thorough `.terraform-docs.yml` file and it cannot be used -with CLI arguments - -To create a new plugin create a new repository called `tfdocs-format-` with -following `main.go`: - -```go -package main - -import ( - _ "embed" //nolint - - "github.com/terraform-docs/terraform-docs/plugin" - "github.com/terraform-docs/terraform-docs/print" - "github.com/terraform-docs/terraform-docs/template" - "github.com/terraform-docs/terraform-docs/terraform" -) - -func main() { - plugin.Serve(&plugin.ServeOpts{ - Name: "", - Version: "0.1.0", - Printer: printerFunc, - }) -} - -//go:embed sections.tmpl -var tplCustom []byte - -// printerFunc the function being executed by the plugin client. -func printerFunc(config *print.Config, module *terraform.Module) (string, error) { - tpl := template.New(config, - &template.Item{Name: "custom", Text: string(tplCustom)}, - ) - - rendered, err := tpl.Render("custom", module) - if err != nil { - return "", err - } - - return rendered, nil -} -``` - -Please refer to [tfdocs-format-template] for more details. You can create a new -repository from it by clicking on `Use this template` button. - -## Documentation - -- **Users** - - Read the [User Guide] to learn how to use terraform-docs - - Read the [Formats Guide] to learn about different output formats of terraform-docs - - Refer to [Config File Reference] for all the available configuration options -- **Developers** - - Read [Contributing Guide] before submitting a pull request - -Visit [our website] for all documentation. - -## Community - -- Discuss terraform-docs on [Slack] - -## License - -MIT License - Copyright (c) 2021 The terraform-docs Authors. - -[Chocolatey]: https://www.chocolatey.org -[Config File Reference]: https://terraform-docs.io/user-guide/configuration/ -[`content`]: https://terraform-docs.io/user-guide/configuration/content/ -[Contributing Guide]: CONTRIBUTING.md -[Formats Guide]: https://terraform-docs.io/reference/terraform-docs/ -[`formatter`]: https://terraform-docs.io/user-guide/configuration/formatter/ -[here]: https://golang.org/doc/code.html#GOPATH -[Homebrew]: https://brew.sh -[install pre-commit]: https://pre-commit.com/#install -[`output`]: https://terraform-docs.io/user-guide/configuration/output/ -[releases]: https://github.com/terraform-docs/terraform-docs/releases -[Scoop]: https://scoop.sh/ -[Slack]: https://slack.terraform-docs.io/ -[terraform-docs GitHub Action]: https://github.com/terraform-docs/gh-actions -[Terraform module]: https://pkg.go.dev/github.com/terraform-docs/terraform-docs/terraform#Module -[tfdocs-format-template]: https://github.com/terraform-docs/tfdocs-format-template -[our website]: https://terraform-docs.io/ -[User Guide]: https://terraform-docs.io/user-guide/introduction/ +This backend uses FastAPI. diff --git a/backend/src/api/attractors/attractor_service.py b/backend/src/api/attractors/attractor_service.py index 0a6e82a..8520d17 100644 --- a/backend/src/api/attractors/attractor_service.py +++ b/backend/src/api/attractors/attractor_service.py @@ -36,8 +36,9 @@ def trajectory(self, fn, x0, y0, a, b=0, c=0, d=0, e=0, f=0, n=10000000): def gen_random(self, func, desired_empty=10000): # finds some nice initial conditions non_empty = 0 + tries = 0 # how many non empty pixels - while non_empty < desired_empty: + while non_empty < desired_empty or tries <= 10: initial_conditions = np.c_[ np.zeros((1, 2)), np.random.random((1, 6)) * 4 - 2 ][0] @@ -48,6 +49,7 @@ def gen_random(self, func, desired_empty=10000): agg = cvs.points(df, "x", "y") non_empty = np.count_nonzero(np.array(agg)) logger.info(f"non_empty: {non_empty}") + tries += 1 return initial_conditions def make_dataframe( diff --git a/frontend/.dockerignore b/frontend/.dockerignore index 0b1e1e7..497a1db 100644 --- a/frontend/.dockerignore +++ b/frontend/.dockerignore @@ -1,5 +1,5 @@ **/__pycache__ -**/.venv +**/.venv* **/.classpath **/.dockerignore **/.env @@ -17,7 +17,6 @@ **/charts **/docker-compose* **/compose* -**/Dockerfile* **/node_modules **/npm-debug.log **/obj diff --git a/frontend/README.md b/frontend/README.md index 05020db..ed468cf 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -11,4 +11,4 @@ TODO So dash uses flask for its server backend so it uses gunicorn which is a WSGI (Web server gateway interface) and wont work with uvicorn a ASGI (async server gateway interface) -gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-frontend" +gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-frontend" --ignore-file .dockerignore diff --git a/frontend/src/components.py b/frontend/src/components.py index 886d262..b2d9d44 100644 --- a/frontend/src/components.py +++ b/frontend/src/components.py @@ -17,8 +17,14 @@ id="function-dropdown", options=[ {"label": "Clifford", "value": "Clifford"}, - {"label": "De Jong", "value": "de_jong"}, - {"label": "Bedhead", "value": "bedhead"}, + {"label": "De Jong", "value": "De Jong"}, + {"label": "Svensson", "value": "Svensson"}, + {"label": "Fractal Dream", "value": "Fractal Dream"}, + {"label": "Bedhead", "value": "Bedhead"}, + {"label": "HopaLong", "value": "Hopalong"}, + {"label": "HopaLong2", "value": "Hopalong2"}, + {"label": "Gumowski Mira", "value": "Gumowski Mira"}, + {"label": "Symmetric Icon", "values": "Symmetric Icon"}, ], value="Clifford", ) diff --git a/frontend/src/settings.py b/frontend/src/settings.py index bff85a3..2c931cf 100644 --- a/frontend/src/settings.py +++ b/frontend/src/settings.py @@ -2,6 +2,7 @@ from pathlib import Path from tempfile import gettempdir +from pydantic import validator from pydantic_settings import BaseSettings from yarl import URL @@ -47,7 +48,12 @@ class Settings(BaseSettings): log_level: LogLevel = LogLevel.INFO - backend_url: URL = URL("https://attractors-service-c6dyl3tniq-uc.a.run.app") + backend_url: URL = URL("https://attractors-backend-service-c6dyl3tniq-uc.a.run.app") + + @validator("backend_url", pre=True) + def parse_backend_url(cls, value: str) -> URL: + """Parse backend url.""" + return URL(value) class Config: env_file = ".env" diff --git a/scripts/devstart.sh b/scripts/devstart.sh index 52ae3a4..ae33e6d 100755 --- a/scripts/devstart.sh +++ b/scripts/devstart.sh @@ -26,3 +26,8 @@ sudo apt update && sudo apt install -y terraform # && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ # && sudo apt update \ # && sudo apt install gh -y + +# Brew ---Before uncommenting this will only go to bashrc --- +# /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +# (echo; echo 'eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"') >> /home/vscode/.bashrc +# eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" diff --git a/terraform/cloudrun/main.tf b/terraform/cloudrun/main.tf index 97b8ebd..87a6bed 100644 --- a/terraform/cloudrun/main.tf +++ b/terraform/cloudrun/main.tf @@ -15,8 +15,8 @@ resource "google_cloud_run_service" "cloudrun_service" { } } env { - name = "MY_ENV_VAR" - value = "my-value" + name = "APP_BACKEND_URL" + value = var.backend_url_env } ports { container_port = 8080 diff --git a/terraform/cloudrun/variables.tf b/terraform/cloudrun/variables.tf index 2e38bc0..16bd7ed 100644 --- a/terraform/cloudrun/variables.tf +++ b/terraform/cloudrun/variables.tf @@ -28,3 +28,9 @@ variable "is_public" { type = bool description = "Should the service be public" } + +variable "backend_url_env" { + type = string + description = "The URL of the backend service" + default = "None" +} diff --git a/terraform/main.tf b/terraform/main.tf index fd69a49..8f3f468 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -36,10 +36,13 @@ module "backend" { is_public = true } + + module "frontend" { source = "./cloudrun" service_name = "attractors-frontend-service" container_image = "attractors-frontend" + backend_url_env = module.backend.cloudrun_service_url ar_repo_name = google_artifact_registry_repository.my_ar_repo.name ar_repo_location = google_artifact_registry_repository.my_ar_repo.location is_public = true From 6c809ad9b83a355730d85df095e30b278ef60464 Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 01:40:51 +0000 Subject: [PATCH 03/17] testing gh-federation --- .github/workflows/cloudrun-deploy.yaml | 109 ++++++++++++++++++++++++ README.md | 2 +- backend/README.md | 2 + backend/requirements.txt | 33 +++++++ terraform/.terraform.lock.hcl | 20 +++++ terraform/gh-id-federation/main.tf | 43 ++++++++++ terraform/gh-id-federation/outputs.tf | 30 +++++++ terraform/gh-id-federation/variables.tf | 20 +++++ terraform/main.tf | 10 ++- 9 files changed, 265 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/cloudrun-deploy.yaml create mode 100644 terraform/gh-id-federation/main.tf create mode 100644 terraform/gh-id-federation/outputs.tf create mode 100644 terraform/gh-id-federation/variables.tf diff --git a/.github/workflows/cloudrun-deploy.yaml b/.github/workflows/cloudrun-deploy.yaml new file mode 100644 index 0000000..3dff10c --- /dev/null +++ b/.github/workflows/cloudrun-deploy.yaml @@ -0,0 +1,109 @@ +# This workflow build and push a Docker container to Google Artifact Registry and deploy it on Cloud Run when a commit is pushed to the "main" branch +# +# Overview: +# +# 1. Authenticate to Google Cloud +# 2. Authenticate Docker to Artifact Registry +# 3. Build a docker container +# 4. Publish it to Google Artifact Registry +# 5. Deploy it to Cloud Run +# +# To configure this workflow: +# +# 1. Ensure the required Google Cloud APIs are enabled: +# +# Cloud Run run.googleapis.com +# Artifact Registry artifactregistry.googleapis.com +# +# 2. Create and configure Workload Identity Federation for GitHub (https://github.com/google-github-actions/auth#setting-up-workload-identity-federation) +# +# 3. Ensure the required IAM permissions are granted +# +# Cloud Run +# roles/run.admin +# roles/iam.serviceAccountUser (to act as the Cloud Run runtime service account) +# +# Artifact Registry +# roles/artifactregistry.admin (project or repository level) +# +# NOTE: You should always follow the principle of least privilege when assigning IAM roles +# +# 4. Create GitHub secrets for WIF_PROVIDER and WIF_SERVICE_ACCOUNT +# +# 5. Change the values for the GAR_LOCATION, SERVICE and REGION environment variables (below). +# +# NOTE: To use Google Container Registry instead, replace ${{ env.GAR_LOCATION }}-docker.pkg.dev with gcr.io +# +# For more support on how to run this workflow, please visit https://github.com/marketplace/actions/deploy-to-cloud-run +# +# Further reading: +# Cloud Run IAM permissions - https://cloud.google.com/run/docs/deploying +# Artifact Registry IAM permissions - https://cloud.google.com/artifact-registry/docs/access-control#roles +# Container Registry vs Artifact Registry - https://cloud.google.com/blog/products/application-development/understanding-artifact-registry-vs-container-registry +# Principle of least privilege - https://cloud.google.com/blog/products/identity-security/dont-get-pwned-practicing-the-principle-of-least-privilege +name: Build and Deploy to Cloud Run + +on: + push #: + # branches: [ main ] + +env: + PROJECT_ID: "geo-attractors" + GAR_LOCATION: "us-central1" + SERVICES: ['frontend', 'backend'] + REGION: 'us-central1' + REPO: 'attractors' + +jobs: + deploy: + permissions: + contents: 'read' + id-token: 'write' + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v1' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' # e.g. - projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' # e.g. - my-service-account@my-project.iam.gserviceaccount.com + + # Authenticate Docker to Google Cloud Artifact Registry + - name: Login to GAR + uses: docker/login-action@v3 + with: + registry: '${{ env.GAR_LOCATION }}-docker.pkg.dev' + username: 'oauth2accesstoken' + password: ${{ steps.auth.outputs.access_token }} + + - name: Build and Push Backend + run: |- + cd backend/ + docker build --target prod -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest" ./ + docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest" + + - name: Build and Push Frontend + run: |- + cd frontend/ + docker build --target prod -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest" ./ + docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest" + + # END - Docker auth and build + + - name: Deploy to Cloud Run + id: deploy + uses: google-github-actions/deploy-cloudrun@v1 + with: + service: attractors-backend-service + region: ${{ env.REGION }} + # NOTE: If using a pre-built image, update the image name here + image: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/${{ env.SERVICE }}:latest + + # If required, use the Cloud Run url output in later steps + - name: Show Output + run: echo ${{ steps.deploy.outputs.url }} diff --git a/README.md b/README.md index 95ff022..f091ead 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Im now seeing in jit that it can do caching. But that wouldn't be helpful as my - starting with cloud run made some terraform and some startup scripts had to send it to gcloud to build 1. on the m1 mac the build is strange and 2. can't build in the docker container -`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-fastapi"` +`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-backend"` this will be solved when its just automatic with the ci/cd pipeline thats totally coming soon diff --git a/backend/README.md b/backend/README.md index 6de0c1c..c1bde1a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1 +1,3 @@ This backend uses FastAPI. + +`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-backend" --ignore-file .dockerignore` diff --git a/backend/requirements.txt b/backend/requirements.txt index 687d13a..377bd89 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,16 +1,49 @@ +aiocache==0.12.2 annotated-types==0.6.0 anyio==4.2.0 +certifi==2023.11.17 +charset-normalizer==3.3.2 click==8.1.7 +cloudpickle==3.0.0 +colorcet==3.0.1 +dask==2023.12.1 +datashader==0.16.0 fastapi==0.108.0 +fsspec==2023.12.2 h11==0.14.0 idna==3.6 +importlib-metadata==7.0.1 +llvmlite==0.41.1 +locket==1.0.0 +loguru==0.7.2 +multipledispatch==1.0.0 +numba==0.58.1 +numpy==1.26.3 +orjson==3.9.10 +packaging==23.2 +pandas==2.1.4 +param==2.0.1 +partd==1.4.1 +pillow==10.2.0 prometheus-client==0.19.0 prometheus-fastapi-instrumentator==6.1.0 +pyct==0.5.0 pydantic==2.5.3 pydantic-settings==2.1.0 pydantic_core==2.14.6 +python-dateutil==2.8.2 python-dotenv==1.0.0 +pytz==2023.3.post1 +PyYAML==6.0.1 +requests==2.31.0 +scipy==1.11.4 +six==1.16.0 sniffio==1.3.0 starlette==0.32.0.post1 +toolz==0.12.0 typing_extensions==4.9.0 +tzdata==2023.4 +urllib3==2.1.0 uvicorn==0.25.0 +xarray==2023.12.0 +zipp==3.17.0 diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 38c94fa..b18059a 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -20,3 +20,23 @@ provider "registry.terraform.io/hashicorp/google" { "zh:fb82f6b5d1f992243ab8fe417659cdf9831202cf1e16fe7593d3967888b035cc", ] } + +provider "registry.terraform.io/hashicorp/google-beta" { + version = "5.11.0" + constraints = ">= 3.64.0, < 6.0.0" + hashes = [ + "h1:yoAwpGJ3gxIoctH5U4yEcBfE2IJowny7ul1EWkRusww=", + "zh:0efa82e6fe2c83bd5280c3009db1c3acc9cdad3c9419b6ec721fbefc9f832449", + "zh:371df01e4f38b828195d115c9a8bebddebec4d34e9ef74cf3a79161da08e44b2", + "zh:5089967c420c5e4a4ba0d4c8c6ca344c7bb2476ec928f8319856260eacded369", + "zh:798a65c79386d356d6a097de680f4ece8982daae1cb0e10d6c53b383efef45f0", + "zh:90178911ac0e624c69a54a992fb3425ef09fdfb3e34b496ad7b6e168e80d4e0c", + "zh:b59c60f8479b8f0c8e91a93a4e707ce6d17c8e50e2f5afaf1d9a03c03cfedbf8", + "zh:c7f946282d80223ab3a6b284c22e4b53ffcd7b1a02449bb95a350007f30c87dc", + "zh:cd60e76987c2fdce2c84219eaff9390cd135f88aa9a27bc4d79a8fd4a8d09622", + "zh:de06bfa0393206c0253ebdea70821cb3b08ef87d5d4844be3ae463abfb4e1884", + "zh:de494bad600cca78986ce63d1018f5dbc1a1fcc2d4c41c94c15d5346f2b0dd1e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f97a8b6e83e0083dcb42a87e8e418ab33f12d641f9cdfdc92d154ba7fd7398fb", + ] +} diff --git a/terraform/gh-id-federation/main.tf b/terraform/gh-id-federation/main.tf new file mode 100644 index 0000000..cfa3db2 --- /dev/null +++ b/terraform/gh-id-federation/main.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +resource "google_service_account" "sa" { + project = var.project_id + account_id = "gh-federation-sa" +} + +resource "google_project_iam_member" "project" { + project = var.project_id + role = "roles/storage.admin" + member = "serviceAccount:${google_service_account.sa.email}" +} + +module "oidc" { + source = "terraform-google-modules/github-actions-runners/google//modules/gh-oidc" + version = "~> 3.0" + + project_id = var.project_id + pool_id = "my-pool" + provider_id = "gh-provider" + sa_mapping = { + (google_service_account.sa.account_id) = { + sa_name = google_service_account.sa.name + # Note this is the Attractors repo in my CupOfGeo user + attribute = "attribute.repository/CupOfGeo/Attractors" + } + } +} diff --git a/terraform/gh-id-federation/outputs.tf b/terraform/gh-id-federation/outputs.tf new file mode 100644 index 0000000..ac46dd9 --- /dev/null +++ b/terraform/gh-id-federation/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "pool_name" { + description = "Pool name" + value = module.oidc.pool_name +} + +output "provider_name" { + description = "Provider name" + value = module.oidc.provider_name +} + +output "sa_email" { + description = "Example SA email" + value = google_service_account.sa.email +} diff --git a/terraform/gh-id-federation/variables.tf b/terraform/gh-id-federation/variables.tf new file mode 100644 index 0000000..f7bf043 --- /dev/null +++ b/terraform/gh-id-federation/variables.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string + description = "The project id to create WIF pool and example SA" +} diff --git a/terraform/main.tf b/terraform/main.tf index 8f3f468..03de753 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -30,14 +30,12 @@ resource "google_project_iam_member" "artifact_registry_reader" { module "backend" { source = "./cloudrun" service_name = "attractors-backend-service" - container_image = "attractors-fastapi" + container_image = "attractors-backend" ar_repo_name = google_artifact_registry_repository.my_ar_repo.name ar_repo_location = google_artifact_registry_repository.my_ar_repo.location is_public = true } - - module "frontend" { source = "./cloudrun" service_name = "attractors-frontend-service" @@ -47,3 +45,9 @@ module "frontend" { ar_repo_location = google_artifact_registry_repository.my_ar_repo.location is_public = true } + +# https://github.com/terraform-google-modules/terraform-google-github-actions-runners/blob/master/modules/gh-oidc/README.md +module "gh-federation" { + source = "./gh-id-federation" + project_id = data.google_project.current.name +} From 9d2f279334b2165a4343022e924a308c73001557 Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 01:42:23 +0000 Subject: [PATCH 04/17] fixing workflow --- .github/workflows/cloudrun-deploy.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cloudrun-deploy.yaml b/.github/workflows/cloudrun-deploy.yaml index 3dff10c..160050a 100644 --- a/.github/workflows/cloudrun-deploy.yaml +++ b/.github/workflows/cloudrun-deploy.yaml @@ -50,7 +50,6 @@ on: env: PROJECT_ID: "geo-attractors" GAR_LOCATION: "us-central1" - SERVICES: ['frontend', 'backend'] REGION: 'us-central1' REPO: 'attractors' From 8c75795781961d9cac8163cb7db9da9ff7ebb826 Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 02:16:51 +0000 Subject: [PATCH 05/17] added WIF secrets --- README.md | 13 +++++++++++++ terraform/gh-id-federation/main.tf | 3 +-- terraform/gh-id-federation/variables.tf | 10 ++++++++++ terraform/main.tf | 14 +++++++++++++- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f091ead..7fd61d4 100644 --- a/README.md +++ b/README.md @@ -93,3 +93,16 @@ each service will have its own venv and requirements.txt. But the requirements.t graph LR A[Dash Frontend] <--> B[Backend FastAPI] ``` + + +# GH workload federation stuff + +[Here is the module I used](https://github.com/terraform-google-modules/terraform-google-github-actions-runners/blob/master/modules/gh-oidc/README.md +) + +- two variables need to be set you get them after applying the terraform and then running `terraform output` and then use the value from the two outputs +`gh_federation_provider_name and gh_federation_sa_email` +```bash +gh secret set WIF_PROVIDER -b"gh_federation_provider_name" +gh secret set WIF_SERVICE_ACCOUNT -b"gh_federation_sa_email" +``` diff --git a/terraform/gh-id-federation/main.tf b/terraform/gh-id-federation/main.tf index cfa3db2..63ba253 100644 --- a/terraform/gh-id-federation/main.tf +++ b/terraform/gh-id-federation/main.tf @@ -36,8 +36,7 @@ module "oidc" { sa_mapping = { (google_service_account.sa.account_id) = { sa_name = google_service_account.sa.name - # Note this is the Attractors repo in my CupOfGeo user - attribute = "attribute.repository/CupOfGeo/Attractors" + attribute = "attribute.repository/${var.github_org}/${var.github_repo}" } } } diff --git a/terraform/gh-id-federation/variables.tf b/terraform/gh-id-federation/variables.tf index f7bf043..acd7b7b 100644 --- a/terraform/gh-id-federation/variables.tf +++ b/terraform/gh-id-federation/variables.tf @@ -18,3 +18,13 @@ variable "project_id" { type = string description = "The project id to create WIF pool and example SA" } + +variable "github_repo" { + type = string + description = "The name of the repo" +} + +variable "github_org" { + type = string + description = "The org or user the github repo is under" +} diff --git a/terraform/main.tf b/terraform/main.tf index 03de753..3401eb4 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -46,8 +46,20 @@ module "frontend" { is_public = true } -# https://github.com/terraform-google-modules/terraform-google-github-actions-runners/blob/master/modules/gh-oidc/README.md module "gh-federation" { source = "./gh-id-federation" project_id = data.google_project.current.name + github_org = "CupOfGeo" + github_repo = "Attractors" +} + + +output "gh_federation_provider_name" { + description = "Provider name from gh-federation module" + value = module.gh-federation.provider_name +} + +output "gh_federation_sa_email" { + description = "Service account email from gh-federation module" + value = module.gh-federation.sa_email } From 1beaf711b6d50c97dae36c63ae5e6b0d63a2d651 Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 02:19:20 +0000 Subject: [PATCH 06/17] enable IAM Service Account Credentials API --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7fd61d4..de9ed68 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,9 @@ graph LR [Here is the module I used](https://github.com/terraform-google-modules/terraform-google-github-actions-runners/blob/master/modules/gh-oidc/README.md ) -- two variables need to be set you get them after applying the terraform and then running `terraform output` and then use the value from the two outputs +- need to enable IAM Service Account Credentials API + +- two variables need to be set you get them after applying the terraform and then running `terraform output` and then use the value from the outputs `gh_federation_provider_name and gh_federation_sa_email` ```bash gh secret set WIF_PROVIDER -b"gh_federation_provider_name" From a3bbd8b7f414ca1e80609594a93f66f3d65141ac Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 02:34:58 +0000 Subject: [PATCH 07/17] giving ar.writer permissions to gh sa --- .github/workflows/cloudrun-deploy.yaml | 4 ++-- backend/Dockerfile | 4 ++-- frontend/Dockerfile | 4 ++-- terraform/gh-id-federation/main.tf | 6 ++++++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cloudrun-deploy.yaml b/.github/workflows/cloudrun-deploy.yaml index 160050a..5c7e835 100644 --- a/.github/workflows/cloudrun-deploy.yaml +++ b/.github/workflows/cloudrun-deploy.yaml @@ -83,13 +83,13 @@ jobs: - name: Build and Push Backend run: |- cd backend/ - docker build --target prod -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest" ./ + docker build --target BUILD -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest" ./ docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest" - name: Build and Push Frontend run: |- cd frontend/ - docker build --target prod -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest" ./ + docker build --target BUILD -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest" ./ docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest" # END - Docker auth and build diff --git a/backend/Dockerfile b/backend/Dockerfile index cd07ff4..bcc0f70 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -### PROD ### -FROM python:3.11-slim-buster as prod +### BUILD ### +FROM python:3.11-slim-buster as BUILD RUN apt-get update # Set the working directory to /app WORKDIR /app diff --git a/frontend/Dockerfile b/frontend/Dockerfile index cd07ff4..bcc0f70 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,5 +1,5 @@ -### PROD ### -FROM python:3.11-slim-buster as prod +### BUILD ### +FROM python:3.11-slim-buster as BUILD RUN apt-get update # Set the working directory to /app WORKDIR /app diff --git a/terraform/gh-id-federation/main.tf b/terraform/gh-id-federation/main.tf index 63ba253..d21793d 100644 --- a/terraform/gh-id-federation/main.tf +++ b/terraform/gh-id-federation/main.tf @@ -20,6 +20,12 @@ resource "google_service_account" "sa" { account_id = "gh-federation-sa" } +resource "google_project_iam_member" "artifact_registry_writer" { + project = var.project_id + role = "roles/artifactregistry.writer" + member = "serviceAccount:${google_service_account.sa.email}" +} + resource "google_project_iam_member" "project" { project = var.project_id role = "roles/storage.admin" From f8dff866297e6c01635ac2323ba629035cc411c6 Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 02:48:48 +0000 Subject: [PATCH 08/17] one more permission --- .github/workflows/cloudrun-deploy.yaml | 2 +- README.md | 2 +- terraform/gh-id-federation/main.tf | 6 ++++++ terraform/main.tf | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cloudrun-deploy.yaml b/.github/workflows/cloudrun-deploy.yaml index 5c7e835..38b2f50 100644 --- a/.github/workflows/cloudrun-deploy.yaml +++ b/.github/workflows/cloudrun-deploy.yaml @@ -101,7 +101,7 @@ jobs: service: attractors-backend-service region: ${{ env.REGION }} # NOTE: If using a pre-built image, update the image name here - image: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/${{ env.SERVICE }}:latest + image: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest # If required, use the Cloud Run url output in later steps - name: Show Output diff --git a/README.md b/README.md index de9ed68..2ae3dfd 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ so im now in a clean 3.11 dev container :) I have a bash terminal Cool so i updated my pre-commit hooks as well -Im running without a venv bc im already in a clean room +Im running without a venv bc im already in a clean room. This is great now I can install all my dev dependencies and then make a clean .venv with only whats needed to deploy. # Redis Idea to add some sort of caching for the attractors so if i want to recolor it i can pull the initial conditions from a cache diff --git a/terraform/gh-id-federation/main.tf b/terraform/gh-id-federation/main.tf index d21793d..e0df545 100644 --- a/terraform/gh-id-federation/main.tf +++ b/terraform/gh-id-federation/main.tf @@ -26,6 +26,12 @@ resource "google_project_iam_member" "artifact_registry_writer" { member = "serviceAccount:${google_service_account.sa.email}" } +resource "google_project_iam_member" "cloud_run_admin" { + project = var.project_id + role = "roles/run.admin" + member = "serviceAccount:${google_service_account.sa.email}" +} + resource "google_project_iam_member" "project" { project = var.project_id role = "roles/storage.admin" diff --git a/terraform/main.tf b/terraform/main.tf index 3401eb4..8d3b3b2 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -47,9 +47,9 @@ module "frontend" { } module "gh-federation" { - source = "./gh-id-federation" + source = "./gh-id-federation" project_id = data.google_project.current.name - github_org = "CupOfGeo" + github_org = "CupOfGeo" github_repo = "Attractors" } From f7f319be9be4cdaf7d8ee506ee68410224081180 Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 02:54:49 +0000 Subject: [PATCH 09/17] ok maybe two more permissions? --- terraform/gh-id-federation/main.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/terraform/gh-id-federation/main.tf b/terraform/gh-id-federation/main.tf index e0df545..b27f5f5 100644 --- a/terraform/gh-id-federation/main.tf +++ b/terraform/gh-id-federation/main.tf @@ -32,6 +32,12 @@ resource "google_project_iam_member" "cloud_run_admin" { member = "serviceAccount:${google_service_account.sa.email}" } +resource "google_project_iam_member" "service_account_user" { + project = var.project_id + role = "roles/iam.serviceAccountUser" + member = "serviceAccount:${google_service_account.sa.email}" +} + resource "google_project_iam_member" "project" { project = var.project_id role = "roles/storage.admin" From 5caf18b708c0921482ea8b8289852e127729a2b8 Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 03:01:06 +0000 Subject: [PATCH 10/17] updating deploy-cloudrun version --- .github/workflows/cloudrun-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cloudrun-deploy.yaml b/.github/workflows/cloudrun-deploy.yaml index 38b2f50..a5cc07c 100644 --- a/.github/workflows/cloudrun-deploy.yaml +++ b/.github/workflows/cloudrun-deploy.yaml @@ -96,7 +96,7 @@ jobs: - name: Deploy to Cloud Run id: deploy - uses: google-github-actions/deploy-cloudrun@v1 + uses: google-github-actions/deploy-cloudrun@v2 with: service: attractors-backend-service region: ${{ env.REGION }} From 08a7ec33fe16fdba86b627525cae8550911cec94 Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 03:06:53 +0000 Subject: [PATCH 11/17] coolio now deploy frontend as well --- .github/workflows/cloudrun-deploy.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cloudrun-deploy.yaml b/.github/workflows/cloudrun-deploy.yaml index a5cc07c..7364e56 100644 --- a/.github/workflows/cloudrun-deploy.yaml +++ b/.github/workflows/cloudrun-deploy.yaml @@ -94,7 +94,7 @@ jobs: # END - Docker auth and build - - name: Deploy to Cloud Run + - name: Deploy Backend to Cloud Run id: deploy uses: google-github-actions/deploy-cloudrun@v2 with: @@ -104,5 +104,17 @@ jobs: image: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-backend:latest # If required, use the Cloud Run url output in later steps - - name: Show Output + - name: Show Backend URL run: echo ${{ steps.deploy.outputs.url }} + + - name: Deploy Frontend to Cloud Run + id: deploy-frontend + uses: google-github-actions/deploy-cloudrun@v2 + with: + service: attractors-frontend-service + region: ${{ env.REGION }} + # NOTE: If using a pre-built image, update the image name here + image: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.REPO }}/attractors-frontend:latest + + - name: Show Frontend URL + run: echo ${{ steps.deploy-frontend.outputs.url }} From 8985f0ddcc416329c7d6c5ac465157d349b351bd Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 03:38:12 +0000 Subject: [PATCH 12/17] terraform pre-commit hooks --- .pre-commit-config.yaml | 7 +++++++ terraform/README.md | 34 ------------------------------ terraform/cloudrun/README.md | 40 ------------------------------------ terraform/main.tf | 14 +++++++++++++ 4 files changed, 21 insertions(+), 74 deletions(-) delete mode 100644 terraform/README.md delete mode 100644 terraform/cloudrun/README.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 274722a..a6f56b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,3 +54,10 @@ repos: rev: v1.10.0 hooks: - id: python-check-blanket-noqa + + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.86.0 + hooks: + - id: terraform_tflint + - id: terraform_fmt + - id: terraform_docs diff --git a/terraform/README.md b/terraform/README.md deleted file mode 100644 index 7f95f89..0000000 --- a/terraform/README.md +++ /dev/null @@ -1,34 +0,0 @@ - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | 5.10.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [backend](#module\_backend) | ./cloudrun | n/a | -| [frontend](#module\_frontend) | ./cloudrun | n/a | - -## Resources - -| Name | Type | -|------|------| -| [google_artifact_registry_repository.my_ar_repo](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository) | resource | -| [google_project_iam_member.artifact_registry_reader](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource | -| [google_project.current](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -No inputs. - -## Outputs - -No outputs. - diff --git a/terraform/cloudrun/README.md b/terraform/cloudrun/README.md deleted file mode 100644 index 5f4b7d8..0000000 --- a/terraform/cloudrun/README.md +++ /dev/null @@ -1,40 +0,0 @@ - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | n/a | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_cloud_run_service.cloudrun_service](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service) | resource | -| [google_cloud_run_service_iam_member.public](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_run_service_iam_member) | resource | -| [google_project.current](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [ar\_repo\_location](#input\_ar\_repo\_location) | The location of the Artifact Registry repository and the cloudrun service | `string` | n/a | yes | -| [ar\_repo\_name](#input\_ar\_repo\_name) | The name of the Artifact Registry repository | `string` | n/a | yes | -| [container\_image](#input\_container\_image) | image in GAR | `string` | n/a | yes | -| [is\_public](#input\_is\_public) | Should the service be public | `bool` | n/a | yes | -| [location](#input\_location) | GCP region | `string` | `"us-central1"` | no | -| [service\_name](#input\_service\_name) | cloudrun service name | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [cloudrun\_service\_url](#output\_cloudrun\_service\_url) | The URL of the cloud run service | - diff --git a/terraform/main.tf b/terraform/main.tf index 8d3b3b2..81681df 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,3 +1,17 @@ +terraform { + required_version = ">= 1.6.6" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.10" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 5.11" + } + } +} + provider "google" { project = "geo-attractors" region = "us-central1" From 1b730cdf0b3f1f83c7cdc0dbb4a690b3a96d4123 Mon Sep 17 00:00:00 2001 From: Geo Date: Wed, 10 Jan 2024 03:40:16 +0000 Subject: [PATCH 13/17] pre-commit auto fixes --- .pre-commit-config.yaml | 1 - frontend/src/application.py | 3 +-- frontend/src/backend_check.py | 1 - frontend/src/frontend.py | 1 - frontend/src/logging.py | 1 - scripts/devstart.sh | 2 ++ 6 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6f56b0..9ff91ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,4 +60,3 @@ repos: hooks: - id: terraform_tflint - id: terraform_fmt - - id: terraform_docs diff --git a/frontend/src/application.py b/frontend/src/application.py index 1a99932..2e16c5d 100644 --- a/frontend/src/application.py +++ b/frontend/src/application.py @@ -1,7 +1,6 @@ import dash_bootstrap_components as dbc -from dash import Dash, html - import src.frontend as frontend +from dash import Dash, html from src.backend_check import backend_check, backend_check_layout from src.components import layout diff --git a/frontend/src/backend_check.py b/frontend/src/backend_check.py index 7553295..8a583fd 100644 --- a/frontend/src/backend_check.py +++ b/frontend/src/backend_check.py @@ -2,7 +2,6 @@ from dash import dcc, html from dash.dependencies import Input, Output from loguru import logger - from src.settings import settings backend_check_layout = html.Div( diff --git a/frontend/src/frontend.py b/frontend/src/frontend.py index af88c40..699d442 100644 --- a/frontend/src/frontend.py +++ b/frontend/src/frontend.py @@ -5,7 +5,6 @@ from dash.dependencies import Input, Output, State from dash.exceptions import PreventUpdate from loguru import logger - from src.client_mem_model import ClientMemModel from src.settings import settings diff --git a/frontend/src/logging.py b/frontend/src/logging.py index bf02c09..b16a221 100644 --- a/frontend/src/logging.py +++ b/frontend/src/logging.py @@ -3,7 +3,6 @@ from typing import Union from loguru import logger - from src.settings import settings diff --git a/scripts/devstart.sh b/scripts/devstart.sh index ae33e6d..61da0dc 100755 --- a/scripts/devstart.sh +++ b/scripts/devstart.sh @@ -31,3 +31,5 @@ sudo apt update && sudo apt install -y terraform # /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" # (echo; echo 'eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"') >> /home/vscode/.bashrc # eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + +# brew install tflint From ebece063e73a45a078a433861b47f6503b70316f Mon Sep 17 00:00:00 2001 From: George Mazzeo Date: Tue, 16 Jan 2024 20:45:05 +0000 Subject: [PATCH 14/17] removed zsh stuff --- .devcontainer/devcontainer.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e9f3df9..d48136f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -26,11 +26,7 @@ "vscode": { "extensions": [ "streetsidesoftware.code-spell-checker" - ], - "settings": { - "terminal.integrated.shell.linux": "/bin/zsh" - } - + ] } } From 26e3e16649ba4dff3e0b90c7eed18574bd8c7861 Mon Sep 17 00:00:00 2001 From: George Mazzeo Date: Tue, 16 Jan 2024 20:46:45 +0000 Subject: [PATCH 15/17] only run deploy action on push to main --- .github/workflows/cloudrun-deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cloudrun-deploy.yaml b/.github/workflows/cloudrun-deploy.yaml index 7364e56..f40ae87 100644 --- a/.github/workflows/cloudrun-deploy.yaml +++ b/.github/workflows/cloudrun-deploy.yaml @@ -44,8 +44,8 @@ name: Build and Deploy to Cloud Run on: - push #: - # branches: [ main ] + push: + branches: [ main ] env: PROJECT_ID: "geo-attractors" From f8f8c9b15a75e4bffa4dafa8fc74cbcfdea1dfa0 Mon Sep 17 00:00:00 2001 From: George Mazzeo Date: Tue, 16 Jan 2024 23:09:46 +0000 Subject: [PATCH 16/17] Readme --- README.md | 69 +++++++++++++++++----------------------------------- geo-notes.md | 0 2 files changed, 22 insertions(+), 47 deletions(-) create mode 100644 geo-notes.md diff --git a/README.md b/README.md index 2ae3dfd..6a0cd84 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,11 @@ Hey so a long time ago i found this https://examples.holoviz.org/attractors/attractors.html project and thought it looked really cool so i played around with it and wanted to turn it into a rest api. It was a super cool demo learned about jit and datashaders +TODO thank the guy that gave the talk scaling open source science. It going to make me pretty pictures and the server will be a good test service for other things. +# dev containers and CodeSpaces going to learn to use the .devcontainers so im now in a clean 3.11 dev container :) I have a bash terminal @@ -12,50 +14,7 @@ so im now in a clean 3.11 dev container :) I have a bash terminal Cool so i updated my pre-commit hooks as well Im running without a venv bc im already in a clean room. This is great now I can install all my dev dependencies and then make a clean .venv with only whats needed to deploy. - -# Redis -Idea to add some sort of caching for the attractors so if i want to recolor it i can pull the initial conditions from a cache - -```bash -docker pull redis -# -p and --name must come before -d -docker run -p 6379:6379 --name my-redis -d redis -docker exec -it my-redis redis-cli - -set my-key "hello redis" -get my-key -``` - -```bash -pip install redis -pip install aioredis -``` - -Redis is cool but its overkill I can just use `from cachetools import TTLCache` -No thats not async enough im going to use `from aiocache import caches` - -Im now seeing in jit that it can do caching. But that wouldn't be helpful as my function does one iteration at a time and i just cache the resulting 10000000 iterations with the initial conditions. I also compress it with gzip so which is nice too. - -# Deploying -- I created a new GCP project geo-attractors. -- I enabled `Cloud Run Admin API has been enabled` by going to it and clicking create service -- Maybe i want it in Kubernetes :thinking: just with the prometheus and grafana. (maybe redis) -- Maybe i first get it in cloud run to show people then bring it to gke - -- starting with cloud run made some terraform and some startup scripts -had to send it to gcloud to build 1. on the m1 mac the build is strange and 2. can't build in the docker container -`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-backend"` -this will be solved when its just automatic with the ci/cd pipeline thats totally coming soon - - -- wow that was kinda super easy like i did it in under an hour service account artifact registry public cloud run and all (thanks copilot <3) - -- I never saved the terraform.tfstate - - `gsutils mb gs://geo-attractors-tf` - - added backend to main.tf then did a `terraform init` - - -# Issues +## Issues - Learn how to write tests - Terminal auto complete - Can't build or run Dockerfile in the container. @@ -73,8 +32,6 @@ docker run -v /var/run/docker.sock:/var/run/docker.sock -it your-dev-container-i docker run --name my_redis_container --network my_network -p 6379:6379 -d redis # i had to also add my_network to the build args of this in devcontainer.json ``` - -- Brew 🍺 would be nice - super strange git issue ``` git add src/ @@ -83,7 +40,25 @@ To add an exception for this directory, call: git config --global --add safe.directory /workspaces/attractorsIII ``` -- doesn't open properly in code-space got an error on creations will check later at the airport on my ipad. not sure if it was the my network or i had an extra comma that broke it probably the latter as the error log didn't say much about the build arg its very slow probably need to give it more compute. could it be that its using my wifi to download packages I don't think so. + + +# Deploying +- I created a new GCP project geo-attractors. +- I enabled `Cloud Run Admin API has been enabled` by going to it and clicking create service +- Maybe i want it in Kubernetes :thinking: just with the prometheus and grafana. (maybe redis) +- Maybe i first get it in cloud run to show people then bring it to gke + +- starting with cloud run made some terraform and some startup scripts +had to send it to gcloud to build 1. on the m1 mac the build is strange and 2. can't build in the docker container +`gcloud builds submit --tag "us-central1-docker.pkg.dev/geo-attractors/attractors/attractors-backend"` +this will be solved when its just automatic with the ci/cd pipeline thats totally coming soon + + +- wow that was kinda super easy like i did it in under an hour service account artifact registry public cloud run and all (thanks copilot <3) + +- I never saved the terraform.tfstate + - `gsutils mb gs://geo-attractors-tf` + - added backend to main.tf then did a `terraform init` # two services one repo diff --git a/geo-notes.md b/geo-notes.md new file mode 100644 index 0000000..e69de29 From 58c1845aa2041f3aa77dd5235d8732be6067d8a7 Mon Sep 17 00:00:00 2001 From: George Mazzeo Date: Tue, 16 Jan 2024 23:46:39 +0000 Subject: [PATCH 17/17] readme update --- .devcontainer/devcontainer.json | 1 - README.md | 4 ++-- geo-notes.md | 22 ++++++++++++++++++++++ scripts/alias.sh | 0 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 scripts/alias.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d48136f..a6a7937 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -29,7 +29,6 @@ ] } } - // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "root" } diff --git a/README.md b/README.md index 6a0cd84..b6a5baf 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ so im now in a clean 3.11 dev container :) I have a bash terminal Cool so i updated my pre-commit hooks as well Im running without a venv bc im already in a clean room. This is great now I can install all my dev dependencies and then make a clean .venv with only whats needed to deploy. + ## Issues -- Learn how to write tests -- Terminal auto complete +- How can i set the default shell to zsh? it is an option - Can't build or run Dockerfile in the container. possibly I could pass the local machines docker into the container ```bash diff --git a/geo-notes.md b/geo-notes.md index e69de29..18b7b13 100644 --- a/geo-notes.md +++ b/geo-notes.md @@ -0,0 +1,22 @@ +# Redis +Idea to add some sort of caching for the attractors so if i want to recolor it i can pull the initial conditions from a cache + +```bash +docker pull redis +# -p and --name must come before -d +docker run -p 6379:6379 --name my-redis -d redis +docker exec -it my-redis redis-cli + +set my-key "hello redis" +get my-key +``` + +```bash +pip install redis +pip install aioredis +``` + +Redis is cool but its overkill I can just use `from cachetools import TTLCache` +No thats not async enough im going to use `from aiocache import caches` + +Im now seeing in jit that it can do caching. But that wouldn't be helpful as my function does one iteration at a time and i just cache the resulting 10000000 iterations with the initial conditions. I also compress it with gzip so which is nice too. diff --git a/scripts/alias.sh b/scripts/alias.sh new file mode 100644 index 0000000..e69de29