diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..86ca2a9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# Before the docker CLI sends the context to the docker daemon, it looks for a file +# named .dockerignore in the root directory of the context. If this file exists, the +# CLI modifies the context to exclude files and directories that match patterns in it. +# +# You may want to specify which files to include in the context, rather than which +# to exclude. To achieve this, specify * as the first pattern, followed by one or +# more ! exception patterns. +# +# https://docs.docker.com/engine/reference/builder/#dockerignore-file + +# Exclude everything: +# +* + +# Now un-exclude required files and folders: +# +!coredns diff --git a/.github/workflows/cd-deploy-to-dev.yml b/.github/workflows/cd-deploy-to-dev.yml new file mode 100644 index 0000000..698422e --- /dev/null +++ b/.github/workflows/cd-deploy-to-dev.yml @@ -0,0 +1,48 @@ +name: Deploy to dev + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + paths: + - Corefile + - Dockerfile + - .github/workflows/cd-deploy-to-dev.yml + - .github/workflows/sub-cloudrun-deploy.yml + +concurrency: + # Ensures that only one workflow task will run at a time. Previous builds, if + # already in process, will get cancelled. Only the latest commit will be allowed + # to run, cancelling any workflows in between + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # TODO: Add a job to run unit tests + # test: + # uses: ./.github/workflows/sub-unit-tests.yml + + build: + uses: ./.github/workflows/sub-build-docker-image.yml + with: + environment: dev + dockerfile_path: ./docker/Dockerfile + dockerfile_target: runner + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + secrets: inherit + + deploy: + needs: [build] + uses: ./.github/workflows/sub-cloudrun-deploy.yml + with: + environment: dev + project_id: ${{ vars.GCP_PROJECT }} + region: ${{ vars.GCP_REGION }} + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + image_digest: ${{ needs.build.outputs.image_digest }} + min_instances: '0' + max_instances: '30' + cpu: '1' + memory: 1Gi + secrets: inherit diff --git a/.github/workflows/cd-deploy-to-prod.yml b/.github/workflows/cd-deploy-to-prod.yml new file mode 100644 index 0000000..03ae155 --- /dev/null +++ b/.github/workflows/cd-deploy-to-prod.yml @@ -0,0 +1,46 @@ +name: Deploy to prod + +on: + release: + types: + - published + +concurrency: + # Ensures that only one workflow task will run at a time. Previous builds, if + # already in process, will get cancelled. Only the latest commit will be allowed + # to run, cancelling any workflows in between + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # TODO: Add a job to run unit tests + # test: + # uses: ./.github/workflows/sub-unit-tests.yml + + + build: + # needs: [test] + uses: ./.github/workflows/sub-build-docker-image.yml + with: + environment: prod + dockerfile_path: ./docker/Dockerfile + dockerfile_target: runner + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + secrets: inherit + + deploy: + needs: [build] + uses: ./.github/workflows/sub-cloudrun-deploy.yml + with: + environment: prod + project_id: ${{ vars.GCP_PROJECT }} + region: ${{ vars.GCP_REGION }} + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + image_digest: ${{ needs.build.outputs.image_digest }} + min_instances: '1' + max_instances: '100' + cpu: '1' + memory: 1Gi + secrets: inherit diff --git a/.github/workflows/cd-deploy-to-test.yml b/.github/workflows/cd-deploy-to-test.yml new file mode 100644 index 0000000..6e9ea7b --- /dev/null +++ b/.github/workflows/cd-deploy-to-test.yml @@ -0,0 +1,50 @@ +name: Deploy to test + +on: + push: + branches: + - main + paths: + - Corefile + - Dockerfile + - .github/workflows/cd-deploy-to-test.yml + - .github/workflows/sub-cloudrun-deploy.yml + +concurrency: + # Ensures that only one workflow task will run at a time. Previous builds, if + # already in process, will get cancelled. Only the latest commit will be allowed + # to run, cancelling any workflows in between + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # TODO: Add a job to run unit tests + # test: + # uses: ./.github/workflows/sub-unit-tests.yml + + build: + # needs: [test] + uses: ./.github/workflows/sub-build-docker-image.yml + with: + environment: test + dockerfile_path: ./docker/Dockerfile + dockerfile_target: runner + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + secrets: inherit + + deploy: + needs: [build] + uses: ./.github/workflows/sub-cloudrun-deploy.yml + with: + environment: test + project_id: ${{ vars.GCP_PROJECT }} + region: ${{ vars.GCP_REGION }} + app_name: ${{ vars.APP_NAME }} + registry: ${{ vars.GAR_BASE }} + image_digest: ${{ needs.build.outputs.image_digest }} + min_instances: '0' + max_instances: '30' + cpu: '1' + memory: 1Gi + secrets: inherit diff --git a/.github/workflows/chore-clean-dev.yml b/.github/workflows/chore-clean-dev.yml new file mode 100644 index 0000000..d476d14 --- /dev/null +++ b/.github/workflows/chore-clean-dev.yml @@ -0,0 +1,30 @@ +name: Clean dev instances + +on: + delete: + pull_request: + branches: + - main + types: + - closed + +jobs: + delete: + runs-on: ubuntu-latest + steps: + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2.1.2 + with: + workload_identity_provider: '${{ vars.GCP_WIF }}' + service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1.1.1 + + - name: Removing CR service + run: | + gcloud run services delete ${{ vars.APP_NAME }}-${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }} --region=${{ vars.GOOGLE_CLOUD_REGION }} --quiet diff --git a/.github/workflows/ci-lint-codebase.patch.yml b/.github/workflows/ci-lint-codebase.patch.yml new file mode 100644 index 0000000..89a8892 --- /dev/null +++ b/.github/workflows/ci-lint-codebase.patch.yml @@ -0,0 +1,15 @@ +name: Lint Code Base + +on: + pull_request: + branches: [main] + paths-ignore: + - Corefile + - Dockerfile + - .github/workflows/ci-lint-codebase.yml + +jobs: + linter: + runs-on: ubuntu-latest + steps: + - run: echo "Job not required" diff --git a/.github/workflows/ci-lint-codebase.yaml b/.github/workflows/ci-lint-codebase.yaml new file mode 100644 index 0000000..c0780b4 --- /dev/null +++ b/.github/workflows/ci-lint-codebase.yaml @@ -0,0 +1,54 @@ +name: Lint Code Base + +on: + pull_request: + branches: [main] + paths: + - '**.js*' + - '**.ts*' + - Dockerfile + - package.json + - pnpm-lock.yaml + - .github/workflows/ci-lint-codebase.yml + + push: + branches: [main] + paths: + - Corefile + - Dockerfile + - .github/workflows/ci-lint-codebase.yml + +concurrency: + # Ensures that only one workflow task will run at a time. Previous builds, if + # already in process, will get cancelled. Only the latest commit will be allowed + # to run, cancelling any workflows in between + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + linter: + runs-on: ubuntu-latest + steps: + - name: Checkout Code Repository + uses: actions/checkout@v4.1.1 + with: + # Full git history is needed to get a proper + # list of changed files within `super-linter` + fetch-depth: 0 + + - name: Lint Code Base + uses: super-linter/super-linter/slim@v5.2.1 + env: + LOG_LEVEL: ERROR + VALIDATE_ALL_CODEBASE: false + VALIDATE_SHELL_SHFMT: false + VALIDATE_JSCPD: false + VALIDATE_CSS: false + VALIDATE_EDITORCONFIG: false + VALIDATE_MARKDOWN: false + VALIDATE_DOCKERFILE_HADOLINT: false + LINTER_RULES_PATH: / + JAVASCRIPT_DEFAULT_STYLE: prettier + TYPESCRIPT_DEFAULT_STYLE: prettier + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/gcloud.yml b/.github/workflows/gcloud.yml deleted file mode 100644 index 72957e2..0000000 --- a/.github/workflows/gcloud.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: gcloud - -on: - workflow_dispatch: - push: - branches: - - master - - update-gcloud - -env: - PROJECT_ID: ecosystem-infrastructure - GCLOUD_ZONE: us-central1-a - -jobs: - build-deploy: - name: Google Cloud Build - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set project and image names - run: | - BRANCH_NAME=$GITHUB_REPOSITORY/$(expr $GITHUB_REF : '.*/\(.*\)') && \ - BRANCH_NAME=${BRANCH_NAME,,} && \ - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV && \ - echo "SHORT_SHA=$(git rev-parse --short=7 $GITHUB_SHA)" >> $GITHUB_ENV - - # Setup gcloud CLI - - uses: google-github-actions/setup-gcloud@v2 - with: - version: '295.0.0' - project_id: ${{ env.PROJECT_ID }} - service_account_key: ${{ secrets.GCLOUD_SA_KEY }} - - # Build and push image to Google Container Registry - - name: Build - # Tagging w/ the commit SHA blocks the :latest tag on GCR - run: | - gcloud builds submit \ - --tag "gcr.io/$PROJECT_ID/$BRANCH_NAME:$SHORT_SHA" - - # Deploy image to Compute Engine - - name: Deploy - run: | - gcloud compute instances create-with-container "zfnd-seeder-$SHORT_SHA" \ - --zone "$GCLOUD_ZONE" \ - --service-account instance-service-account@ecosystem-infrastructure.iam.gserviceaccount.com \ - --scopes cloud-platform \ - --machine-type n1-highcpu-4 \ - --container-image "gcr.io/$PROJECT_ID/$BRANCH_NAME:$SHORT_SHA" \ - --tags seeder \ - --metadata-from-file startup-script=scripts/seeder-zfnd-org-startup.sh \ - --container-mount-host-path mount-path=/etc/dnsseeder,host-path=/etc/dnsseeder,mode=ro diff --git a/.github/workflows/sub-build-docker-image.yml b/.github/workflows/sub-build-docker-image.yml new file mode 100644 index 0000000..555547c --- /dev/null +++ b/.github/workflows/sub-build-docker-image.yml @@ -0,0 +1,117 @@ +name: Build docker image + +on: + workflow_call: + inputs: + app_name: + required: true + type: string + dockerfile_path: + required: true + type: string + dockerfile_target: + required: true + type: string + registry: + required: true + type: string + environment: + required: true + type: string + outputs: + image_digest: + description: The image digest to be used on a caller workflow + value: ${{ jobs.build.outputs.image_digest }} + +jobs: + build: + name: Build images + timeout-minutes: 15 + runs-on: ubuntu-latest + outputs: + image_digest: ${{ steps.docker_build.outputs.digest }} + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4.1.1 + with: + persist-credentials: false + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + with: + short-length: 7 + + # Automatic tag management and OCI Image Format Specification for labels + - name: Docker meta + id: meta + uses: docker/metadata-action@v5.5.0 + with: + # list of Docker images to use as base name for tags + images: | + ${{ inputs.registry }}/${{ inputs.app_name }} + # generate Docker tags based on the following events/attributes + tags: | + type=schedule + # semver and ref,tag automatically add a "latest" tag, but only on stable releases + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=ref,event=tag + type=ref,event=branch + type=ref,event=pr + type=sha + # edge is the latest commit on the default branch. + type=edge,enable={{is_default_branch}} + + # Setup Docker Buildx to allow use of docker cache layers from GH + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3.0.0 + + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2.1.2 + with: + workload_identity_provider: '${{ vars.GCP_WIF }}' + service_account: '${{ vars.GCP_ARTIFACTS_SA }}' + token_format: access_token + # Some builds might take over an hour, and Google's default lifetime duration for + # an access token is 1 hour (3600s). We increase this to 3 hours (10800s) + # as some builds take over an hour. + access_token_lifetime: 10800s + + - name: Login to Google Artifact Registry + uses: docker/login-action@v3.0.0 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + # Build and push image to Google Artifact Registry, and possibly DockerHub + - name: Build & push + id: docker_build + uses: docker/build-push-action@v5.1.0 + with: + target: ${{ inputs.dockerfile_target }} + context: . + file: ${{ inputs.dockerfile_path }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true + # To improve build speeds, for each branch we push an additional image to the registry, + # to be used as the caching layer, using the `max` caching mode. + # + # We use multiple cache sources to confirm a cache hit, starting from a per-branch cache, + # and if there's no hit, then continue with the `main` branch. When changes are added to a PR, + # they are usually smaller than the diff between the PR and `main` branch. So this provides the + # best performance. + # + # The caches are tried in top-down order, the first available cache is used: + # https://github.com/moby/moby/pull/26839#issuecomment-277383550 + cache-from: | + type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name }}:${{ env.GITHUB_REF_SLUG_URL }}-cache + type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name }}:${{ github.event.repository.default_branch }}-cache + cache-to: | + type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name }}:${{ env.GITHUB_REF_SLUG_URL }}-cache,mode=min diff --git a/.github/workflows/sub-cloudrun-deploy.yml b/.github/workflows/sub-cloudrun-deploy.yml new file mode 100644 index 0000000..c21eb6a --- /dev/null +++ b/.github/workflows/sub-cloudrun-deploy.yml @@ -0,0 +1,112 @@ +name: Deploy to Cloud Run + +on: + workflow_call: + inputs: + app_name: + required: true + type: string + registry: + required: true + type: string + image_digest: + required: true + type: string + description: The image digest to deploy + project_id: + required: true + type: string + description: The project to deploy to + region: + required: true + type: string + description: The region to deploy to + environment: + required: false + type: string + description: The environment to deploy to + min_instances: + required: false + type: string + description: The minimum number of instances to deploy + max_instances: + required: false + type: string + description: The maximum number of instances to deploy + cpu: + required: false + type: string + description: The number of CPUs to use for the service + memory: + required: false + type: string + description: The amount of memory to use for the service + +jobs: + versioning: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.set.outputs.version }} + steps: + - name: Getting API Version + id: get + uses: actions/github-script@v7 + if: ${{ github.event_name == 'release' }} + with: + result-encoding: string + script: | + return context.payload.release.tag_name.substring(0,2) + - name: Setting API Version + id: set + run: echo "version=${{ steps.get.outputs.result }}" >> "$GITHUB_OUTPUT" + + deploy: + name: Deploy to Cloud Run + needs: [versioning] + timeout-minutes: 10 + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + url: ${{ steps.deploy.outputs.url }} + permissions: + contents: read + id-token: write + steps: + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2.1.2 + with: + workload_identity_provider: '${{ vars.GCP_WIF }}' + service_account: '${{ vars.GCP_DEPLOYMENTS_SA }}' + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2.1.0 + + - name: Deploy to cloud run + id: deploy + uses: google-github-actions/deploy-cloudrun@v2.2.0 + with: + service: ${{ inputs.app_name }}-${{ needs.versioning.outputs.version || env.GITHUB_HEAD_REF_SLUG || inputs.environment }} + image: ${{ inputs.registry }}/${{ inputs.app_name }}@${{ inputs.image_digest }} + region: ${{ inputs.region }} + gcloud_component: alpha + # env_vars: | + # secrets: | + flags: | + --min-instances=${{ inputs.min_instances }} + --max-instances=${{ inputs.max_instances }} + --cpu=${{ inputs.cpu }} + --memory=${{ inputs.memory }} + --network=projects/zfnd-dev-net-spoke-0/global/networks/dev-spoke-0 + --subnet=projects/zfnd-dev-net-spoke-0/regions/us-east1/subnetworks/dev-default-ue1 + + - name: Allow unauthenticated calls to the service + run: | + gcloud run services add-iam-policy-binding ${{ inputs.app_name }}-${{ needs.versioning.outputs.version || env.GITHUB_HEAD_REF_SLUG || inputs.environment }} \ + --region=${{ inputs.region }} --member=allUsers --role=roles/run.invoker --quiet + + - name: Test service with cURL + run: curl "${{ steps.deploy.outputs.url }}" diff --git a/.github/workflows/sub-unit-tests.yml b/.github/workflows/sub-unit-tests.yml new file mode 100644 index 0000000..9fafdf6 --- /dev/null +++ b/.github/workflows/sub-unit-tests.yml @@ -0,0 +1,37 @@ +name: Unit Tests + +on: + workflow_call: + +concurrency: + # Ensures that only one workflow task will run at a time. Previous builds, if + # already in process, will get cancelled. Only the latest commit will be allowed + # to run, cancelling any workflows in between + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + name: Test with Go + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout Code Repository + uses: actions/checkout@v4.1.1 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: 1.21.x + + - name: Install dependencies + run: | + echo "dnsseed:github.com/zcashfoundation/dnsseeder/dnsseed" >> plugin.cfg + echo "replace github.com/btcsuite/btcd => github.com/gtank/btcd v0.0.0-20191012142736-b43c61a68604" >> go.mod + go get github.com/zcashfoundation/dnsseeder/dnsseed@v0.2.4-beta + + - name: Build + run: go build -v ./... + + - name: Test with the Go CLI + run: go test diff --git a/.gitignore b/.gitignore index b25c15b..1ccd1f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,140 @@ +# Created by https://www.toptal.com/developers/gitignore/api/go,windows,macos,linux,vim,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=go,windows,macos,linux,vim,visualstudiocode + +### Go ### +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +### Linux ### *~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/go,windows,macos,linux,vim,visualstudiocode diff --git a/Dockerfile b/docker/Dockerfile similarity index 100% rename from Dockerfile rename to docker/Dockerfile diff --git a/scripts/seeder-zfnd-org-startup.sh b/scripts/seeder-zfnd-org-startup.sh deleted file mode 100644 index 9cf456a..0000000 --- a/scripts/seeder-zfnd-org-startup.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -systemctl stop systemd-resolved - -if [ ! -d /etc/dnsseeder ]; then - mkdir -p /etc/dnsseeder -fi - -cat < /etc/dnsseeder/Corefile -mainnet.seeder.zfnd.org { - dnsseed { - network mainnet - bootstrap_peers mainnet.z.cash:8233 dnsseed.str4d.xyz:8233 mainnet.is.yolo.money:8233 mainnet.seeder.zfnd.org:8233 - crawl_interval 30m - record_ttl 600 - } -} - -testnet.seeder.zfnd.org { - dnsseed { - network testnet - bootstrap_peers testnet.z.cash:18233 testnet.is.yolo.money:18233 testnet.seeder.zfnd.org:18233 - crawl_interval 15m - record_ttl 300 - } -} -EOF