Build container + deploy to k8s

.github/workflows/build-deploy.yml

@@ -0,0 +1,87 @@
+name: Deploy & Configure Application
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+  schedule:
+    - cron: '0 10 * * 1'
+
+permissions:
+  id-token: write
+  contents: read
+  packages: write
+
+concurrency:
+  group: "build-deploy"
+
+jobs:
+  package:
+    uses: Chia-Network/actions/.github/workflows/docker-build.yaml@main
+
+  deploy:
+    name: Deploy
+    runs-on: "k8s-internal-${{ matrix.colo.region }}"
+    needs:
+      - package
+    container:
+      image: registry.gitlab.com/cmmarslender/kubectl-helm:v3
+    strategy:
+      fail-fast: false
+      matrix:
+        colo:
+          - region: fmt
+          - region: msp
+          - region: ldn
+          - region: sin
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Vault Login
+        uses: Chia-Network/actions/vault/login@main
+        with:
+          vault_url: ${{ secrets.VAULT_URL }}
+          role_name: github
+
+      - name: Get secrets from vault
+        uses: hashicorp/vault-action@v3
+        with:
+          url: ${{ secrets.VAULT_URL }}
+          token: ${{ env.VAULT_TOKEN }}
+          secrets: |
+            secret/data/${{ matrix.colo.region }}/k8s/k8s-${{ matrix.colo.region }} api_server_url | K8S_API_SERVER_URL;
+            secret/data/github/ghcr_image_pull username | IMAGE_PULL_USERNAME;
+            secret/data/github/ghcr_image_pull password | IMAGE_PULL_PASSWORD;
+
+      - name: Login to k8s cluster
+        uses: Chia-Network/actions/vault/k8s-login@main
+        with:
+          vault_url: ${{ secrets.VAULT_URL }}
+          vault_token: ${{ env.VAULT_TOKEN }}
+          backend_name: k8s-${{ matrix.colo.region }}
+          role_name: github-actions
+          cluster_url: ${{ env.K8S_API_SERVER_URL }}
+
+      - uses: Chia-Network/actions/k8s/image-pull-secret@main
+        with:
+          secret_name: crawler-image-pull
+          namespace: go-chia-crawler
+          username: ${{ env.IMAGE_PULL_USERNAME }}
+          password: ${{ env.IMAGE_PULL_PASSWORD }}
+          docker_server: "ghcr.io"
+
+      - name: Ensure istio injection is enabled
+        run: |
+          kubectl label namespace go-chia-crawler istio-injection=enabled
+
+      - uses: Chia-Network/actions/helm/deploy@main
+        env:
+          DOCKER_TAG: "sha-${{ inputs.deploy_ref }}"
+          REGION: ${{ matrix.colo.region }}
+        with:
+          namespace: go-chia-crawler
+          app_name: "crawler"
+          helm_chart_repo: "https://chia-network.github.io/helm-charts"
+          helm_chart: "generic"
+          helm_values: "./k8s/values.yaml"

cmd/crawl.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path"
 	"sync"
 	"time"
 
@@ -200,7 +201,7 @@ func persist() {
 	log.Println("Persisting peers to crawler.dat")
 	// Iterate through the hostTimestamps, and for each host that meets the reporting threshold, save all the details about it
 
-	file, err := os.Create("crawler.dat")
+	file, err := os.Create(path.Join(viper.GetString("data-dir"), "crawler.dat"))
 	if err != nil {
 		log.Printf("Error writing data: %s\n", err.Error())
 		return
@@ -236,7 +237,7 @@ func persist() {
 
 func load() {
 	log.Println("Checking for peers in crawler.dat")
-	file, err := os.Open("crawler.dat")
+	file, err := os.Open(path.Join(viper.GetString("data-dir"), "crawler.dat"))
 	if err != nil {
 		log.Printf("Error opening data file: %s\n", err.Error())
 		return

cmd/root.go

@@ -27,21 +27,17 @@ func Execute() {
 }
 
 func init() {
-	var (
-		bootstrapPeer string
-		metrics       bool
-		metricsPort   int
-	)
-
 	cobra.OnInitialize(initConfig)
 
-	rootCmd.PersistentFlags().StringVar(&bootstrapPeer, "bootstrap-peer", "node.chia.net", "The initial bootstrap peer to try and connect to")
-	rootCmd.PersistentFlags().BoolVar(&metrics, "metrics", false, "Enable the metrics server")
-	rootCmd.PersistentFlags().IntVar(&metricsPort, "metrics-port", 9914, "The port the metrics server binds to")
+	rootCmd.PersistentFlags().String("bootstrap-peer", "node.chia.net", "The initial bootstrap peer to try and connect to")
+	rootCmd.PersistentFlags().Bool("metrics", false, "Enable the metrics server")
+	rootCmd.PersistentFlags().Int("metrics-port", 9914, "The port the metrics server binds to")
+	rootCmd.PersistentFlags().String("data-dir", "", "The directory to store crawler data in")
 
 	cobra.CheckErr(viper.BindPFlag("bootstrap-peer", rootCmd.PersistentFlags().Lookup("bootstrap-peer")))
 	cobra.CheckErr(viper.BindPFlag("metrics", rootCmd.PersistentFlags().Lookup("metrics")))
 	cobra.CheckErr(viper.BindPFlag("metrics-port", rootCmd.PersistentFlags().Lookup("metrics-port")))
+	cobra.CheckErr(viper.BindPFlag("data-dir", rootCmd.PersistentFlags().Lookup("data-dir")))
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.chia-crawler.yaml)")
 }

k8s/values.yaml.j2

@@ -0,0 +1,43 @@
+image:
+  repository: ghcr.io/chia-network/go-chia-crawler
+  tag: {{ DOCKER_TAG }}
+
+imagePullSecrets:
+  - name: crawler-image-pull
+
+deployment:
+  containerPort: 9914
+  containerPortName: metrics
+
+service:
+  enabled: true
+  type: ClusterIP
+  additionalLabels:
+    application: chia-blockchain
+    component: dns-introducer
+    network: mainnet
+  port: 9914
+
+networkPolicy:
+  enabled: true
+  policyTypes:
+    - Egress
+  egressRules:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              name: chia-blockchain-mainnet
+      ports:
+        - protocol: TCP
+          port: 8444
+
+configmapEnvironment:
+  CHIA_CRAWLER_BOOTSTRAP_PEER: node-node-all.chia-blockchain-mainnet
+  CHIA_CRAWLER_DATA_DIR: /crawler-data
+
+volumes:
+  - name: crawler-data
+    mountPath: /crawler-data
+    readOnly: false
+    size: 256Mi
+    storageClassName: ceph-nvme