diff --git a/.dockerignore b/.dockerignore index 4d0d10c087..7032d8f7fb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,4 @@ **/*.sw* frontend/dist **/.DS_Store +**/target diff --git a/Dockerfile.controller b/Dockerfile.controller index 057ba38f42..8867aa0571 100644 --- a/Dockerfile.controller +++ b/Dockerfile.controller @@ -17,7 +17,6 @@ RUN node --version COPY go.mod go.sum ./ RUN go mod download -x - # Build COPY . /src/ RUN just errtrace diff --git a/backend/controller/controller.go b/backend/controller/controller.go index 45c68ca4a9..29c2935d2c 100644 --- a/backend/controller/controller.go +++ b/backend/controller/controller.go @@ -139,6 +139,7 @@ func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScali return err } logger.Debugf("Listening on %s", config.Bind) + logger.Debugf("Advertising as %s", config.Advertise) cm := cf.ConfigFromContext(ctx) sm := cf.SecretsFromContext(ctx) diff --git a/deployment/Justfile b/deployment/Justfile index 4ad0d8caaf..aae63a660b 100755 --- a/deployment/Justfile +++ b/deployment/Justfile @@ -1,11 +1,56 @@ _help: @just -l +registry_full := "k3d-ftl-registry.localhost:5000" +registry_short := "ftl-registry.localhost" +registry_full_nolocal := "k3d-ftl-registry:5000" +registry_local := "localhost:5000" +git_tag_hash := `git rev-parse --short HEAD` + +mirrors := ''' + mirrors: + "ftl:5000": + endpoint: + - "http://''' + registry_full + '''" +''' + +start: setup full-deploy + +rm: teardown + +full-deploy: + just build-controller + just build-runner + just deploy + kubectl wait --for=condition=available deployment/ftl-controller --timeout=5m + kubectl wait --for=condition=available deployment/ftl-runner --timeout=5m + kubectl wait --for=condition=ready pod/ftl-pg-cluster-1-0 --timeout=5m + + ftl status + +setup-registry: + k3d registry create {{registry_short}} --port 5000 + +setup-cluster: + k3d cluster create ftl --api-port 6550 -p "8892:80@loadbalancer" --agents 2 \ + --registry-use {{registry_full}} \ + --registry-config '{{mirrors}}' + +setup: setup-registry setup-cluster + +teardown-registry: + k3d registry delete {{registry_short}} + +teardown-cluster: + k3d cluster delete ftl + +teardown: teardown-cluster teardown-registry + deploy: - kubectl kustomize --load-restrictor=LoadRestrictionsNone | kubectl apply -f - + kubectl kustomize --load-restrictor=LoadRestrictionsNone overlays/dev | kubectl apply -f - delete: - kubectl kustomize --load-restrictor=LoadRestrictionsNone | kubectl delete -f - + kubectl kustomize --load-restrictor=LoadRestrictionsNone overlays/dev | kubectl delete -f - delete-db: kubectl delete pvc postgres-db-ftl-pg-cluster-1-0 @@ -30,3 +75,15 @@ psql: ftl-status: just exec deployment/ftl-controller ./ftl status + +build-controller: + cd .. && docker build --platform linux/amd64 -t ftl-controller:latest -f Dockerfile.controller . + docker tag ftl-controller:latest {{registry_local}}/ftl-controller:latest + docker push {{registry_local}}/ftl-controller:latest + +build-runner: + cd .. && docker build --platform linux/amd64 -t ftl-runner:latest -f Dockerfile.runner . + docker tag ftl0/ftl-runner:latest {{registry_local}}/ftl-runner:latest + docker push {{registry_local}}/ftl-runner:latest + +build: build-controller build-runner diff --git a/deployment/README.md b/deployment/README.md index bd89e2b220..942fd3cf0c 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -1,73 +1,188 @@ # Deploy ftl-controller to k3d -## Create a k3d cluster with a local Docker registry +This is a kubernetes environment that runs under k3d for testing purposes, and can also be used as a basis for deployments. -``` -# -k3d registry create registry.localhost --port 5000 -k3d cluster create --api-port 6550 -p "8892:80@loadbalancer" --agents 2 \ - --registry-use k3d-registry.localhost:5000 \ - --registry-config <(cat < +pod/ftl-runner-79b546fb4d-jb242 1/1 Running 0 2m19s 10.42.2.8 k3d-ftl-server-0 +pod/ftl-runner-79b546fb4d-96fk9 1/1 Running 0 2m18s 10.42.2.9 k3d-ftl-server-0 +pod/ftl-runner-79b546fb4d-h85ws 1/1 Running 0 2m19s 10.42.1.5 k3d-ftl-agent-1 +pod/ftl-runner-79b546fb4d-hb4zq 1/1 Running 0 2m19s 10.42.1.7 k3d-ftl-agent-1 +pod/ftl-runner-79b546fb4d-l852m 1/1 Running 0 2m18s 10.42.1.8 k3d-ftl-agent-1 +pod/ftl-runner-79b546fb4d-9qb7h 1/1 Running 0 2m19s 10.42.0.9 k3d-ftl-agent-0 +pod/ftl-runner-79b546fb4d-rtzw9 1/1 Running 0 2m19s 10.42.0.8 k3d-ftl-agent-0 +pod/ftl-runner-79b546fb4d-xjsm9 1/1 Running 0 2m18s 10.42.0.10 k3d-ftl-agent-0 +pod/ftl-runner-79b546fb4d-gr5h4 1/1 Running 0 2m19s 10.42.0.6 k3d-ftl-agent-0 +pod/localstack-57b975d597-lj6vl 1/1 Running 0 2m19s 10.42.1.6 k3d-ftl-agent-1 +pod/ftl-pg-cluster-1-0 1/1 Running 0 111s 10.42.2.11 k3d-ftl-server-0 +pod/ftl-db-migrate-n8h2f 0/1 Completed 3 2m19s 10.42.2.6 k3d-ftl-server-0 +pod/ftl-controller-7f8b5f5785-xvxlm 1/1 Running 4 (84s ago) 2m19s 10.42.0.7 k3d-ftl-agent-0 +pod/ftl-controller-7f8b5f5785-wnj74 1/1 Running 4 (92s ago) 2m19s 10.42.2.5 k3d-ftl-server-0 + +NAME READY AGE CONTAINERS IMAGES +statefulset.apps/ftl-pg-cluster-1 1/1 111s ftl-pg-cluster-1 postgres:14.1 + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR +service/kubernetes ClusterIP 10.43.0.1 443/TCP 7m17s +service/ftl-controller ClusterIP 10.43.49.8 8891/TCP,8892/TCP 2m19s app=ftl-controller +service/localstack ClusterIP 10.43.231.229 4566/TCP 2m19s app=localstack +service/ftl-pg-cluster ClusterIP None 5432/TCP 79s app=ftl-pg-cluster,replicationRole=primary + +NAME DATA AGE +configmap/kube-root-ca.crt 1 7m2s +configmap/ftl-pg-cluster-conf 1 2m19s +configmap/ftl-db-migrate-config-h4fmggb56d 2 2m19s +configmap/base-kubegres-config 7 112s + +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE VOLUMEMODE +persistentvolume/pvc-f26addf6-c4d3-487b-a289-acdc97b73a32 200Mi RWO Delete Bound default/postgres-db-ftl-pg-cluster-1-0 local-path 101s Filesystem + +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE VOLUMEMODE +persistentvolumeclaim/postgres-db-ftl-pg-cluster-1-0 Bound pvc-f26addf6-c4d3-487b-a289-acdc97b73a32 200Mi RWO local-path 111s Filesystem + +NAME CLASS HOSTS ADDRESS PORTS AGE +ingress.networking.k8s.io/ftl-controller traefik * 192.168.247.3,192.168.247.4,192.168.247.5 80 2m17s ``` -Exec into the PG cluster: +View logs with `just logs `: ``` -kubectl exec -it ftl-pg-cluster-1-0 -- /bin/bash +just logs pod/ftl-controller-7f8b5f5785-xvxlm + +kubectl logs -f pod/ftl-controller-7f8b5f5785-xvxlm +debug: Starting FTL controller +info: Web console available at: http://10.42.0.7:8892 +debug: Listening on http://10.42.0.7:8892 +debug: Advertising as http://10.42.0.7:8892 +info: HTTP ingress server listening on: http://10.42.0.7:8891 +debug: new leader for /system/asm: http://10.42.0.7:8892 +debug:lease:/system/asm: Acquired lease +debug: Seeded 0 deployments +debug:lease:/system/scheduledtask/reconcileRunners: Acquired lease +debug:lease:/system/scheduledtask/reconcileDeployments: Acquired lease +debug:lease:/system/scheduledtask/reapStaleRunners: Acquired lease +debug:lease:/system/scheduledtask/releaseExpiredReservations: Acquired lease ``` -Create a one-shot shell pod: +You can access the DB with `just psql`: ``` -kubectl run -it --rm --restart=Never --image ubuntu:22.04 tempshell -- bash +just psql + +just enter statefulset.apps/ftl-pg-cluster-1 env PGPASSWORD=secret psql -U postgres ftl +kubectl exec -it statefulset.apps/ftl-pg-cluster-1 -- env PGPASSWORD=secret psql -U postgres ftl +psql (14.1 (Debian 14.1-1.pgdg110+1)) +Type "help" for help. + +ftl=# \d + List of relations + Schema | Name | Type | Owner +--------+-----------------------------+----------+---------- + public | artefacts | table | postgres + public | artefacts_id_seq | sequence | postgres + public | async_calls | table | postgres + +... ``` -List all the things: +Or shell into a pod with `just enter `: ``` -kubectl get deployment,pod,statefulset,svc,configmap,pv,pvc,ingress -o wide +just enter pod/ftl-controller-7f8b5f5785-xvxlm + +kubectl exec -it pod/ftl-controller-7f8b5f5785-xvxlm -- bash +root@ftl-controller-7f8b5f5785-xvxlm:~# ps aux +USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND +root 1 108 0.6 1686216 51600 ? Ssl 00:51 4:18 [rosetta] /root/ftl-controller /root/ftl-controller +root 30 3.8 0.0 418324 7424 pts/1 Ss 00:55 0:00 [rosetta] /usr/bin/bash bash +root 38 100 0.0 421360 6272 pts/1 R+ 00:55 0:00 ps aux +root@ftl-controller-7f8b5f5785-xvxlm:~# +``` + +Create a one-shot shell pod: + +``` +kubectl run -it --rm --restart=Never --image ubuntu:22.04 tempshell -- bash ``` diff --git a/deployment/db-create/kustomization.yml b/deployment/base/db-create/kustomization.yml similarity index 53% rename from deployment/db-create/kustomization.yml rename to deployment/base/db-create/kustomization.yml index 6cfb278e2a..85b1e81fae 100644 --- a/deployment/db-create/kustomization.yml +++ b/deployment/base/db-create/kustomization.yml @@ -1,3 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: - - https://raw.githubusercontent.com/reactive-tech/kubegres/v1.16/kubegres.yaml + - https://raw.githubusercontent.com/reactive-tech/kubegres/v1.18/kubegres.yaml - pg-cluster.yaml diff --git a/deployment/db-create/pg-cluster.yaml b/deployment/base/db-create/pg-cluster.yaml similarity index 100% rename from deployment/db-create/pg-cluster.yaml rename to deployment/base/db-create/pg-cluster.yaml diff --git a/deployment/db-migrate/db-migrate.yml b/deployment/base/db-migrate/db-migrate.yml similarity index 100% rename from deployment/db-migrate/db-migrate.yml rename to deployment/base/db-migrate/db-migrate.yml diff --git a/deployment/db-migrate/kustomization.yml b/deployment/base/db-migrate/kustomization.yml similarity index 73% rename from deployment/db-migrate/kustomization.yml rename to deployment/base/db-migrate/kustomization.yml index 039e222c61..7f0d8eebe0 100644 --- a/deployment/db-migrate/kustomization.yml +++ b/deployment/base/db-migrate/kustomization.yml @@ -1,3 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: - db-migrate.yml configMapGenerator: diff --git a/deployment/base/db-migrate/schema b/deployment/base/db-migrate/schema new file mode 120000 index 0000000000..45985bc500 --- /dev/null +++ b/deployment/base/db-migrate/schema @@ -0,0 +1 @@ +../../../backend/controller/sql/schema \ No newline at end of file diff --git a/deployment/ftl-controller/ftl-controller.yml b/deployment/base/ftl-controller/ftl-controller.yml similarity index 69% rename from deployment/ftl-controller/ftl-controller.yml rename to deployment/base/ftl-controller/ftl-controller.yml index e2813092f6..396b82f5a6 100644 --- a/deployment/ftl-controller/ftl-controller.yml +++ b/deployment/base/ftl-controller/ftl-controller.yml @@ -22,6 +22,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: LOG_LEVEL + value: "debug" - name: FTL_CONTROLLER_DSN value: "postgres://ftl-pg-cluster/ftl?sslmode=disable&user=postgres&password=secret" - name: FTL_CONTROLLER_INGRESS_BIND @@ -29,7 +31,15 @@ spec: - name: FTL_CONTROLLER_BIND value: "http://$(MY_POD_IP):8892" - name: FTL_CONTROLLER_ADVERTISE - value: "" + value: "http://$(MY_POD_IP):8892" + - name: AWS_REGION + value: "us-west-2" + - name: AWS_ACCESS_KEY_ID + value: "test" + - name: AWS_SECRET_ACCESS_KEY + value: "test" + - name: AWS_ENDPOINT_URL + value: "http://localstack:4566" ports: - containerPort: 8891 - containerPort: 8892 @@ -74,21 +84,21 @@ spec: # name: ftl-controller # minReplicas: 1 # maxReplicas: 10 -#--- -#apiVersion: networking.k8s.io/v1 -#kind: Ingress -#metadata: -# name: ftl-controller -# annotations: -# ingress.kubernetes.io/ssl-redirect: "false" -#spec: -# rules: -# - http: -# paths: -# - path: / -# pathType: Prefix -# backend: -# service: -# name: ftl-controller -# port: -# number: 8892 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ftl-controller + annotations: + ingress.kubernetes.io/ssl-redirect: "false" +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ftl-controller + port: + number: 8892 diff --git a/deployment/base/ftl-controller/kustomization.yml b/deployment/base/ftl-controller/kustomization.yml new file mode 100644 index 0000000000..a27896251c --- /dev/null +++ b/deployment/base/ftl-controller/kustomization.yml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ftl-controller.yml diff --git a/deployment/ftl-runner/ftl-runner.yml b/deployment/base/ftl-runner/ftl-runner.yml similarity index 80% rename from deployment/ftl-runner/ftl-runner.yml rename to deployment/base/ftl-runner/ftl-runner.yml index 028555496c..365f806fef 100644 --- a/deployment/ftl-runner/ftl-runner.yml +++ b/deployment/base/ftl-runner/ftl-runner.yml @@ -22,12 +22,16 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: LOG_LEVEL + value: "debug" - name: FTL_ENDPOINT - value: http://ftl-controller + value: "http://ftl-controller:8892" - name: FTL_RUNNER_BIND value: "http://$(MY_POD_IP):8893" - name: FTL_RUNNER_ADVERTISE - value: "" + value: "http://$(MY_POD_IP):8893" + - name: FTL_LANGUAGE + value: "go,kotlin" ports: - containerPort: 8893 readinessProbe: diff --git a/deployment/base/ftl-runner/kustomization.yml b/deployment/base/ftl-runner/kustomization.yml new file mode 100644 index 0000000000..3e7c99590f --- /dev/null +++ b/deployment/base/ftl-runner/kustomization.yml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ftl-runner.yml diff --git a/deployment/base/kustomization.yml b/deployment/base/kustomization.yml new file mode 100644 index 0000000000..06c258e6f1 --- /dev/null +++ b/deployment/base/kustomization.yml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- db-create +- db-migrate +- ftl-controller +- ftl-runner +- localstack +images: +- name: ftl0/ftl-controller + newName: ftl0/ftl-controller + newTag: latest +- name: ftl0/ftl-runner + newName: ftl0/ftl-runner + newTag: latest diff --git a/deployment/base/localstack/kustomization.yml b/deployment/base/localstack/kustomization.yml new file mode 100644 index 0000000000..1d1ab04ca5 --- /dev/null +++ b/deployment/base/localstack/kustomization.yml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - localstack.yml diff --git a/deployment/base/localstack/localstack.yml b/deployment/base/localstack/localstack.yml new file mode 100644 index 0000000000..b24eb24fa3 --- /dev/null +++ b/deployment/base/localstack/localstack.yml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: localstack +spec: + replicas: 1 + selector: + matchLabels: + app: localstack + template: + metadata: + labels: + app: localstack + spec: + containers: + - name: localstack + image: localstack/localstack + ports: + - containerPort: 4566 + env: + - name: DEBUG + value: "1" + - name: SERVICES + value: "secretsmanager" + volumeMounts: + - name: localstack-data + mountPath: /var/lib/localstack + volumes: + - name: localstack-data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: localstack +spec: + selector: + app: localstack + type: ClusterIP + ports: + - name: localstack + port: 4566 + targetPort: 4566 diff --git a/deployment/db-migrate/schema b/deployment/db-migrate/schema deleted file mode 120000 index 0d445ae294..0000000000 --- a/deployment/db-migrate/schema +++ /dev/null @@ -1 +0,0 @@ -../../backend/controller/sql/schema \ No newline at end of file diff --git a/deployment/ftl-controller/kustomization.yml b/deployment/ftl-controller/kustomization.yml deleted file mode 100644 index b453980633..0000000000 --- a/deployment/ftl-controller/kustomization.yml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- ftl-controller.yml diff --git a/deployment/ftl-runner/kustomization.yml b/deployment/ftl-runner/kustomization.yml deleted file mode 100644 index fc269c8fcd..0000000000 --- a/deployment/ftl-runner/kustomization.yml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- ftl-runner.yml diff --git a/deployment/kustomization.yml b/deployment/kustomization.yml deleted file mode 100644 index 56dc2dc10c..0000000000 --- a/deployment/kustomization.yml +++ /dev/null @@ -1,12 +0,0 @@ -resources: - - db-create - - db-migrate - - ftl-controller - - ftl-runner -images: - - name: ftl0/ftl-controller - newName: ftl0/ftl-controller - newTag: latest - - name: ftl0/ftl-runner - newName: ftl0/ftl-runner - newTag: latest diff --git a/deployment/overlays/dev/kustomization.yml b/deployment/overlays/dev/kustomization.yml new file mode 100644 index 0000000000..93ca71fa45 --- /dev/null +++ b/deployment/overlays/dev/kustomization.yml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base + +images: + - name: ftl0/ftl-controller + newName: ftl:5000/ftl-controller + newTag: latest + - name: ftl0/ftl-runner + newName: ftl:5000/ftl-runner + newTag: latest