From b0bdc86b57931dae8be096aef9303125954bdcd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Sat, 23 Mar 2024 00:37:13 +0000 Subject: [PATCH] PubSub. Like Kafka, but with GCP auth (#1524) Shuffle some files around so lua.LState setup code can be moved to `connectors/utils`. Because it brought a dependency on `connectors/utils/catalog` which was then recursive. Moved `connectors/utils/catalog` to `peerdbenv`, & moved dependency in `shared` to `peerdbenv`, so that `shared` once again has no dependencies on other packages in repo. Required also moving `connectors/utils/postgres.go` to `shared/postgres.go`, since that file contains catalog related functionality, not postgres connector functionality --- e2e_cleanup/go.mod | 65 ++--- e2e_cleanup/go.sum | 170 ++++++------ e2e_cleanup/main.go | 47 +++- flow/activities/flowable.go | 2 +- flow/cmd/api.go | 6 +- flow/cmd/handler.go | 7 + flow/cmd/snapshot_worker.go | 6 +- flow/cmd/worker.go | 6 +- flow/connectors/bigquery/bigquery.go | 76 +---- flow/connectors/clickhouse/clickhouse.go | 3 +- flow/connectors/core.go | 4 + flow/connectors/external_metadata/store.go | 4 +- flow/connectors/kafka/kafka.go | 87 +----- flow/connectors/postgres/client.go | 3 +- flow/connectors/postgres/postgres.go | 6 +- .../postgres/postgres_schema_delta_test.go | 4 +- flow/connectors/postgres/qrep.go | 2 +- flow/connectors/postgres/qrep_bench_test.go | 4 +- .../postgres/qrep_partition_test.go | 4 +- .../postgres/qrep_query_executor_test.go | 4 +- flow/connectors/postgres/qrep_sql_sync.go | 2 +- flow/connectors/postgres/ssh_wrapped_pool.go | 3 +- flow/connectors/pubsub/pubsub.go | 262 ++++++++++++++++++ flow/connectors/utils/gcp.go | 115 ++++++++ flow/connectors/utils/lua.go | 74 +++++ flow/dynamicconf/dynamicconf.go | 4 +- flow/e2e/bigquery/peer_flow_bq_test.go | 4 +- flow/e2e/congen.go | 6 +- flow/e2e/kafka/kafka_test.go | 10 +- flow/e2e/postgres/peer_flow_pg_test.go | 3 +- flow/e2e/pubsub/pubsub_test.go | 239 ++++++++++++++++ flow/e2e/snowflake/peer_flow_sf_test.go | 4 +- flow/e2e/test_utils.go | 6 +- flow/go.mod | 1 + flow/go.sum | 2 + .../catalog/env.go => peerdbenv/catalog.go} | 17 +- flow/peerdbenv/config.go | 11 + flow/peerdbenv/env.go | 3 +- flow/pua/peerdb.go | 4 +- flow/shared/constants.go | 18 -- flow/{connectors/utils => shared}/postgres.go | 2 +- flow/shared/string.go | 7 + flow/workflows/cdc_flow.go | 2 +- flow/workflows/snapshot_flow.go | 3 +- nexus/analyzer/src/lib.rs | 79 +++++- nexus/catalog/src/lib.rs | 6 + protos/peers.proto | 19 ++ ui/app/api/peers/getTruePeer.ts | 6 + ui/app/api/peers/info/[peerName]/route.ts | 5 + ui/app/api/peers/route.ts | 7 + ui/app/dto/PeersDTO.ts | 4 +- ui/app/peers/create/[peerType]/handlers.ts | 5 + .../peers/create/[peerType]/helpers/common.ts | 6 + ui/app/peers/create/[peerType]/helpers/ka.ts | 2 +- ui/app/peers/create/[peerType]/helpers/ps.ts | 16 ++ ui/app/peers/create/[peerType]/page.tsx | 3 + ui/app/peers/create/[peerType]/schema.ts | 70 +++++ ui/app/peers/peersTable.tsx | 5 +- ui/components/PeerComponent.tsx | 3 + ui/components/PeerForms/BigqueryConfig.tsx | 2 +- ui/components/PeerForms/PubSubConfig.tsx | 92 ++++++ ui/components/PeerTypeComponent.tsx | 2 + ui/components/SelectSource.tsx | 3 +- ui/public/svgs/pubsub.svg | 29 ++ 64 files changed, 1311 insertions(+), 365 deletions(-) create mode 100644 flow/connectors/pubsub/pubsub.go create mode 100644 flow/connectors/utils/gcp.go create mode 100644 flow/connectors/utils/lua.go create mode 100644 flow/e2e/pubsub/pubsub_test.go rename flow/{connectors/utils/catalog/env.go => peerdbenv/catalog.go} (68%) rename flow/{connectors/utils => shared}/postgres.go (99%) create mode 100644 flow/shared/string.go create mode 100644 ui/app/peers/create/[peerType]/helpers/ps.ts create mode 100644 ui/components/PeerForms/PubSubConfig.tsx create mode 100644 ui/public/svgs/pubsub.svg diff --git a/e2e_cleanup/go.mod b/e2e_cleanup/go.mod index 524b7ff1cc..27625f98d9 100644 --- a/e2e_cleanup/go.mod +++ b/e2e_cleanup/go.mod @@ -4,35 +4,36 @@ go 1.22.0 require ( cloud.google.com/go/bigquery v1.59.1 + cloud.google.com/go/pubsub v1.37.0 github.com/snowflakedb/gosnowflake v1.8.0 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a - google.golang.org/api v0.167.0 + google.golang.org/api v0.171.0 ) require ( - cloud.google.com/go v0.112.0 // indirect - cloud.google.com/go/compute v1.24.0 // indirect + cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go/compute v1.25.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.6 // indirect + cloud.google.com/go/iam v1.1.7 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/apache/arrow/go/v14 v14.0.2 // indirect - github.com/aws/aws-sdk-go-v2 v1.25.2 // indirect + github.com/aws/aws-sdk-go-v2 v1.26.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.4 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.9 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.51.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 // indirect github.com/aws/smithy-go v1.20.1 // indirect github.com/danieljoos/wincred v1.2.1 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect @@ -44,12 +45,12 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/flatbuffers v24.3.7+incompatible // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.1 // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.17.7 // indirect @@ -65,22 +66,22 @@ require ( go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/crypto v0.19.0 // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/mod v0.15.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/oauth2 v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 // indirect + golang.org/x/mod v0.16.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.18.0 // indirect + golang.org/x/tools v0.19.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240221002015-b0ce06bbee7c // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect - google.golang.org/grpc v1.62.0 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/genproto v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/grpc v1.62.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/e2e_cleanup/go.sum b/e2e_cleanup/go.sum index b412b17e30..c25fbe13fd 100644 --- a/e2e_cleanup/go.sum +++ b/e2e_cleanup/go.sum @@ -1,34 +1,38 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= -cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= cloud.google.com/go/bigquery v1.59.1 h1:CpT+/njKuKT3CEmswm6IbhNu9u35zt5dO4yPDLW+nG4= cloud.google.com/go/bigquery v1.59.1/go.mod h1:VP1UJYgevyTwsV7desjzNzDND5p6hZB+Z8gZJN1GQUc= -cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= -cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datacatalog v1.19.3 h1:A0vKYCQdxQuV4Pi0LL9p39Vwvg4jH5yYveMv50gU5Tw= -cloud.google.com/go/datacatalog v1.19.3/go.mod h1:ra8V3UAsciBpJKQ+z9Whkxzxv7jmQg1hfODr3N3YPJ4= -cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= -cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= -cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= -cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= -cloud.google.com/go/storage v1.37.0 h1:WI8CsaFO8Q9KjPVtsZ5Cmi0dXV25zMoX0FklT7c3Jm4= -cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k= +cloud.google.com/go/datacatalog v1.20.0 h1:BGDsEjqpAo0Ka+b9yDLXnE5k+jU3lXGMh//NsEeDMIg= +cloud.google.com/go/datacatalog v1.20.0/go.mod h1:fSHaKjIroFpmRrYlwz9XBB2gJBpXufpnxyAKaT4w6L0= +cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= +cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= +cloud.google.com/go/kms v1.15.8 h1:szIeDCowID8th2i8XE4uRev5PMxQFqW+JjwYxL9h6xs= +cloud.google.com/go/kms v1.15.8/go.mod h1:WoUHcDjD9pluCg7pNds131awnH429QGvRM3N/4MyoVs= +cloud.google.com/go/longrunning v0.5.6 h1:xAe8+0YaWoCKr9t1+aWe+OeQgN/iJK1fEgZSXmjuEaE= +cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA= +cloud.google.com/go/pubsub v1.37.0 h1:0uEEfaB1VIJzabPpwpZf44zWAKAme3zwKKxHk7vJQxQ= +cloud.google.com/go/pubsub v1.37.0/go.mod h1:YQOQr1uiUM092EXwKs56OPT650nwnawc+8/IjoUeGzQ= +cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkpJg= +cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= github.com/99designs/keyring v1.2.2/go.mod h1:wes/FrByc8j7lFOAGLGSNEg8f/PaI3cgTBqhFkHUrPk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 h1:n1DH8TPV4qqPTje2RcUBYwtrTWlabVp4n46+74X2pn4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0/go.mod h1:HDcZnuGbiyppErN6lB+idp4CKhjbc8gwjto6OPpyggM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 h1:IfFdxTUDiV58iZqPKgyWiz4X4fCxZeQ1pTQPImLYXpY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -36,42 +40,42 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvK github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= -github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w= -github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo= +github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA= +github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1/go.mod h1:sxpLb+nZk7tIfCWChfd+h4QwHNUR57d8hA1cleTkjJo= -github.com/aws/aws-sdk-go-v2/config v1.27.4 h1:AhfWb5ZwimdsYTgP7Od8E9L1u4sKmDW2ZVeLcf2O42M= -github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g= -github.com/aws/aws-sdk-go-v2/credentials v1.17.4 h1:h5Vztbd8qLppiPwX+y0Q6WiwMZgpd9keKe2EAENgAuI= -github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 h1:AK0J8iYBFeUk2Ax7O8YpLtFsfhdOByh2QIkHmigpRYk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.6 h1:prcsGA3onmpc7ea1W/m+SMj4uOn5vZ63uJp805UhJJs= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.6/go.mod h1:7eQrvATnVFDY0WfMYhfKkSQ1YtZlClT71fAAlsA1s34= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 h1:bNo4LagzUKbjdxE0tIcR9pMzLR2U/Tgie1Hq1HQ3iH8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 h1:EtOU5jsPdIQNP+6Q2C5e3d65NKT1PeCiQk+9OdzO12Q= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM= +github.com/aws/aws-sdk-go-v2/config v1.27.9 h1:gRx/NwpNEFSk+yQlgmk1bmxxvQ5TyJ76CWXs9XScTqg= +github.com/aws/aws-sdk-go-v2/config v1.27.9/go.mod h1:dK1FQfpwpql83kbD873E9vz4FyAxuJtR22wzoXn3qq0= +github.com/aws/aws-sdk-go-v2/credentials v1.17.9 h1:N8s0/7yW+h8qR8WaRlPQeJ6czVMNQVNtNdUqf6cItao= +github.com/aws/aws-sdk-go-v2/credentials v1.17.9/go.mod h1:446YhIdmSV0Jf/SLafGZalQo+xr2iw7/fzXGDPTU1yQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 h1:af5YzcLf80tv4Em4jWVD75lpnOHSBkPUZxZfGkrI3HI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13 h1:F+PUZee9mlfpEJVZdgyewRumKekS9O3fftj8fEMt0rQ= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13/go.mod h1:Rl7i2dEWGHGsBIJCpUxlRt7VwK/HyXxICxdvIRssQHE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2 h1:en92G0Z7xlksoOylkUhuBSfJgijC7rHVLRdnIlHEs0E= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2/go.mod h1:HgtQ/wN5G+8QSlK62lbOtNwQ3wTSByJ4wH2rCkPt+AE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 h1:SIkD6T4zGQ+1YIit22wi37CGNkrE7mXV1vNA5VpI3TI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4/go.mod h1:XfeqbsG0HNedNs0GT+ju4Bs+pFAwsrlzcRdMvdNVf5s= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.2 h1:zSdTXYLwuXDNPUS+V41i1SFDXG7V0ITp0D9UT9Cvl18= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.2/go.mod h1:v8m8k+qVy95nYi7d56uP1QImleIIY25BPiNJYzPBdFE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 h1:5ffmXjPtwRExp1zc7gENLgCPyHFbhEPwVTkTiH9niSk= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2 h1:1oY1AVEisRI4HNuFoLdRUB0hC63ylDAN6Me3MrfclEg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.2/go.mod h1:KZ03VgvZwSjkT7fOetQ/wF3MZUvYFirlI1H5NklUNsY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.51.1 h1:juZ+uGargZOrQGNxkVHr9HHR/0N+Yu8uekQnV7EAVRs= -github.com/aws/aws-sdk-go-v2/service/s3 v1.51.1/go.mod h1:SoR0c7Jnq8Tpmt0KSLXIavhjmaagRqQpe9r70W3POJg= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 h1:utEGkfdQ4L6YW/ietH7111ZYglLJvS+sLriHJ1NBJEQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 h1:9/GylMS45hGGFCcMrUZDVayQE1jYSIN6da9jo7RAYIw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 h1:3I2cBEYgKhrWlwyZgfpSO2BpaMY1LHPqXYk/QGlu2ew= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 h1:NkHCgg0Ck86c5PTOzBZ0JRccI51suJDg5lgFtxBu1ek= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6/go.mod h1:mjTpxjC8v4SeINTngrnKFgm2QUi+Jm+etTbCxh8W4uU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 h1:uDj2K47EM1reAYU9jVlQ1M5YENI1u6a/TxJpf6AeOLA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4/go.mod h1:XKCODf4RKHppc96c2EZBGV/oCUC7OClxAo2MEyg4pIk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 h1:r3o2YsgW9zRcIP3Q0WCmttFVhTuugeKIvT5z9xDspc0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0/go.mod h1:w2E4f8PUfNtyjfL6Iu+mWI96FGttE03z3UdNcUEC4tA= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0= github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -123,10 +127,10 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= -github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/flatbuffers v24.3.7+incompatible h1:BxGUkIQnOciBu33bd5BdvqY8Qvo0O/GR4SPhh7x9Ed0= +github.com/google/flatbuffers v24.3.7+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -145,8 +149,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.1 h1:9F8GV9r9ztXyAi00gsMQHNoF51xPZm8uj1dpYt2ZETM= -github.com/googleapis/gax-go/v2 v2.12.1/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -198,6 +202,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8= +go.einride.tech/aip v0.66.0/go.mod h1:qAhMsfT7plxBX+Oy7Huol6YUvZ0ZzdUz26yZsQwfl1M= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= @@ -208,25 +214,25 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N6LWlf9KlzibaCapmqc= +golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -236,11 +242,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= -golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -258,12 +264,12 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -279,16 +285,16 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= -golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= -google.golang.org/api v0.167.0 h1:CKHrQD1BLRii6xdkatBDXyKzM0mkawt2QP+H3LtPmSE= -google.golang.org/api v0.167.0/go.mod h1:4FcBc686KFi7QI/U51/2GKKevfZMpM17sCdibqe/bSA= +google.golang.org/api v0.171.0 h1:w174hnBPqut76FzW5Qaupt7zY8Kql6fiVjgys4f58sU= +google.golang.org/api v0.171.0/go.mod h1:Hnq5AHm4OTMt2BUVjael2CWZFD6vksJdWCWiUAmjC9o= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= @@ -296,19 +302,19 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240221002015-b0ce06bbee7c h1:Zmyn5CV/jxzKnF+3d+xzbomACPwLQqVpLTpyXN5uTaQ= -google.golang.org/genproto v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c h1:9g7erC9qu44ks7UK4gDNlnk4kOxZG707xKm4jVniy6o= -google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c h1:NUsgEN92SQQqzfA+YtqYNqYmB3DMMYLlIwUZAQFVFbo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/genproto v0.0.0-20240318140521-94a12d6c2237 h1:PgNlNSx2Nq2/j4juYzQBG0/Zdr+WP4z5N01Vk4VYBCY= +google.golang.org/genproto v0.0.0-20240318140521-94a12d6c2237/go.mod h1:9sVD8c25Af3p0rGs7S7LLsxWKFiJt/65LdSyqXBkX/Y= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -320,8 +326,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/e2e_cleanup/main.go b/e2e_cleanup/main.go index 1fe7ba6bac..b46d95529c 100644 --- a/e2e_cleanup/main.go +++ b/e2e_cleanup/main.go @@ -15,6 +15,7 @@ import ( "time" "cloud.google.com/go/bigquery" + "cloud.google.com/go/pubsub" "github.com/snowflakedb/gosnowflake" "github.com/youmark/pkcs8" "google.golang.org/api/iterator" @@ -59,6 +60,16 @@ func ParseJsonKeyVal[T any](path string) (T, error) { return result, err } +func handleIteratorError(err error) bool { + if err != nil { + if err == iterator.Done { + return true + } + panic(err) + } + return false +} + func CleanupBQ(ctx context.Context) { config, err := ParseJsonKeyVal[map[string]string](os.Getenv("TEST_BQ_CREDS")) if err != nil { @@ -86,11 +97,8 @@ func CleanupBQ(ctx context.Context) { datasetPrefix := config["dataset_id"] for { ds, err := datasets.Next() - if err != nil { - if err == iterator.Done { - return - } - panic(err) + if handleIteratorError(err) { + break } if strings.HasPrefix(ds.DatasetID, datasetPrefix) { @@ -106,6 +114,35 @@ func CleanupBQ(ctx context.Context) { } } } + + // now pubsub too, lack metadata to avoid deleting currently running tests + psclient, err := pubsub.NewClient(ctx, config["project_id"], option.WithCredentialsJSON(config_json)) + if err != nil { + panic(err) + } + defer psclient.Close() + + topics := psclient.Topics(ctx) + for { + topic, err := topics.Next() + if handleIteratorError(err) { + break + } + if strings.HasPrefix(topic.ID(), "e2e") { + topic.Delete(ctx) + } + } + + subscriptions := psclient.Subscriptions(ctx) + for { + subscription, err := subscriptions.Next() + if handleIteratorError(err) { + break + } + if strings.HasPrefix(subscription.ID(), "e2e") { + subscription.Delete(ctx) + } + } } func CleanupSF(ctx context.Context) { diff --git a/flow/activities/flowable.go b/flow/activities/flowable.go index 3d6e7815b7..1f7601b719 100644 --- a/flow/activities/flowable.go +++ b/flow/activities/flowable.go @@ -855,7 +855,7 @@ func (a *FlowableActivity) SendWALHeartbeat(ctx context.Context) error { func() { pgConfig := pgPeer.GetPostgresConfig() - peerConn, peerErr := pgx.Connect(ctx, utils.GetPGConnectionString(pgConfig)) + peerConn, peerErr := pgx.Connect(ctx, shared.GetPGConnectionString(pgConfig)) if peerErr != nil { logger.Error(fmt.Sprintf("error creating pool for postgres peer %v with host %v: %v", pgPeer.Name, pgConfig.Host, peerErr)) diff --git a/flow/cmd/api.go b/flow/cmd/api.go index 5803d42317..cf4d84b678 100644 --- a/flow/cmd/api.go +++ b/flow/cmd/api.go @@ -21,9 +21,9 @@ import ( "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/reflection" - utils "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" peerflow "github.com/PeerDB-io/peer-flow/workflows" ) @@ -119,12 +119,12 @@ func APIMain(ctx context.Context, args *APIServerParams) error { grpcServer := grpc.NewServer() - catalogConn, err := utils.GetCatalogConnectionPoolFromEnv(ctx) + catalogConn, err := peerdbenv.GetCatalogConnectionPoolFromEnv(ctx) if err != nil { return fmt.Errorf("unable to get catalog connection pool: %w", err) } - taskQueue := shared.GetPeerFlowTaskQueueName(shared.PeerFlowTaskQueue) + taskQueue := peerdbenv.PeerFlowTaskQueueName(shared.PeerFlowTaskQueue) flowHandler := NewFlowRequestHandler(tc, catalogConn, taskQueue) err = killExistingScheduleFlows(ctx, tc, args.TemporalNamespace, taskQueue) diff --git a/flow/cmd/handler.go b/flow/cmd/handler.go index f0264608bb..7c528d8333 100644 --- a/flow/cmd/handler.go +++ b/flow/cmd/handler.go @@ -576,6 +576,13 @@ func (h *FlowRequestHandler) CreatePeer( } kaConfig := kaConfigObject.KafkaConfig encodedConfig, encodingErr = proto.Marshal(kaConfig) + case protos.DBType_PUBSUB: + psConfigObject, ok := config.(*protos.Peer_PubsubConfig) + if !ok { + return wrongConfigResponse, nil + } + psConfig := psConfigObject.PubsubConfig + encodedConfig, encodingErr = proto.Marshal(psConfig) default: return wrongConfigResponse, nil } diff --git a/flow/cmd/snapshot_worker.go b/flow/cmd/snapshot_worker.go index b7418e9ceb..128759a474 100644 --- a/flow/cmd/snapshot_worker.go +++ b/flow/cmd/snapshot_worker.go @@ -12,8 +12,8 @@ import ( "github.com/PeerDB-io/peer-flow/activities" "github.com/PeerDB-io/peer-flow/alerting" - utils "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" peerflow "github.com/PeerDB-io/peer-flow/workflows" ) @@ -47,7 +47,7 @@ func SnapshotWorkerMain(opts *SnapshotWorkerOptions) (client.Client, worker.Work clientOptions.ConnectionOptions = connOptions } - conn, err := utils.GetCatalogConnectionPoolFromEnv(context.Background()) + conn, err := peerdbenv.GetCatalogConnectionPoolFromEnv(context.Background()) if err != nil { return nil, nil, fmt.Errorf("unable to create catalog connection pool: %w", err) } @@ -57,7 +57,7 @@ func SnapshotWorkerMain(opts *SnapshotWorkerOptions) (client.Client, worker.Work return nil, nil, fmt.Errorf("unable to create Temporal client: %w", err) } - taskQueue := shared.GetPeerFlowTaskQueueName(shared.SnapshotFlowTaskQueue) + taskQueue := peerdbenv.PeerFlowTaskQueueName(shared.SnapshotFlowTaskQueue) w := worker.New(c, taskQueue, worker.Options{ EnableSessionWorker: true, OnFatalError: func(err error) { diff --git a/flow/cmd/worker.go b/flow/cmd/worker.go index 8977108be7..a499a26e2d 100644 --- a/flow/cmd/worker.go +++ b/flow/cmd/worker.go @@ -16,8 +16,8 @@ import ( "github.com/PeerDB-io/peer-flow/activities" "github.com/PeerDB-io/peer-flow/alerting" "github.com/PeerDB-io/peer-flow/connectors" - utils "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" peerflow "github.com/PeerDB-io/peer-flow/workflows" ) @@ -98,7 +98,7 @@ func WorkerMain(opts *WorkerOptions) (client.Client, worker.Worker, error) { clientOptions.ConnectionOptions = connOptions } - conn, err := utils.GetCatalogConnectionPoolFromEnv(context.Background()) + conn, err := peerdbenv.GetCatalogConnectionPoolFromEnv(context.Background()) if err != nil { return nil, nil, fmt.Errorf("unable to create catalog connection pool: %w", err) } @@ -109,7 +109,7 @@ func WorkerMain(opts *WorkerOptions) (client.Client, worker.Worker, error) { } slog.Info("Created temporal client") - taskQueue := shared.GetPeerFlowTaskQueueName(shared.PeerFlowTaskQueue) + taskQueue := peerdbenv.PeerFlowTaskQueueName(shared.PeerFlowTaskQueue) w := worker.New(c, taskQueue, worker.Options{ EnableSessionWorker: true, OnFatalError: func(err error) { diff --git a/flow/connectors/bigquery/bigquery.go b/flow/connectors/bigquery/bigquery.go index db56cfd9fc..7dc1b3b77e 100644 --- a/flow/connectors/bigquery/bigquery.go +++ b/flow/connectors/bigquery/bigquery.go @@ -2,7 +2,6 @@ package connbigquery import ( "context" - "encoding/json" "errors" "fmt" "log/slog" @@ -17,16 +16,15 @@ import ( "go.temporal.io/sdk/activity" "go.temporal.io/sdk/log" "google.golang.org/api/iterator" - "google.golang.org/api/option" metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" "github.com/PeerDB-io/peer-flow/connectors/utils" - cc "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/numeric" "github.com/PeerDB-io/peer-flow/model/qvalue" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" ) @@ -34,19 +32,6 @@ const ( SyncRecordsBatchSize = 1024 ) -type BigQueryServiceAccount struct { - Type string `json:"type"` - ProjectID string `json:"project_id"` - PrivateKeyID string `json:"private_key_id"` - PrivateKey string `json:"private_key"` - ClientEmail string `json:"client_email"` - ClientID string `json:"client_id"` - AuthURI string `json:"auth_uri"` - TokenURI string `json:"token_uri"` - AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"` - ClientX509CertURL string `json:"client_x509_cert_url"` -} - type BigQueryConnector struct { bqConfig *protos.BigqueryConfig client *bigquery.Client @@ -58,8 +43,8 @@ type BigQueryConnector struct { logger log.Logger } -func NewBigQueryServiceAccount(bqConfig *protos.BigqueryConfig) (*BigQueryServiceAccount, error) { - var serviceAccount BigQueryServiceAccount +func NewBigQueryServiceAccount(bqConfig *protos.BigqueryConfig) (*utils.GcpServiceAccount, error) { + var serviceAccount utils.GcpServiceAccount serviceAccount.Type = bqConfig.AuthType serviceAccount.ProjectID = bqConfig.ProjectId serviceAccount.PrivateKeyID = bqConfig.PrivateKeyId @@ -78,59 +63,6 @@ func NewBigQueryServiceAccount(bqConfig *protos.BigqueryConfig) (*BigQueryServic return &serviceAccount, nil } -// Validate validates a BigQueryServiceAccount, that none of the fields are empty. -func (bqsa *BigQueryServiceAccount) Validate() error { - v := reflect.ValueOf(*bqsa) - for i := range v.NumField() { - if v.Field(i).String() == "" { - return fmt.Errorf("field %s is empty", v.Type().Field(i).Name) - } - } - return nil -} - -// Return BigQueryServiceAccount as JSON byte array -func (bqsa *BigQueryServiceAccount) ToJSON() ([]byte, error) { - return json.Marshal(bqsa) -} - -// CreateBigQueryClient creates a new BigQuery client from a BigQueryServiceAccount. -func (bqsa *BigQueryServiceAccount) CreateBigQueryClient(ctx context.Context) (*bigquery.Client, error) { - bqsaJSON, err := bqsa.ToJSON() - if err != nil { - return nil, fmt.Errorf("failed to get json: %v", err) - } - - client, err := bigquery.NewClient( - ctx, - bqsa.ProjectID, - option.WithCredentialsJSON(bqsaJSON), - ) - if err != nil { - return nil, fmt.Errorf("failed to create BigQuery client: %v", err) - } - - return client, nil -} - -// CreateStorageClient creates a new Storage client from a BigQueryServiceAccount. -func (bqsa *BigQueryServiceAccount) CreateStorageClient(ctx context.Context) (*storage.Client, error) { - bqsaJSON, err := bqsa.ToJSON() - if err != nil { - return nil, fmt.Errorf("failed to get json: %v", err) - } - - client, err := storage.NewClient( - ctx, - option.WithCredentialsJSON(bqsaJSON), - ) - if err != nil { - return nil, fmt.Errorf("failed to create Storage client: %v", err) - } - - return client, nil -} - // ValidateCheck: // 1. Creates a table // 2. Inserts one row into the table @@ -212,7 +144,7 @@ func NewBigQueryConnector(ctx context.Context, config *protos.BigqueryConfig) (* return nil, fmt.Errorf("failed to create Storage client: %v", err) } - catalogPool, err := cc.GetCatalogConnectionPoolFromEnv(ctx) + catalogPool, err := peerdbenv.GetCatalogConnectionPoolFromEnv(ctx) if err != nil { return nil, fmt.Errorf("failed to create catalog connection pool: %v", err) } diff --git a/flow/connectors/clickhouse/clickhouse.go b/flow/connectors/clickhouse/clickhouse.go index a4e959b505..fbd9b75eaa 100644 --- a/flow/connectors/clickhouse/clickhouse.go +++ b/flow/connectors/clickhouse/clickhouse.go @@ -16,6 +16,7 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" ) @@ -113,7 +114,7 @@ func NewClickhouseConnector( if clickhouseS3Creds.AccessKeyID == "" && clickhouseS3Creds.SecretAccessKey == "" && clickhouseS3Creds.Region == "" && clickhouseS3Creds.BucketPath == "" { - deploymentUID := shared.GetDeploymentUID() + deploymentUID := peerdbenv.PeerDBDeploymentUID() flowName, _ := ctx.Value(shared.FlowNameKey).(string) bucketPathSuffix := fmt.Sprintf("%s/%s", url.PathEscape(deploymentUID), url.PathEscape(flowName)) diff --git a/flow/connectors/core.go b/flow/connectors/core.go index 4251c1726a..3387cf963b 100644 --- a/flow/connectors/core.go +++ b/flow/connectors/core.go @@ -13,6 +13,7 @@ import ( conneventhub "github.com/PeerDB-io/peer-flow/connectors/eventhub" connkafka "github.com/PeerDB-io/peer-flow/connectors/kafka" connpostgres "github.com/PeerDB-io/peer-flow/connectors/postgres" + connpubsub "github.com/PeerDB-io/peer-flow/connectors/pubsub" conns3 "github.com/PeerDB-io/peer-flow/connectors/s3" connsnowflake "github.com/PeerDB-io/peer-flow/connectors/snowflake" connsqlserver "github.com/PeerDB-io/peer-flow/connectors/sqlserver" @@ -205,6 +206,8 @@ func GetConnector(ctx context.Context, config *protos.Peer) (Connector, error) { return connclickhouse.NewClickhouseConnector(ctx, inner.ClickhouseConfig) case *protos.Peer_KafkaConfig: return connkafka.NewKafkaConnector(ctx, inner.KafkaConfig) + case *protos.Peer_PubsubConfig: + return connpubsub.NewPubSubConnector(ctx, inner.PubsubConfig) default: return nil, ErrUnsupportedFunctionality } @@ -264,6 +267,7 @@ var ( _ CDCSyncConnector = &connsnowflake.SnowflakeConnector{} _ CDCSyncConnector = &conneventhub.EventHubConnector{} _ CDCSyncConnector = &connkafka.KafkaConnector{} + _ CDCSyncConnector = &connpubsub.PubSubConnector{} _ CDCSyncConnector = &conns3.S3Connector{} _ CDCSyncConnector = &connclickhouse.ClickhouseConnector{} diff --git a/flow/connectors/external_metadata/store.go b/flow/connectors/external_metadata/store.go index bbe986b497..49a4ed4784 100644 --- a/flow/connectors/external_metadata/store.go +++ b/flow/connectors/external_metadata/store.go @@ -12,9 +12,9 @@ import ( "go.temporal.io/sdk/log" "google.golang.org/protobuf/encoding/protojson" - cc "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/peerdbenv" ) const ( @@ -28,7 +28,7 @@ type PostgresMetadataStore struct { } func NewPostgresMetadataStore(ctx context.Context) (*PostgresMetadataStore, error) { - pool, err := cc.GetCatalogConnectionPoolFromEnv(ctx) + pool, err := peerdbenv.GetCatalogConnectionPoolFromEnv(ctx) if err != nil { return nil, fmt.Errorf("failed to create catalog connection pool: %w", err) } diff --git a/flow/connectors/kafka/kafka.go b/flow/connectors/kafka/kafka.go index b879e31def..cefe73ca4b 100644 --- a/flow/connectors/kafka/kafka.go +++ b/flow/connectors/kafka/kafka.go @@ -3,12 +3,10 @@ package connkafka import ( "context" "crypto/tls" - "errors" "fmt" "log/slog" "strings" "sync" - "unsafe" "github.com/twmb/franz-go/pkg/kgo" "github.com/twmb/franz-go/pkg/sasl/plain" @@ -17,13 +15,14 @@ import ( "github.com/yuin/gopher-lua" "go.temporal.io/sdk/log" - "github.com/PeerDB-io/gluaflatbuffers" metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" + "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/pua" + "github.com/PeerDB-io/peer-flow/shared" ) type KafkaConnector struct { @@ -32,31 +31,6 @@ type KafkaConnector struct { logger log.Logger } -func unsafeFastStringToReadOnlyBytes(s string) []byte { - return unsafe.Slice(unsafe.StringData(s), len(s)) -} - -func LVAsReadOnlyBytes(ls *lua.LState, v lua.LValue) ([]byte, error) { - str, err := LVAsStringOrNil(ls, v) - if err != nil { - return nil, err - } else if str == "" { - return nil, nil - } else { - return unsafeFastStringToReadOnlyBytes(str), nil - } -} - -func LVAsStringOrNil(ls *lua.LState, v lua.LValue) (string, error) { - if lstr, ok := v.(lua.LString); ok { - return string(lstr), nil - } else if v == lua.LNil { - return "", nil - } else { - return "", fmt.Errorf("invalid bytes, must be nil or string: %s", v) - } -} - func NewKafkaConnector( ctx context.Context, config *protos.KafkaConfig, @@ -158,59 +132,21 @@ func (c *KafkaConnector) SyncFlowCleanup(ctx context.Context, jobName string) er return c.pgMetadata.DropMetadata(ctx, jobName) } -func loadScript(ctx context.Context, script string, printfn lua.LGFunction) (*lua.LState, error) { - if script == "" { - return nil, errors.New("kafka mirror must have script") - } - - ls := lua.NewState(lua.Options{SkipOpenLibs: true}) - ls.SetContext(ctx) - for _, pair := range []struct { - n string - f lua.LGFunction - }{ - {lua.LoadLibName, lua.OpenPackage}, // Must be first - {lua.BaseLibName, lua.OpenBase}, - {lua.TabLibName, lua.OpenTable}, - {lua.StringLibName, lua.OpenString}, - {lua.MathLibName, lua.OpenMath}, - } { - ls.Push(ls.NewFunction(pair.f)) - ls.Push(lua.LString(pair.n)) - err := ls.PCall(1, 0, nil) - if err != nil { - return nil, fmt.Errorf("failed to initialize Lua runtime: %w", err) - } - } - ls.PreloadModule("flatbuffers", gluaflatbuffers.Loader) - pua.RegisterTypes(ls) - ls.Env.RawSetString("print", ls.NewFunction(printfn)) - err := ls.GPCall(pua.LoadPeerdbScript, lua.LString(script)) - if err != nil { - return nil, fmt.Errorf("error loading script %s: %w", script, err) - } - err = ls.PCall(0, 0, nil) - if err != nil { - return nil, fmt.Errorf("error executing script %s: %w", script, err) - } - return ls, nil -} - func lvalueToKafkaRecord(ls *lua.LState, value lua.LValue) (*kgo.Record, error) { var kr *kgo.Record switch v := value.(type) { case lua.LString: kr = kgo.StringRecord(string(v)) case *lua.LTable: - key, err := LVAsReadOnlyBytes(ls, ls.GetField(v, "key")) + key, err := utils.LVAsReadOnlyBytes(ls, ls.GetField(v, "key")) if err != nil { return nil, fmt.Errorf("invalid key, %w", err) } - value, err := LVAsReadOnlyBytes(ls, ls.GetField(v, "value")) + value, err := utils.LVAsReadOnlyBytes(ls, ls.GetField(v, "value")) if err != nil { return nil, fmt.Errorf("invalid value, %w", err) } - topic, err := LVAsStringOrNil(ls, ls.GetField(v, "topic")) + topic, err := utils.LVAsStringOrNil(ls, ls.GetField(v, "topic")) if err != nil { return nil, fmt.Errorf("invalid topic, %w", err) } @@ -225,9 +161,9 @@ func lvalueToKafkaRecord(ls *lua.LState, value lua.LValue) (*kgo.Record, error) if headers, ok := lheaders.(*lua.LTable); ok { headers.ForEach(func(k, v lua.LValue) { kstr := k.String() - vbytes, err := LVAsReadOnlyBytes(ls, v) + vbytes, err := utils.LVAsReadOnlyBytes(ls, v) if err != nil { - vbytes = unsafeFastStringToReadOnlyBytes(err.Error()) + vbytes = shared.UnsafeFastStringToReadOnlyBytes(err.Error()) } kr.Headers = append(kr.Headers, kgo.RecordHeader{ Key: kstr, @@ -257,7 +193,7 @@ func (c *KafkaConnector) SyncRecords(ctx context.Context, req *model.SyncRecords numRecords := int64(0) tableNameRowsMapping := make(map[string]uint32) - ls, err := loadScript(wgCtx, req.Script, func(ls *lua.LState) int { + ls, err := utils.LoadScript(wgCtx, req.Script, func(ls *lua.LState) int { top := ls.GetTop() ss := make([]string, top) for i := range top { @@ -307,6 +243,9 @@ func (c *KafkaConnector) SyncRecords(ctx context.Context, req *model.SyncRecords ls.SetTop(0) } + if err := c.client.Flush(ctx); err != nil { + return nil, fmt.Errorf("could not flush transaction: %w", err) + } waitChan := make(chan struct{}) go func() { wg.Wait() @@ -318,10 +257,6 @@ func (c *KafkaConnector) SyncRecords(ctx context.Context, req *model.SyncRecords case <-waitChan: } - if err := c.client.Flush(ctx); err != nil { - return nil, fmt.Errorf("could not flush transaction: %w", err) - } - lastCheckpoint := req.Records.GetLastCheckpoint() err = c.pgMetadata.FinishBatch(ctx, req.FlowJobName, req.SyncBatchID, lastCheckpoint) if err != nil { diff --git a/flow/connectors/postgres/client.go b/flow/connectors/postgres/client.go index f667107bb2..54df692986 100644 --- a/flow/connectors/postgres/client.go +++ b/flow/connectors/postgres/client.go @@ -16,6 +16,7 @@ import ( "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/numeric" + "github.com/PeerDB-io/peer-flow/shared" ) type PGVersion int @@ -424,7 +425,7 @@ func (c *PostgresConnector) createSlotAndPublication( func (c *PostgresConnector) createMetadataSchema(ctx context.Context) error { _, err := c.conn.Exec(ctx, fmt.Sprintf(createSchemaSQL, c.metadataSchema)) - if err != nil && !utils.IsUniqueError(err) { + if err != nil && !shared.IsUniqueError(err) { return fmt.Errorf("error while creating internal schema: %w", err) } return nil diff --git a/flow/connectors/postgres/postgres.go b/flow/connectors/postgres/postgres.go index c57154bd7c..82cbd9dd6d 100644 --- a/flow/connectors/postgres/postgres.go +++ b/flow/connectors/postgres/postgres.go @@ -54,7 +54,7 @@ type ReplState struct { } func NewPostgresConnector(ctx context.Context, pgConfig *protos.PostgresConfig) (*PostgresConnector, error) { - connectionString := utils.GetPGConnectionString(pgConfig) + connectionString := shared.GetPGConnectionString(pgConfig) // create a separate connection pool for non-replication queries as replication connections cannot // be used for extended query protocol, i.e. prepared statements @@ -82,7 +82,7 @@ func NewPostgresConnector(ctx context.Context, pgConfig *protos.PostgresConfig) replConfig.Config.RuntimeParams["replication"] = "database" replConfig.Config.RuntimeParams["bytea_output"] = "hex" - customTypeMap, err := utils.GetCustomDataTypes(ctx, conn) + customTypeMap, err := shared.GetCustomDataTypes(ctx, conn) if err != nil { return nil, fmt.Errorf("failed to get custom type map: %w", err) } @@ -272,7 +272,7 @@ func (c *PostgresConnector) SetupMetadataTables(ctx context.Context) error { _, err = c.conn.Exec(ctx, fmt.Sprintf(createMirrorJobsTableSQL, c.metadataSchema, mirrorJobsTableIdentifier)) - if err != nil && !utils.IsUniqueError(err) { + if err != nil && !shared.IsUniqueError(err) { return fmt.Errorf("error creating table %s: %w", mirrorJobsTableIdentifier, err) } diff --git a/flow/connectors/postgres/postgres_schema_delta_test.go b/flow/connectors/postgres/postgres_schema_delta_test.go index e5603b02d9..c71c77c25c 100644 --- a/flow/connectors/postgres/postgres_schema_delta_test.go +++ b/flow/connectors/postgres/postgres_schema_delta_test.go @@ -9,10 +9,10 @@ import ( "github.com/jackc/pgx/v5" "github.com/stretchr/testify/require" - "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/e2eshared" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model/qvalue" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" ) @@ -25,7 +25,7 @@ type PostgresSchemaDeltaTestSuite struct { func SetupSuite(t *testing.T) PostgresSchemaDeltaTestSuite { t.Helper() - connector, err := NewPostgresConnector(context.Background(), utils.GetCatalogPostgresConfigFromEnv()) + connector, err := NewPostgresConnector(context.Background(), peerdbenv.GetCatalogPostgresConfigFromEnv()) require.NoError(t, err) setupTx, err := connector.conn.Begin(context.Background()) diff --git a/flow/connectors/postgres/qrep.go b/flow/connectors/postgres/qrep.go index ce9521ff4c..0544c16d3c 100644 --- a/flow/connectors/postgres/qrep.go +++ b/flow/connectors/postgres/qrep.go @@ -483,7 +483,7 @@ func (c *PostgresConnector) SetupQRepMetadataTables(ctx context.Context, config )`, metadataTableIdentifier.Sanitize()) // execute create table query _, err = c.conn.Exec(ctx, createQRepMetadataTableSQL) - if err != nil && !utils.IsUniqueError(err) { + if err != nil && !shared.IsUniqueError(err) { return fmt.Errorf("failed to create table %s: %w", qRepMetadataTableName, err) } c.logger.Info("Setup metadata table.") diff --git a/flow/connectors/postgres/qrep_bench_test.go b/flow/connectors/postgres/qrep_bench_test.go index f5882bf299..252b7520eb 100644 --- a/flow/connectors/postgres/qrep_bench_test.go +++ b/flow/connectors/postgres/qrep_bench_test.go @@ -4,14 +4,14 @@ import ( "context" "testing" - "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" + "github.com/PeerDB-io/peer-flow/peerdbenv" ) func BenchmarkQRepQueryExecutor(b *testing.B) { query := "SELECT * FROM bench.large_table" ctx := context.Background() - connector, err := NewPostgresConnector(ctx, utils.GetCatalogPostgresConfigFromEnv()) + connector, err := NewPostgresConnector(ctx, peerdbenv.GetCatalogPostgresConfigFromEnv()) if err != nil { b.Fatalf("failed to create connection: %v", err) } diff --git a/flow/connectors/postgres/qrep_partition_test.go b/flow/connectors/postgres/qrep_partition_test.go index 0512c68415..bcafbe2f76 100644 --- a/flow/connectors/postgres/qrep_partition_test.go +++ b/flow/connectors/postgres/qrep_partition_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/assert" "go.temporal.io/sdk/log" - "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" ) @@ -64,7 +64,7 @@ func newTestCaseForCTID(schema string, name string, rows uint32, expectedNum int } func TestGetQRepPartitions(t *testing.T) { - connStr := utils.GetCatalogConnectionStringFromEnv() + connStr := peerdbenv.GetCatalogConnectionStringFromEnv() // Setup the DB config, err := pgx.ParseConfig(connStr) diff --git a/flow/connectors/postgres/qrep_query_executor_test.go b/flow/connectors/postgres/qrep_query_executor_test.go index c2bc037372..cdb37b0349 100644 --- a/flow/connectors/postgres/qrep_query_executor_test.go +++ b/flow/connectors/postgres/qrep_query_executor_test.go @@ -11,13 +11,13 @@ import ( "github.com/jackc/pgx/v5" "github.com/shopspring/decimal" - "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" + "github.com/PeerDB-io/peer-flow/peerdbenv" ) func setupDB(t *testing.T) (*PostgresConnector, string) { t.Helper() - connector, err := NewPostgresConnector(context.Background(), utils.GetCatalogPostgresConfigFromEnv()) + connector, err := NewPostgresConnector(context.Background(), peerdbenv.GetCatalogPostgresConfigFromEnv()) if err != nil { t.Fatalf("unable to create connector: %v", err) } diff --git a/flow/connectors/postgres/qrep_sql_sync.go b/flow/connectors/postgres/qrep_sql_sync.go index 16ed755553..a6ccc5e224 100644 --- a/flow/connectors/postgres/qrep_sql_sync.go +++ b/flow/connectors/postgres/qrep_sql_sync.go @@ -59,7 +59,7 @@ func (s *QRepStagingTableSync) SyncQRepRecords( } defer txConn.Close(ctx) - err = utils.RegisterHStore(ctx, txConn) + err = shared.RegisterHStore(ctx, txConn) if err != nil { return 0, fmt.Errorf("failed to register hstore: %w", err) } diff --git a/flow/connectors/postgres/ssh_wrapped_pool.go b/flow/connectors/postgres/ssh_wrapped_pool.go index bfb19a48f3..ccfb6c6488 100644 --- a/flow/connectors/postgres/ssh_wrapped_pool.go +++ b/flow/connectors/postgres/ssh_wrapped_pool.go @@ -14,6 +14,7 @@ import ( "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/shared" ) type SSHTunnel struct { @@ -78,7 +79,7 @@ func (tunnel *SSHTunnel) NewPostgresConnFromPostgresConfig( ctx context.Context, pgConfig *protos.PostgresConfig, ) (*pgx.Conn, error) { - connectionString := utils.GetPGConnectionString(pgConfig) + connectionString := shared.GetPGConnectionString(pgConfig) connConfig, err := pgx.ParseConfig(connectionString) if err != nil { diff --git a/flow/connectors/pubsub/pubsub.go b/flow/connectors/pubsub/pubsub.go new file mode 100644 index 0000000000..6b51bb246a --- /dev/null +++ b/flow/connectors/pubsub/pubsub.go @@ -0,0 +1,262 @@ +package connpubsub + +import ( + "context" + "fmt" + "strings" + "sync" + + "cloud.google.com/go/pubsub" + "github.com/yuin/gopher-lua" + "go.temporal.io/sdk/log" + + metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata" + "github.com/PeerDB-io/peer-flow/connectors/utils" + "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/model" + "github.com/PeerDB-io/peer-flow/pua" + "github.com/PeerDB-io/peer-flow/shared" +) + +type PubSubConnector struct { + client *pubsub.Client + pgMetadata *metadataStore.PostgresMetadataStore + logger log.Logger +} + +func NewPubSubConnector( + ctx context.Context, + config *protos.PubSubConfig, +) (*PubSubConnector, error) { + sa := utils.GcpServiceAccountFromProto(config.ServiceAccount) + client, err := sa.CreatePubSubClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create pubsub client: %w", err) + } + + pgMetadata, err := metadataStore.NewPostgresMetadataStore(ctx) + if err != nil { + return nil, err + } + + return &PubSubConnector{ + client: client, + pgMetadata: pgMetadata, + logger: logger.LoggerFromCtx(ctx), + }, nil +} + +func (c *PubSubConnector) Close() error { + if c != nil { + c.client.Close() + } + return nil +} + +func (c *PubSubConnector) ConnectionActive(ctx context.Context) error { + topic := c.client.Topic("test") + _, err := topic.Exists(ctx) + return err +} + +func (c *PubSubConnector) CreateRawTable(ctx context.Context, req *protos.CreateRawTableInput) (*protos.CreateRawTableOutput, error) { + return &protos.CreateRawTableOutput{TableIdentifier: "n/a"}, nil +} + +func (c *PubSubConnector) GetLastSyncBatchID(ctx context.Context, jobName string) (int64, error) { + return c.pgMetadata.GetLastBatchID(ctx, jobName) +} + +func (c *PubSubConnector) GetLastOffset(ctx context.Context, jobName string) (int64, error) { + return c.pgMetadata.FetchLastOffset(ctx, jobName) +} + +func (c *PubSubConnector) SetLastOffset(ctx context.Context, jobName string, offset int64) error { + return c.pgMetadata.UpdateLastOffset(ctx, jobName, offset) +} + +func (c *PubSubConnector) NeedsSetupMetadataTables(_ context.Context) bool { + return false +} + +func (c *PubSubConnector) SetupMetadataTables(_ context.Context) error { + return nil +} + +func (c *PubSubConnector) ReplayTableSchemaDeltas(_ context.Context, flowJobName string, schemaDeltas []*protos.TableSchemaDelta) error { + return nil +} + +func (c *PubSubConnector) SyncFlowCleanup(ctx context.Context, jobName string) error { + return c.pgMetadata.DropMetadata(ctx, jobName) +} + +func lvalueToPubSubMessage(ls *lua.LState, value lua.LValue) (string, *pubsub.Message, error) { + var topic string + var msg *pubsub.Message + switch v := value.(type) { + case lua.LString: + msg = &pubsub.Message{ + Data: shared.UnsafeFastStringToReadOnlyBytes(string(v)), + } + case *lua.LTable: + key, err := utils.LVAsStringOrNil(ls, ls.GetField(v, "key")) + if err != nil { + return "", nil, fmt.Errorf("invalid key, %w", err) + } + value, err := utils.LVAsReadOnlyBytes(ls, ls.GetField(v, "value")) + if err != nil { + return "", nil, fmt.Errorf("invalid value, %w", err) + } + topic, err = utils.LVAsStringOrNil(ls, ls.GetField(v, "topic")) + if err != nil { + return "", nil, fmt.Errorf("invalid topic, %w", err) + } + msg = &pubsub.Message{ + OrderingKey: key, + Data: value, + } + lheaders := ls.GetField(v, "headers") + if headers, ok := lheaders.(*lua.LTable); ok { + msg.Attributes = make(map[string]string) + headers.ForEach(func(k, v lua.LValue) { + msg.Attributes[k.String()] = v.String() + }) + } else if lua.LVAsBool(lheaders) { + return "", nil, fmt.Errorf("invalid headers, must be nil or table: %s", lheaders) + } + case *lua.LNilType: + default: + return "", nil, fmt.Errorf("script returned invalid value: %s", value) + } + return topic, msg, nil +} + +func (c *PubSubConnector) SyncRecords(ctx context.Context, req *model.SyncRecordsRequest) (*model.SyncResponse, error) { + numRecords := int64(0) + tableNameRowsMapping := make(map[string]uint32) + + ls, err := utils.LoadScript(ctx, req.Script, func(ls *lua.LState) int { + top := ls.GetTop() + ss := make([]string, top) + for i := range top { + ss[i] = ls.ToStringMeta(ls.Get(i + 1)).String() + } + _ = c.pgMetadata.LogFlowInfo(ctx, req.FlowJobName, strings.Join(ss, "\t")) + return 0 + }) + if err != nil { + return nil, err + } + defer ls.Close() + + lfn := ls.Env.RawGetString("onRecord") + fn, ok := lfn.(*lua.LFunction) + if !ok { + return nil, fmt.Errorf("script should define `onRecord` as function, not %s", lfn) + } + + var wg sync.WaitGroup + wgCtx, wgErr := context.WithCancelCause(ctx) + publish := make(chan *pubsub.PublishResult, 60) + go func() { + var curpub *pubsub.PublishResult + for { + select { + case curpub, ok = <-publish: + if !ok { + return + } + case <-ctx.Done(): + wgErr(ctx.Err()) + return + } + _, err := curpub.Get(ctx) + if err != nil { + wgErr(err) + return + } + wg.Done() + } + }() + + topiccache := make(map[string]*pubsub.Topic) + for record := range req.Records.GetRecords() { + if err := ctx.Err(); err != nil { + return nil, err + } + ls.Push(fn) + ls.Push(pua.LuaRecord.New(ls, record)) + err := ls.PCall(1, -1, nil) + if err != nil { + return nil, fmt.Errorf("script failed: %w", err) + } + args := ls.GetTop() + for i := range args { + topic, msg, err := lvalueToPubSubMessage(ls, ls.Get(i-args)) + if err != nil { + return nil, err + } + if msg != nil { + if topic == "" { + topic = record.GetDestinationTableName() + } + topicClient, ok := topiccache[topic] + if !ok { + topicClient = c.client.Topic(topic) + exists, err := topicClient.Exists(ctx) + if err != nil { + return nil, fmt.Errorf("error checking if topic exists: %w", err) + } + if !exists { + topicClient, err = c.client.CreateTopic(ctx, topic) + if err != nil { + return nil, fmt.Errorf("error creating topic: %w", err) + } + } + topiccache[topic] = topicClient + } + + pubresult := topicClient.Publish(ctx, msg) + wg.Add(1) + publish <- pubresult + tableNameRowsMapping[topic] += 1 + } + } + numRecords += 1 + ls.SetTop(0) + } + + close(publish) + for _, topicClient := range topiccache { + if err := ctx.Err(); err != nil { + return nil, err + } + topicClient.Stop() + } + waitChan := make(chan struct{}) + go func() { + wg.Wait() + close(waitChan) + }() + select { + case <-wgCtx.Done(): + return nil, wgCtx.Err() + case <-waitChan: + } + + lastCheckpoint := req.Records.GetLastCheckpoint() + err = c.pgMetadata.FinishBatch(ctx, req.FlowJobName, req.SyncBatchID, lastCheckpoint) + if err != nil { + return nil, err + } + + return &model.SyncResponse{ + CurrentSyncBatchID: req.SyncBatchID, + LastSyncedCheckpointID: lastCheckpoint, + NumRecordsSynced: numRecords, + TableNameRowsMapping: tableNameRowsMapping, + TableSchemaDeltas: req.Records.SchemaDeltas, + }, nil +} diff --git a/flow/connectors/utils/gcp.go b/flow/connectors/utils/gcp.go new file mode 100644 index 0000000000..09aa19c8a7 --- /dev/null +++ b/flow/connectors/utils/gcp.go @@ -0,0 +1,115 @@ +package utils + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + + "cloud.google.com/go/bigquery" + "cloud.google.com/go/pubsub" + "cloud.google.com/go/storage" + "google.golang.org/api/option" + + "github.com/PeerDB-io/peer-flow/generated/protos" +) + +type GcpServiceAccount struct { + Type string `json:"type"` + ProjectID string `json:"project_id"` + PrivateKeyID string `json:"private_key_id"` + PrivateKey string `json:"private_key"` + ClientEmail string `json:"client_email"` + ClientID string `json:"client_id"` + AuthURI string `json:"auth_uri"` + TokenURI string `json:"token_uri"` + AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"` + ClientX509CertURL string `json:"client_x509_cert_url"` +} + +func GcpServiceAccountFromProto(sa *protos.GcpServiceAccount) *GcpServiceAccount { + return &GcpServiceAccount{ + Type: sa.AuthType, + ProjectID: sa.ProjectId, + PrivateKeyID: sa.PrivateKeyId, + PrivateKey: sa.PrivateKey, + ClientEmail: sa.ClientEmail, + ClientID: sa.ClientId, + AuthURI: sa.AuthUri, + TokenURI: sa.TokenUri, + AuthProviderX509CertURL: sa.AuthProviderX509CertUrl, + ClientX509CertURL: sa.ClientX509CertUrl, + } +} + +// Validates a GcpServiceAccount, that none of the fields are empty. +func (sa *GcpServiceAccount) Validate() error { + v := reflect.ValueOf(*sa) + for i := range v.NumField() { + if v.Field(i).String() == "" { + return fmt.Errorf("field %s is empty", v.Type().Field(i).Name) + } + } + return nil +} + +// Return GcpServiceAccount as JSON byte array +func (sa *GcpServiceAccount) ToJSON() ([]byte, error) { + return json.Marshal(sa) +} + +// CreateBigQueryClient creates a new BigQuery client from a GcpServiceAccount. +func (sa *GcpServiceAccount) CreateBigQueryClient(ctx context.Context) (*bigquery.Client, error) { + saJSON, err := sa.ToJSON() + if err != nil { + return nil, fmt.Errorf("failed to get json: %v", err) + } + + client, err := bigquery.NewClient( + ctx, + sa.ProjectID, + option.WithCredentialsJSON(saJSON), + ) + if err != nil { + return nil, fmt.Errorf("failed to create BigQuery client: %v", err) + } + + return client, nil +} + +// CreateStorageClient creates a new Storage client from a GcpServiceAccount. +func (sa *GcpServiceAccount) CreateStorageClient(ctx context.Context) (*storage.Client, error) { + saJSON, err := sa.ToJSON() + if err != nil { + return nil, fmt.Errorf("failed to get json: %v", err) + } + + client, err := storage.NewClient( + ctx, + option.WithCredentialsJSON(saJSON), + ) + if err != nil { + return nil, fmt.Errorf("failed to create Storage client: %v", err) + } + + return client, nil +} + +// CreatePubSubClient creates a new PubSub client from a GcpServiceAccount. +func (sa *GcpServiceAccount) CreatePubSubClient(ctx context.Context) (*pubsub.Client, error) { + saJSON, err := sa.ToJSON() + if err != nil { + return nil, fmt.Errorf("failed to get json: %v", err) + } + + client, err := pubsub.NewClient( + ctx, + sa.ProjectID, + option.WithCredentialsJSON(saJSON), + ) + if err != nil { + return nil, fmt.Errorf("failed to create BigQuery client: %v", err) + } + + return client, nil +} diff --git a/flow/connectors/utils/lua.go b/flow/connectors/utils/lua.go new file mode 100644 index 0000000000..2acd8821b9 --- /dev/null +++ b/flow/connectors/utils/lua.go @@ -0,0 +1,74 @@ +package utils + +import ( + "context" + "errors" + "fmt" + + "github.com/yuin/gopher-lua" + + "github.com/PeerDB-io/gluaflatbuffers" + "github.com/PeerDB-io/peer-flow/pua" + "github.com/PeerDB-io/peer-flow/shared" +) + +var errEmptyScript = errors.New("mirror must have script") + +func LVAsReadOnlyBytes(ls *lua.LState, v lua.LValue) ([]byte, error) { + str, err := LVAsStringOrNil(ls, v) + if err != nil { + return nil, err + } else if str == "" { + return nil, nil + } else { + return shared.UnsafeFastStringToReadOnlyBytes(str), nil + } +} + +func LVAsStringOrNil(ls *lua.LState, v lua.LValue) (string, error) { + if lstr, ok := v.(lua.LString); ok { + return string(lstr), nil + } else if v == lua.LNil { + return "", nil + } else { + return "", fmt.Errorf("invalid bytes, must be nil or string: %s", v) + } +} + +func LoadScript(ctx context.Context, script string, printfn lua.LGFunction) (*lua.LState, error) { + if script == "" { + return nil, errEmptyScript + } + + ls := lua.NewState(lua.Options{SkipOpenLibs: true}) + ls.SetContext(ctx) + for _, pair := range []struct { + n string + f lua.LGFunction + }{ + {lua.LoadLibName, lua.OpenPackage}, // Must be first + {lua.BaseLibName, lua.OpenBase}, + {lua.TabLibName, lua.OpenTable}, + {lua.StringLibName, lua.OpenString}, + {lua.MathLibName, lua.OpenMath}, + } { + ls.Push(ls.NewFunction(pair.f)) + ls.Push(lua.LString(pair.n)) + err := ls.PCall(1, 0, nil) + if err != nil { + return nil, fmt.Errorf("failed to initialize Lua runtime: %w", err) + } + } + ls.PreloadModule("flatbuffers", gluaflatbuffers.Loader) + pua.RegisterTypes(ls) + ls.Env.RawSetString("print", ls.NewFunction(printfn)) + err := ls.GPCall(pua.LoadPeerdbScript, lua.LString(script)) + if err != nil { + return nil, fmt.Errorf("error loading script %s: %w", script, err) + } + err = ls.PCall(0, 0, nil) + if err != nil { + return nil, fmt.Errorf("error executing script %s: %w", script, err) + } + return ls, nil +} diff --git a/flow/dynamicconf/dynamicconf.go b/flow/dynamicconf/dynamicconf.go index 5225c400b1..080ae4ec00 100644 --- a/flow/dynamicconf/dynamicconf.go +++ b/flow/dynamicconf/dynamicconf.go @@ -8,8 +8,8 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" - utils "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/logger" + "github.com/PeerDB-io/peer-flow/peerdbenv" ) func dynamicConfKeyExists(ctx context.Context, conn *pgxpool.Pool, key string) bool { @@ -25,7 +25,7 @@ func dynamicConfKeyExists(ctx context.Context, conn *pgxpool.Pool, key string) b } func dynamicConfUint32(ctx context.Context, key string, defaultValue uint32) uint32 { - conn, err := utils.GetCatalogConnectionPoolFromEnv(ctx) + conn, err := peerdbenv.GetCatalogConnectionPoolFromEnv(ctx) if err != nil { logger.LoggerFromCtx(ctx).Error("Failed to get catalog connection pool: %v", err) return defaultValue diff --git a/flow/e2e/bigquery/peer_flow_bq_test.go b/flow/e2e/bigquery/peer_flow_bq_test.go index c5c4170fbd..51ba631e60 100644 --- a/flow/e2e/bigquery/peer_flow_bq_test.go +++ b/flow/e2e/bigquery/peer_flow_bq_test.go @@ -12,11 +12,11 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/require" - "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/e2e" "github.com/PeerDB-io/peer-flow/e2eshared" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model/qvalue" + "github.com/PeerDB-io/peer-flow/shared" peerflow "github.com/PeerDB-io/peer-flow/workflows" ) @@ -388,7 +388,7 @@ func (s PeerFlowE2ETestSuiteBQ) Test_Types_BQ() { createMoodEnum := "CREATE TYPE mood AS ENUM ('happy', 'sad', 'angry');" var pgErr *pgconn.PgError _, enumErr := s.Conn().Exec(context.Background(), createMoodEnum) - if errors.As(enumErr, &pgErr) && pgErr.Code != pgerrcode.DuplicateObject && !utils.IsUniqueError(pgErr) { + if errors.As(enumErr, &pgErr) && pgErr.Code != pgerrcode.DuplicateObject && !shared.IsUniqueError(pgErr) { require.NoError(s.t, enumErr) } diff --git a/flow/e2e/congen.go b/flow/e2e/congen.go index 2fd3c180e5..6d19b48d4e 100644 --- a/flow/e2e/congen.go +++ b/flow/e2e/congen.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/require" connpostgres "github.com/PeerDB-io/peer-flow/connectors/postgres" - "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/peerdbenv" ) func cleanPostgres(conn *pgx.Conn, suffix string) error { @@ -106,7 +106,7 @@ func setupPostgresSchema(t *testing.T, conn *pgx.Conn, suffix string) error { func SetupPostgres(t *testing.T, suffix string) (*connpostgres.PostgresConnector, error) { t.Helper() - connector, err := connpostgres.NewPostgresConnector(context.Background(), utils.GetCatalogPostgresConfigFromEnv()) + connector, err := connpostgres.NewPostgresConnector(context.Background(), peerdbenv.GetCatalogPostgresConfigFromEnv()) if err != nil { return nil, fmt.Errorf("failed to create postgres connection: %w", err) } @@ -155,7 +155,7 @@ func GeneratePostgresPeer() *protos.Peer { Name: "test_postgres_peer", Type: protos.DBType_POSTGRES, Config: &protos.Peer_PostgresConfig{ - PostgresConfig: utils.GetCatalogPostgresConfigFromEnv(), + PostgresConfig: peerdbenv.GetCatalogPostgresConfigFromEnv(), }, } } diff --git a/flow/e2e/kafka/kafka_test.go b/flow/e2e/kafka/kafka_test.go index f35e0f568a..f055393545 100644 --- a/flow/e2e/kafka/kafka_test.go +++ b/flow/e2e/kafka/kafka_test.go @@ -95,9 +95,10 @@ func (s KafkaSuite) TestSimple() { ('e2e_kasimple', 'lua', 'function onRecord(r) return r.row and r.row.val end') on conflict do nothing`) require.NoError(s.t, err) + flowName := e2e.AddSuffix(s, "kasimple") connectionGen := e2e.FlowConnectionGenerationConfig{ - FlowJobName: e2e.AddSuffix(s, "kasimple"), - TableNameMapping: map[string]string{srcTableName: "katest"}, + FlowJobName: flowName, + TableNameMapping: map[string]string{srcTableName: flowName}, Destination: s.Peer(), } flowConnConfig := connectionGen.GenerateFlowConnectionConfigs() @@ -115,17 +116,18 @@ func (s KafkaSuite) TestSimple() { e2e.EnvWaitFor(s.t, env, 3*time.Minute, "normalize insert", func() bool { kafka, err := kgo.NewClient( kgo.SeedBrokers("localhost:9092"), - kgo.ConsumeTopics("katest"), + kgo.ConsumeTopics(flowName), ) if err != nil { return false } + defer kafka.Close() ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() fetches := kafka.PollFetches(ctx) fetches.EachTopic(func(ft kgo.FetchTopic) { - require.Equal(s.t, "katest", ft.Topic) + require.Equal(s.t, flowName, ft.Topic) ft.EachRecord(func(r *kgo.Record) { require.Equal(s.t, "testval", string(r.Value)) }) diff --git a/flow/e2e/postgres/peer_flow_pg_test.go b/flow/e2e/postgres/peer_flow_pg_test.go index eeec5e373d..74413a949b 100644 --- a/flow/e2e/postgres/peer_flow_pg_test.go +++ b/flow/e2e/postgres/peer_flow_pg_test.go @@ -13,7 +13,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/e2e" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" @@ -160,7 +159,7 @@ func (s PeerFlowE2ETestSuitePG) Test_Enums_PG() { createMoodEnum := "CREATE TYPE mood AS ENUM ('happy', 'sad', 'angry');" var pgErr *pgconn.PgError _, enumErr := s.Conn().Exec(context.Background(), createMoodEnum) - if errors.As(enumErr, &pgErr) && pgErr.Code != pgerrcode.DuplicateObject && !utils.IsUniqueError(enumErr) { + if errors.As(enumErr, &pgErr) && pgErr.Code != pgerrcode.DuplicateObject && !shared.IsUniqueError(enumErr) { require.NoError(s.t, enumErr) } _, err := s.Conn().Exec(context.Background(), fmt.Sprintf(` diff --git a/flow/e2e/pubsub/pubsub_test.go b/flow/e2e/pubsub/pubsub_test.go new file mode 100644 index 0000000000..c989f33aee --- /dev/null +++ b/flow/e2e/pubsub/pubsub_test.go @@ -0,0 +1,239 @@ +package e2e_kafka + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "cloud.google.com/go/pubsub" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" + + peer_bq "github.com/PeerDB-io/peer-flow/connectors/bigquery" + connpostgres "github.com/PeerDB-io/peer-flow/connectors/postgres" + "github.com/PeerDB-io/peer-flow/connectors/utils" + "github.com/PeerDB-io/peer-flow/e2e" + "github.com/PeerDB-io/peer-flow/e2eshared" + "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/shared" + peerflow "github.com/PeerDB-io/peer-flow/workflows" +) + +type PubSubSuite struct { + t *testing.T + conn *connpostgres.PostgresConnector + suffix string +} + +func (s PubSubSuite) T() *testing.T { + return s.t +} + +func (s PubSubSuite) Connector() *connpostgres.PostgresConnector { + return s.conn +} + +func (s PubSubSuite) Conn() *pgx.Conn { + return s.Connector().Conn() +} + +func (s PubSubSuite) Suffix() string { + return s.suffix +} + +func ServiceAccount() (*utils.GcpServiceAccount, error) { + jsonPath := os.Getenv("TEST_BQ_CREDS") + if jsonPath == "" { + return nil, errors.New("TEST_BQ_CREDS env var not set") + } + + content, err := e2eshared.ReadFileToBytes(jsonPath) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + + var config *protos.BigqueryConfig + err = json.Unmarshal(content, &config) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal json: %w", err) + } + + return peer_bq.NewBigQueryServiceAccount(config) +} + +func (s PubSubSuite) Peer(name string, sa *utils.GcpServiceAccount) *protos.Peer { + return &protos.Peer{ + Name: e2e.AddSuffix(s, name), + Type: protos.DBType_PUBSUB, + Config: &protos.Peer_PubsubConfig{ + PubsubConfig: &protos.PubSubConfig{ + ServiceAccount: &protos.GcpServiceAccount{ + AuthType: sa.Type, + ProjectId: sa.ProjectID, + PrivateKeyId: sa.PrivateKeyID, + PrivateKey: sa.PrivateKey, + ClientEmail: sa.ClientEmail, + ClientId: sa.ClientID, + AuthUri: sa.AuthURI, + TokenUri: sa.TokenURI, + AuthProviderX509CertUrl: sa.AuthProviderX509CertURL, + ClientX509CertUrl: sa.ClientX509CertURL, + }, + }, + }, + } +} + +func (s PubSubSuite) DestinationTable(table string) string { + return table +} + +func (s PubSubSuite) Teardown() { + e2e.TearDownPostgres(s) +} + +func SetupSuite(t *testing.T) PubSubSuite { + t.Helper() + + suffix := "ps_" + strings.ToLower(shared.RandomString(8)) + conn, err := e2e.SetupPostgres(t, suffix) + require.NoError(t, err, "failed to setup postgres") + + return PubSubSuite{ + t: t, + conn: conn, + suffix: suffix, + } +} + +func Test_PubSub(t *testing.T) { + e2eshared.RunSuite(t, SetupSuite) +} + +func (s PubSubSuite) TestCreateTopic() { + srcTableName := e2e.AttachSchema(s, "pscreate") + + _, err := s.Conn().Exec(context.Background(), fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id SERIAL PRIMARY KEY, + val text + ); + `, srcTableName)) + require.NoError(s.t, err) + + sa, err := ServiceAccount() + require.NoError(s.t, err) + + _, err = s.Conn().Exec(context.Background(), `insert into public.scripts (name, lang, source) values + ('e2e_pscreate', 'lua', 'function onRecord(r) return r.row and r.row.val end') on conflict do nothing`) + require.NoError(s.t, err) + + flowName := e2e.AddSuffix(s, "e2epscreate") + connectionGen := e2e.FlowConnectionGenerationConfig{ + FlowJobName: flowName, + TableNameMapping: map[string]string{srcTableName: flowName}, + Destination: s.Peer(flowName, sa), + } + flowConnConfig := connectionGen.GenerateFlowConnectionConfigs() + flowConnConfig.Script = "e2e_pscreate" + + tc := e2e.NewTemporalClient(s.t) + env := e2e.ExecutePeerflow(tc, peerflow.CDCFlowWorkflow, flowConnConfig, nil) + e2e.SetupCDCFlowStatusQuery(s.t, env, connectionGen) + + _, err = s.Conn().Exec(context.Background(), fmt.Sprintf(` + INSERT INTO %s (id, val) VALUES (1, 'testval') + `, srcTableName)) + require.NoError(s.t, err) + + e2e.EnvWaitFor(s.t, env, 3*time.Minute, "create topic", func() bool { + psclient, err := sa.CreatePubSubClient(context.Background()) + defer func() { + _ = psclient.Close() + }() + require.NoError(s.t, err) + topic := psclient.Topic(flowName) + exists, err := topic.Exists(context.Background()) + s.t.Log("WWWW exists", exists) + require.NoError(s.t, err) + return exists + }) + + env.Cancel() + e2e.RequireEnvCanceled(s.t, env) +} + +func (s PubSubSuite) TestSimple() { + srcTableName := e2e.AttachSchema(s, "pssimple") + + _, err := s.Conn().Exec(context.Background(), fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id SERIAL PRIMARY KEY, + val text + ); + `, srcTableName)) + require.NoError(s.t, err) + + sa, err := ServiceAccount() + require.NoError(s.t, err) + + _, err = s.Conn().Exec(context.Background(), `insert into public.scripts (name, lang, source) values + ('e2e_pssimple', 'lua', 'function onRecord(r) return r.row and r.row.val end') on conflict do nothing`) + require.NoError(s.t, err) + + flowName := e2e.AddSuffix(s, "e2epssimple") + connectionGen := e2e.FlowConnectionGenerationConfig{ + FlowJobName: flowName, + TableNameMapping: map[string]string{srcTableName: flowName}, + Destination: s.Peer(flowName, sa), + } + flowConnConfig := connectionGen.GenerateFlowConnectionConfigs() + flowConnConfig.Script = "e2e_pssimple" + + psclient, err := sa.CreatePubSubClient(context.Background()) + require.NoError(s.t, err) + defer psclient.Close() + topic, err := psclient.CreateTopic(context.Background(), flowName) + require.NoError(s.t, err) + sub, err := psclient.CreateSubscription(context.Background(), flowName, pubsub.SubscriptionConfig{ + Topic: topic, + RetentionDuration: 10 * time.Minute, + ExpirationPolicy: 24 * time.Hour, + }) + require.NoError(s.t, err) + + tc := e2e.NewTemporalClient(s.t) + env := e2e.ExecutePeerflow(tc, peerflow.CDCFlowWorkflow, flowConnConfig, nil) + e2e.SetupCDCFlowStatusQuery(s.t, env, connectionGen) + + _, err = s.Conn().Exec(context.Background(), fmt.Sprintf(` + INSERT INTO %s (id, val) VALUES (1, 'testval') + `, srcTableName)) + require.NoError(s.t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + + msgs := make(chan *pubsub.Message) + go func() { + _ = sub.Receive(ctx, func(_ context.Context, m *pubsub.Message) { + msgs <- m + }) + }() + select { + case msg := <-msgs: + require.NotNil(s.t, msg) + require.Equal(s.t, "testval", string(msg.Data)) + case <-ctx.Done(): + s.t.Log("UNEXPECTED TIMEOUT PubSub subscription waiting on message") + s.t.Fail() + } + + env.Cancel() + e2e.RequireEnvCanceled(s.t, env) +} diff --git a/flow/e2e/snowflake/peer_flow_sf_test.go b/flow/e2e/snowflake/peer_flow_sf_test.go index 27e6156245..f75730658b 100644 --- a/flow/e2e/snowflake/peer_flow_sf_test.go +++ b/flow/e2e/snowflake/peer_flow_sf_test.go @@ -11,10 +11,10 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/require" - "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/e2e" "github.com/PeerDB-io/peer-flow/e2eshared" "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/shared" peerflow "github.com/PeerDB-io/peer-flow/workflows" ) @@ -389,7 +389,7 @@ func (s PeerFlowE2ETestSuiteSF) Test_Types_SF() { createMoodEnum := "CREATE TYPE mood AS ENUM ('happy', 'sad', 'angry');" var pgErr *pgconn.PgError _, enumErr := s.Conn().Exec(context.Background(), createMoodEnum) - if errors.As(enumErr, &pgErr) && pgErr.Code != pgerrcode.DuplicateObject && !utils.IsUniqueError(pgErr) { + if errors.As(enumErr, &pgErr) && pgErr.Code != pgerrcode.DuplicateObject && !shared.IsUniqueError(pgErr) { require.NoError(s.t, enumErr) } _, err := s.Conn().Exec(context.Background(), fmt.Sprintf(` diff --git a/flow/e2e/test_utils.go b/flow/e2e/test_utils.go index d2bcef6acb..f653c4732e 100644 --- a/flow/e2e/test_utils.go +++ b/flow/e2e/test_utils.go @@ -25,12 +25,12 @@ import ( "github.com/PeerDB-io/peer-flow/connectors" connpostgres "github.com/PeerDB-io/peer-flow/connectors/postgres" connsnowflake "github.com/PeerDB-io/peer-flow/connectors/snowflake" - "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/e2eshared" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/qvalue" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" peerflow "github.com/PeerDB-io/peer-flow/workflows" ) @@ -280,7 +280,7 @@ func CreateTableForQRep(conn *pgx.Conn, suffix string, tableName string) error { tblFieldStr := strings.Join(tblFields, ",") var pgErr *pgconn.PgError _, enumErr := conn.Exec(context.Background(), createMoodEnum) - if errors.As(enumErr, &pgErr) && pgErr.Code != pgerrcode.DuplicateObject && !utils.IsUniqueError(pgErr) { + if errors.As(enumErr, &pgErr) && pgErr.Code != pgerrcode.DuplicateObject && !shared.IsUniqueError(pgErr) { return enumErr } _, err := conn.Exec(context.Background(), fmt.Sprintf(` @@ -568,7 +568,7 @@ func ExecutePeerflow(tc client.Client, wf interface{}, args ...interface{}) Work } func ExecuteWorkflow(tc client.Client, taskQueueID shared.TaskQueueID, wf interface{}, args ...interface{}) WorkflowRun { - taskQueue := shared.GetPeerFlowTaskQueueName(taskQueueID) + taskQueue := peerdbenv.PeerFlowTaskQueueName(taskQueueID) wr, err := tc.ExecuteWorkflow( context.Background(), diff --git a/flow/go.mod b/flow/go.mod index 5a5fb83094..ad4d89786f 100644 --- a/flow/go.mod +++ b/flow/go.mod @@ -5,6 +5,7 @@ go 1.22.0 require ( cloud.google.com/go v0.112.1 cloud.google.com/go/bigquery v1.59.1 + cloud.google.com/go/pubsub v1.37.0 cloud.google.com/go/storage v1.39.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.0.4 diff --git a/flow/go.sum b/flow/go.sum index aacfa2da10..98ed8f42cc 100644 --- a/flow/go.sum +++ b/flow/go.sum @@ -13,6 +13,8 @@ cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= +cloud.google.com/go/pubsub v1.37.0 h1:0uEEfaB1VIJzabPpwpZf44zWAKAme3zwKKxHk7vJQxQ= +cloud.google.com/go/pubsub v1.37.0/go.mod h1:YQOQr1uiUM092EXwKs56OPT650nwnawc+8/IjoUeGzQ= cloud.google.com/go/storage v1.39.1 h1:MvraqHKhogCOTXTlct/9C3K3+Uy2jBmFYb3/Sp6dVtY= cloud.google.com/go/storage v1.39.1/go.mod h1:xK6xZmxZmo+fyP7+DEF6FhNc24/JAe95OLyOHCXFH1o= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= diff --git a/flow/connectors/utils/catalog/env.go b/flow/peerdbenv/catalog.go similarity index 68% rename from flow/connectors/utils/catalog/env.go rename to flow/peerdbenv/catalog.go index 5e2cdc109c..1014e83060 100644 --- a/flow/connectors/utils/catalog/env.go +++ b/flow/peerdbenv/catalog.go @@ -1,4 +1,4 @@ -package utils +package peerdbenv import ( "context" @@ -7,9 +7,8 @@ import ( "github.com/jackc/pgx/v5/pgxpool" - "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" - "github.com/PeerDB-io/peer-flow/peerdbenv" + "github.com/PeerDB-io/peer-flow/shared" ) var ( @@ -39,15 +38,15 @@ func GetCatalogConnectionPoolFromEnv(ctx context.Context) (*pgxpool.Pool, error) } func GetCatalogConnectionStringFromEnv() string { - return utils.GetPGConnectionString(GetCatalogPostgresConfigFromEnv()) + return shared.GetPGConnectionString(GetCatalogPostgresConfigFromEnv()) } func GetCatalogPostgresConfigFromEnv() *protos.PostgresConfig { return &protos.PostgresConfig{ - Host: peerdbenv.PeerDBCatalogHost(), - Port: uint32(peerdbenv.PeerDBCatalogPort()), - User: peerdbenv.PeerDBCatalogUser(), - Password: peerdbenv.PeerDBCatalogPassword(), - Database: peerdbenv.PeerDBCatalogDatabase(), + Host: PeerDBCatalogHost(), + Port: uint32(PeerDBCatalogPort()), + User: PeerDBCatalogUser(), + Password: PeerDBCatalogPassword(), + Database: PeerDBCatalogDatabase(), } } diff --git a/flow/peerdbenv/config.go b/flow/peerdbenv/config.go index 1d0eacc01a..3f447e4ea8 100644 --- a/flow/peerdbenv/config.go +++ b/flow/peerdbenv/config.go @@ -1,7 +1,10 @@ package peerdbenv import ( + "fmt" "time" + + "github.com/PeerDB-io/peer-flow/shared" ) // This file contains functions to get the values of various peerdb environment @@ -18,6 +21,14 @@ func PeerDBDeploymentUID() string { return getEnvString("PEERDB_DEPLOYMENT_UID", "") } +func PeerFlowTaskQueueName(taskQueueID shared.TaskQueueID) string { + deploymentUID := PeerDBDeploymentUID() + if deploymentUID == "" { + return string(taskQueueID) + } + return fmt.Sprintf("%s-%s", deploymentUID, taskQueueID) +} + // PEERDB_CDC_CHANNEL_BUFFER_SIZE func PeerDBCDCChannelBufferSize() int { return getEnvInt("PEERDB_CDC_CHANNEL_BUFFER_SIZE", 1<<18) diff --git a/flow/peerdbenv/env.go b/flow/peerdbenv/env.go index 862ae07eb3..b4c4b8d0ac 100644 --- a/flow/peerdbenv/env.go +++ b/flow/peerdbenv/env.go @@ -2,6 +2,7 @@ package peerdbenv import ( "os" + "reflect" "strconv" "golang.org/x/exp/constraints" @@ -32,7 +33,7 @@ func getEnvUint[T constraints.Unsigned](name string, defaultValue T) T { } // widest bit size, truncate later - i, err := strconv.ParseUint(val, 10, 64) + i, err := strconv.ParseUint(val, 10, int(reflect.TypeFor[T]().Size()*8)) if err != nil { return defaultValue } diff --git a/flow/pua/peerdb.go b/flow/pua/peerdb.go index c8f9f8884c..b1746d75c3 100644 --- a/flow/pua/peerdb.go +++ b/flow/pua/peerdb.go @@ -13,9 +13,9 @@ import ( "github.com/PeerDB-io/glua64" "github.com/PeerDB-io/gluabit32" - "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/qvalue" + "github.com/PeerDB-io/peer-flow/peerdbenv" ) var ( @@ -77,7 +77,7 @@ func RegisterTypes(ls *lua.LState) { func LoadPeerdbScript(ls *lua.LState) int { ctx := ls.Context() name := ls.CheckString(1) - pool, err := utils.GetCatalogConnectionPoolFromEnv(ctx) + pool, err := peerdbenv.GetCatalogConnectionPoolFromEnv(ctx) if err != nil { ls.RaiseError("Connection failed loading %s: %s", name, err.Error()) return 0 diff --git a/flow/shared/constants.go b/flow/shared/constants.go index e6e982c321..3e694ecd91 100644 --- a/flow/shared/constants.go +++ b/flow/shared/constants.go @@ -1,11 +1,5 @@ package shared -import ( - "fmt" - - "github.com/PeerDB-io/peer-flow/peerdbenv" -) - type ( ContextKey string TaskQueueID string @@ -34,15 +28,3 @@ const ( ) const FetchAndChannelSize = 256 * 1024 - -func GetPeerFlowTaskQueueName(taskQueueID TaskQueueID) string { - deploymentUID := peerdbenv.PeerDBDeploymentUID() - if deploymentUID == "" { - return string(taskQueueID) - } - return fmt.Sprintf("%s-%s", deploymentUID, taskQueueID) -} - -func GetDeploymentUID() string { - return peerdbenv.PeerDBDeploymentUID() -} diff --git a/flow/connectors/utils/postgres.go b/flow/shared/postgres.go similarity index 99% rename from flow/connectors/utils/postgres.go rename to flow/shared/postgres.go index 0a606c1feb..2467efedc7 100644 --- a/flow/connectors/utils/postgres.go +++ b/flow/shared/postgres.go @@ -1,4 +1,4 @@ -package utils +package shared import ( "context" diff --git a/flow/shared/string.go b/flow/shared/string.go new file mode 100644 index 0000000000..6f3a5c7f29 --- /dev/null +++ b/flow/shared/string.go @@ -0,0 +1,7 @@ +package shared + +import "unsafe" + +func UnsafeFastStringToReadOnlyBytes(s string) []byte { + return unsafe.Slice(unsafe.StringData(s), len(s)) +} diff --git a/flow/workflows/cdc_flow.go b/flow/workflows/cdc_flow.go index 2bcd21773b..d011acd690 100644 --- a/flow/workflows/cdc_flow.go +++ b/flow/workflows/cdc_flow.go @@ -304,7 +304,7 @@ func CDCFlowWorkflow( // next part of the setup is to snapshot-initial-copy and setup replication slots. snapshotFlowID := GetChildWorkflowID("snapshot-flow", cfg.FlowJobName, originalRunID) - taskQueue := shared.GetPeerFlowTaskQueueName(shared.SnapshotFlowTaskQueue) + taskQueue := peerdbenv.PeerFlowTaskQueueName(shared.SnapshotFlowTaskQueue) childSnapshotFlowOpts := workflow.ChildWorkflowOptions{ WorkflowID: snapshotFlowID, ParentClosePolicy: enums.PARENT_CLOSE_POLICY_REQUEST_CANCEL, diff --git a/flow/workflows/snapshot_flow.go b/flow/workflows/snapshot_flow.go index 8ec844cc4b..3c966b4f3f 100644 --- a/flow/workflows/snapshot_flow.go +++ b/flow/workflows/snapshot_flow.go @@ -17,6 +17,7 @@ import ( connpostgres "github.com/PeerDB-io/peer-flow/connectors/postgres" "github.com/PeerDB-io/peer-flow/connectors/utils" "github.com/PeerDB-io/peer-flow/generated/protos" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" ) @@ -122,7 +123,7 @@ func (s *SnapshotFlowExecution) cloneTable( s.logger.Info(fmt.Sprintf("Obtained child id %s for source table %s and destination table %s", childWorkflowID, srcName, dstName), cloneLog) - taskQueue := shared.GetPeerFlowTaskQueueName(shared.PeerFlowTaskQueue) + taskQueue := peerdbenv.PeerFlowTaskQueueName(shared.PeerFlowTaskQueue) childCtx := workflow.WithChildOptions(ctx, workflow.ChildWorkflowOptions{ WorkflowID: childWorkflowID, WorkflowTaskTimeout: 5 * time.Minute, diff --git a/nexus/analyzer/src/lib.rs b/nexus/analyzer/src/lib.rs index da8cab94d4..0c16269cd4 100644 --- a/nexus/analyzer/src/lib.rs +++ b/nexus/analyzer/src/lib.rs @@ -10,8 +10,9 @@ use anyhow::Context; use pt::{ flow_model::{FlowJob, FlowJobTableMapping, QRepFlowJob}, peerdb_peers::{ - peer::Config, BigqueryConfig, ClickhouseConfig, DbType, EventHubConfig, KafkaConfig, - MongoConfig, Peer, PostgresConfig, S3Config, SnowflakeConfig, SqlServerConfig, + peer::Config, BigqueryConfig, ClickhouseConfig, DbType, EventHubConfig, GcpServiceAccount, + KafkaConfig, MongoConfig, Peer, PostgresConfig, PubSubConfig, S3Config, SnowflakeConfig, + SqlServerConfig, }, }; use qrep::process_options; @@ -248,12 +249,11 @@ impl<'a> StatementAnalyzer for PeerDDLAnalyzer<'a> { Some(sqlparser::ast::Value::Number(n, _)) => Some(n.parse::()?), _ => None, }; - let snapshot_staging_path = match raw_options - .remove("snapshot_staging_path") - { - Some(sqlparser::ast::Value::SingleQuotedString(s)) => s.clone(), - _ => String::new(), - }; + let snapshot_staging_path = + match raw_options.remove("snapshot_staging_path") { + Some(sqlparser::ast::Value::SingleQuotedString(s)) => s.clone(), + _ => String::new(), + }; let snapshot_max_parallel_workers: Option = match raw_options .remove("snapshot_max_parallel_workers") @@ -827,11 +827,7 @@ fn parse_db_options( .split(',') .map(String::from) .collect::>(), - username: opts - .get("user") - .cloned() - .unwrap_or_default() - .to_string(), + username: opts.get("user").cloned().unwrap_or_default().to_string(), password: opts .get("password") .cloned() @@ -854,5 +850,62 @@ fn parse_db_options( }; Config::KafkaConfig(kafka_config) } + DbType::Pubsub => { + let pem_str = opts + .get("private_key") + .ok_or_else(|| anyhow::anyhow!("missing private_key option for bigquery"))?; + pem::parse(pem_str.as_bytes()) + .map_err(|err| anyhow::anyhow!("unable to parse private_key: {:?}", err))?; + let ps_config = PubSubConfig { + service_account: Some(GcpServiceAccount { + auth_type: opts + .get("type") + .ok_or_else(|| anyhow::anyhow!("missing type option for bigquery"))? + .to_string(), + project_id: opts + .get("project_id") + .ok_or_else(|| anyhow::anyhow!("missing project_id in peer options"))? + .to_string(), + private_key_id: opts + .get("private_key_id") + .ok_or_else(|| { + anyhow::anyhow!("missing private_key_id option for bigquery") + })? + .to_string(), + private_key: pem_str.to_string(), + client_email: opts + .get("client_email") + .ok_or_else(|| anyhow::anyhow!("missing client_email option for bigquery"))? + .to_string(), + client_id: opts + .get("client_id") + .ok_or_else(|| anyhow::anyhow!("missing client_id option for bigquery"))? + .to_string(), + auth_uri: opts + .get("auth_uri") + .ok_or_else(|| anyhow::anyhow!("missing auth_uri option for bigquery"))? + .to_string(), + token_uri: opts + .get("token_uri") + .ok_or_else(|| anyhow::anyhow!("missing token_uri option for bigquery"))? + .to_string(), + auth_provider_x509_cert_url: opts + .get("auth_provider_x509_cert_url") + .ok_or_else(|| { + anyhow::anyhow!( + "missing auth_provider_x509_cert_url option for bigquery" + ) + })? + .to_string(), + client_x509_cert_url: opts + .get("client_x509_cert_url") + .ok_or_else(|| { + anyhow::anyhow!("missing client_x509_cert_url option for bigquery") + })? + .to_string(), + }), + }; + Config::PubsubConfig(ps_config) + } })) } diff --git a/nexus/catalog/src/lib.rs b/nexus/catalog/src/lib.rs index 0d8b7ed6a6..79a17a98b1 100644 --- a/nexus/catalog/src/lib.rs +++ b/nexus/catalog/src/lib.rs @@ -101,6 +101,7 @@ impl Catalog { } Config::ClickhouseConfig(clickhouse_config) => clickhouse_config.encode_to_vec(), Config::KafkaConfig(kafka_config) => kafka_config.encode_to_vec(), + Config::PubsubConfig(pubsub_config) => pubsub_config.encode_to_vec(), } }; @@ -312,6 +313,11 @@ impl Catalog { pt::peerdb_peers::KafkaConfig::decode(options).with_context(err)?; Config::KafkaConfig(kafka_config) } + DbType::Pubsub => { + let pubsub_config = + pt::peerdb_peers::PubSubConfig::decode(options).with_context(err)?; + Config::PubsubConfig(pubsub_config) + } }) } else { None diff --git a/protos/peers.proto b/protos/peers.proto index 12fc88bfb3..4fa1978568 100644 --- a/protos/peers.proto +++ b/protos/peers.proto @@ -25,6 +25,19 @@ message SnowflakeConfig { optional string metadata_schema = 11; } +message GcpServiceAccount { + string auth_type = 1; + string project_id = 2; + string private_key_id = 3; + string private_key = 4; + string client_email = 5; + string client_id = 6; + string auth_uri = 7; + string token_uri = 8; + string auth_provider_x509_cert_url = 9; + string client_x509_cert_url = 10; +} + message BigqueryConfig { string auth_type = 1; string project_id = 2; @@ -39,6 +52,10 @@ message BigqueryConfig { string dataset_id = 11; } +message PubSubConfig { + GcpServiceAccount service_account = 1; +} + message MongoConfig { string username = 1; string password = 2; @@ -128,6 +145,7 @@ enum DBType { EVENTHUB_GROUP = 7; CLICKHOUSE = 8; KAFKA = 9; + PUBSUB = 10; } message Peer { @@ -144,5 +162,6 @@ message Peer { EventHubGroupConfig eventhub_group_config = 10; ClickhouseConfig clickhouse_config = 11; KafkaConfig kafka_config = 12; + PubSubConfig pubsub_config = 13; } } diff --git a/ui/app/api/peers/getTruePeer.ts b/ui/app/api/peers/getTruePeer.ts index 93da206854..4c29ffd280 100644 --- a/ui/app/api/peers/getTruePeer.ts +++ b/ui/app/api/peers/getTruePeer.ts @@ -7,6 +7,7 @@ import { KafkaConfig, Peer, PostgresConfig, + PubSubConfig, S3Config, SnowflakeConfig, SqlServerConfig, @@ -25,6 +26,7 @@ export const getTruePeer = (peer: CatalogPeer) => { | EventHubGroupConfig | KafkaConfig | PostgresConfig + | PubSubConfig | S3Config | SnowflakeConfig | SqlServerConfig; @@ -65,6 +67,10 @@ export const getTruePeer = (peer: CatalogPeer) => { config = KafkaConfig.decode(options); newPeer.kafkaConfig = config; break; + case 10: + config = PubSubConfig.decode(options); + newPeer.pubsubConfig = config; + break; default: return newPeer; } diff --git a/ui/app/api/peers/info/[peerName]/route.ts b/ui/app/api/peers/info/[peerName]/route.ts index 61dc033efb..1a1cb6cde3 100644 --- a/ui/app/api/peers/info/[peerName]/route.ts +++ b/ui/app/api/peers/info/[peerName]/route.ts @@ -22,6 +22,7 @@ export async function GET( const ehConfig = peerConfig.eventhubConfig; const chConfig = peerConfig.clickhouseConfig; const kaConfig = peerConfig.kafkaConfig; + const psConfig = peerConfig.pubsubConfig; if (pgConfig) { pgConfig.password = '********'; pgConfig.transactionSnapshot = '********'; @@ -47,6 +48,10 @@ export async function GET( if (kaConfig) { kaConfig.password = '********'; } + if (psConfig?.serviceAccount) { + psConfig.serviceAccount.privateKey = '********'; + psConfig.serviceAccount.privateKeyId = '********'; + } return NextResponse.json(peerConfig); } diff --git a/ui/app/api/peers/route.ts b/ui/app/api/peers/route.ts index 0691212af8..398d834e74 100644 --- a/ui/app/api/peers/route.ts +++ b/ui/app/api/peers/route.ts @@ -13,6 +13,7 @@ import { KafkaConfig, Peer, PostgresConfig, + PubSubConfig, S3Config, SnowflakeConfig, } from '@/grpc_generated/peers'; @@ -70,6 +71,12 @@ const constructPeer = ( type: DBType.KAFKA, kafkaConfig: config as KafkaConfig, }; + case 'PUBSUB': + return { + name, + type: DBType.PUBSUB, + pubsubConfig: config as PubSubConfig, + }; default: return; } diff --git a/ui/app/dto/PeersDTO.ts b/ui/app/dto/PeersDTO.ts index 3e7f5c0b41..3035ea86cb 100644 --- a/ui/app/dto/PeersDTO.ts +++ b/ui/app/dto/PeersDTO.ts @@ -3,6 +3,7 @@ import { ClickhouseConfig, KafkaConfig, PostgresConfig, + PubSubConfig, S3Config, SnowflakeConfig, } from '@/grpc_generated/peers'; @@ -45,7 +46,8 @@ export type PeerConfig = | BigqueryConfig | ClickhouseConfig | S3Config - | KafkaConfig; + | KafkaConfig + | PubSubConfig; export type CatalogPeer = { id: number; name: string; diff --git a/ui/app/peers/create/[peerType]/handlers.ts b/ui/app/peers/create/[peerType]/handlers.ts index 42f84a492a..bc0f071491 100644 --- a/ui/app/peers/create/[peerType]/handlers.ts +++ b/ui/app/peers/create/[peerType]/handlers.ts @@ -11,6 +11,7 @@ import { kaSchema, peerNameSchema, pgSchema, + psSchema, s3Schema, sfSchema, } from './schema'; @@ -62,6 +63,10 @@ const validateFields = ( const kaConfig = kaSchema.safeParse(config); if (!kaConfig.success) validationErr = kaConfig.error.issues[0].message; break; + case 'PUBSUB': + const psConfig = psSchema.safeParse(config); + if (!psConfig.success) validationErr = psConfig.error.issues[0].message; + break; default: validationErr = 'Unsupported peer type ' + type; } diff --git a/ui/app/peers/create/[peerType]/helpers/common.ts b/ui/app/peers/create/[peerType]/helpers/common.ts index e418fd9266..2dac029c00 100644 --- a/ui/app/peers/create/[peerType]/helpers/common.ts +++ b/ui/app/peers/create/[peerType]/helpers/common.ts @@ -1,7 +1,9 @@ import { PeerConfig, PeerSetter } from '@/app/dto/PeersDTO'; import { blankBigquerySetting } from './bq'; import { blankClickhouseSetting } from './ch'; +import { blankKafkaSetting } from './ka'; import { blankPostgresSetting } from './pg'; +import { blankPubSubSetting } from './ps'; import { blankS3Setting } from './s3'; import { blankSnowflakeSetting } from './sf'; @@ -27,6 +29,10 @@ export const getBlankSetting = (dbType: string): PeerConfig => { return blankBigquerySetting; case 'CLICKHOUSE': return blankClickhouseSetting; + case 'PUBSUB': + return blankPubSubSetting; + case 'KAFKA': + return blankKafkaSetting; case 'S3': return blankS3Setting; default: diff --git a/ui/app/peers/create/[peerType]/helpers/ka.ts b/ui/app/peers/create/[peerType]/helpers/ka.ts index 5091d29e7e..193ebaba4d 100644 --- a/ui/app/peers/create/[peerType]/helpers/ka.ts +++ b/ui/app/peers/create/[peerType]/helpers/ka.ts @@ -63,7 +63,7 @@ export const kaSetting: PeerSetting[] = [ }, ]; -export const blankKaSetting: KafkaConfig = { +export const blankKafkaSetting: KafkaConfig = { servers: [], username: '', password: '', diff --git a/ui/app/peers/create/[peerType]/helpers/ps.ts b/ui/app/peers/create/[peerType]/helpers/ps.ts new file mode 100644 index 0000000000..88399d0942 --- /dev/null +++ b/ui/app/peers/create/[peerType]/helpers/ps.ts @@ -0,0 +1,16 @@ +import { PubSubConfig } from '@/grpc_generated/peers'; + +export const blankPubSubSetting: PubSubConfig = { + serviceAccount: { + authType: 'service_account', + projectId: '', + privateKeyId: '', + privateKey: '', + clientEmail: '', + clientId: '', + authUri: '', + tokenUri: '', + authProviderX509CertUrl: '', + clientX509CertUrl: '', + }, +}; diff --git a/ui/app/peers/create/[peerType]/page.tsx b/ui/app/peers/create/[peerType]/page.tsx index bbb25b31ad..ffb4495db4 100644 --- a/ui/app/peers/create/[peerType]/page.tsx +++ b/ui/app/peers/create/[peerType]/page.tsx @@ -5,6 +5,7 @@ import BigqueryForm from '@/components/PeerForms/BigqueryConfig'; import ClickhouseForm from '@/components/PeerForms/ClickhouseConfig'; import KafkaForm from '@/components/PeerForms/KafkaConfig'; import PostgresForm from '@/components/PeerForms/PostgresForm'; +import PubSubForm from '@/components/PeerForms/PubSubConfig'; import S3Form from '@/components/PeerForms/S3Form'; import SnowflakeForm from '@/components/PeerForms/SnowflakeForm'; @@ -84,6 +85,8 @@ export default function CreateConfig({ return ; case 'KAFKA': return ; + case 'PUBSUB': + return ; default: return <>; } diff --git a/ui/app/peers/create/[peerType]/schema.ts b/ui/app/peers/create/[peerType]/schema.ts index 117b8738fe..687c5e2174 100644 --- a/ui/app/peers/create/[peerType]/schema.ts +++ b/ui/app/peers/create/[peerType]/schema.ts @@ -351,3 +351,73 @@ export const s3Schema = z.object({ }) .optional(), }); + +export const psSchema = z.object({ + serviceAccount: z.object({ + authType: z + .string({ + required_error: 'Auth Type is required', + invalid_type_error: 'Auth Type must be a string', + }) + .min(1, { message: 'Auth Type must be non-empty' }) + .max(255, 'Auth Type must be less than 255 characters'), + projectId: z + .string({ + required_error: 'Project ID is required', + invalid_type_error: 'Project ID must be a string', + }) + .min(1, { message: 'Project ID must be non-empty' }), + privateKeyId: z + .string({ + required_error: 'Private Key ID is required', + invalid_type_error: 'Private Key ID must be a string', + }) + .min(1, { message: 'Private Key must be non-empty' }), + privateKey: z + .string({ + required_error: 'Private Key is required', + invalid_type_error: 'Private Key must be a string', + }) + .min(1, { message: 'Private Key must be non-empty' }), + clientId: z + .string({ + required_error: 'Client ID is required', + invalid_type_error: 'Client ID must be a string', + }) + .min(1, { message: 'Client ID must be non-empty' }), + clientEmail: z + .string({ + required_error: 'Client Email is required', + invalid_type_error: 'Client Email must be a string', + }) + .min(1, { message: 'Client Email must be non-empty' }), + authUri: z + .string({ + required_error: 'Auth URI is required', + invalid_type_error: 'Auth URI must be a string', + }) + .url({ message: 'Invalid auth URI' }) + .min(1, { message: 'Auth URI must be non-empty' }), + tokenUri: z + .string({ + required_error: 'Token URI is required', + invalid_type_error: 'Token URI must be a string', + }) + .url({ message: 'Invalid token URI' }) + .min(1, { message: 'Token URI must be non-empty' }), + authProviderX509CertUrl: z + .string({ + invalid_type_error: 'Auth Cert URL must be a string', + required_error: 'Auth Cert URL is required', + }) + .url({ message: 'Invalid auth cert URL' }) + .min(1, { message: 'Auth Cert URL must be non-empty' }), + clientX509CertUrl: z + .string({ + invalid_type_error: 'Client Cert URL must be a string', + required_error: 'Client Cert URL is required', + }) + .url({ message: 'Invalid client cert URL' }) + .min(1, { message: 'Client Cert URL must be non-empty' }), + }), +}); diff --git a/ui/app/peers/peersTable.tsx b/ui/app/peers/peersTable.tsx index a592d232f3..6a3c77d149 100644 --- a/ui/app/peers/peersTable.tsx +++ b/ui/app/peers/peersTable.tsx @@ -56,8 +56,9 @@ function PeersTable({ peers }: { peers: Peer[] }) { const availableTypes: { value: DBType | undefined; label: string }[] = Array.from( new Map( // Map filters out duplicates - peers.flatMap((peer) => [ - [peer.type, { value: peer.type, label: DBTypeToGoodText(peer.type) }], + peers.map((peer) => [ + peer.type, + { value: peer.type, label: DBTypeToGoodText(peer.type) }, ]) ).values() ); diff --git a/ui/components/PeerComponent.tsx b/ui/components/PeerComponent.tsx index 8505655c72..78c8db0d2d 100644 --- a/ui/components/PeerComponent.tsx +++ b/ui/components/PeerComponent.tsx @@ -35,6 +35,9 @@ export const DBTypeToImageMapping = (peerType: DBType | string) => { case DBType.KAFKA: case 'KAFKA': return '/svgs/kafka.svg'; + case DBType.PUBSUB: + case 'PUBSUB': + return '/svgs/pubsub.svg'; default: return '/svgs/pg.svg'; } diff --git a/ui/components/PeerForms/BigqueryConfig.tsx b/ui/components/PeerForms/BigqueryConfig.tsx index 9b423de320..eebf7f60dd 100644 --- a/ui/components/PeerForms/BigqueryConfig.tsx +++ b/ui/components/PeerForms/BigqueryConfig.tsx @@ -29,7 +29,7 @@ export default function BigqueryForm(props: BQProps) { return; } const bqConfig: BigqueryConfig = { - authType: bqJson.type, + authType: bqJson.type ?? bqJson.auth_type, projectId: bqJson.project_id, privateKeyId: bqJson.private_key_id, privateKey: bqJson.private_key, diff --git a/ui/components/PeerForms/PubSubConfig.tsx b/ui/components/PeerForms/PubSubConfig.tsx new file mode 100644 index 0000000000..089f9eb0f7 --- /dev/null +++ b/ui/components/PeerForms/PubSubConfig.tsx @@ -0,0 +1,92 @@ +'use client'; +import { PeerSetter } from '@/app/dto/PeersDTO'; +import { blankPubSubSetting } from '@/app/peers/create/[peerType]/helpers/ps'; +import { PubSubConfig } from '@/grpc_generated/peers'; +import { Label } from '@/lib/Label'; +import { RowWithTextField } from '@/lib/Layout'; +import { TextField } from '@/lib/TextField'; +import { Tooltip } from '@/lib/Tooltip'; +import Link from 'next/link'; +import { useState } from 'react'; + +interface PSProps { + setter: PeerSetter; +} +export default function PubSubForm(props: PSProps) { + const [datasetID, setDatasetID] = useState(''); + const handleJSONFile = (file: File) => { + if (file) { + const reader = new FileReader(); + reader.readAsText(file); + reader.onload = () => { + // Read the file as JSON + let psJson; + try { + psJson = JSON.parse(reader.result as string); + } catch (err) { + props.setter(blankPubSubSetting); + return; + } + const psConfig: PubSubConfig = { + serviceAccount: { + authType: psJson.type ?? psJson.auth_type, + projectId: psJson.project_id, + privateKeyId: psJson.private_key_id, + privateKey: psJson.private_key, + clientEmail: psJson.client_email, + clientId: psJson.client_id, + authUri: psJson.auth_uri, + tokenUri: psJson.token_uri, + authProviderX509CertUrl: psJson.auth_provider_x509_cert_url, + clientX509CertUrl: psJson.client_x509_cert_url, + }, + }; + props.setter(psConfig); + }; + reader.onerror = (error) => { + console.log(error); + }; + } + }; + + return ( + <> + + + Creating a service account file + + + Service Account JSON + + + + + } + action={ + ) => + e.target.files && handleJSONFile(e.target.files[0]) + } + /> + } + /> + + ); +} diff --git a/ui/components/PeerTypeComponent.tsx b/ui/components/PeerTypeComponent.tsx index 2db4be9dd2..e1642cb001 100644 --- a/ui/components/PeerTypeComponent.tsx +++ b/ui/components/PeerTypeComponent.tsx @@ -26,6 +26,8 @@ export const DBTypeToGoodText = (ptype: DBType) => { return 'Clickhouse'; case DBType.KAFKA: return 'Kafka'; + case DBType.PUBSUB: + return 'PubSub'; case DBType.UNRECOGNIZED: return 'Unrecognised'; } diff --git a/ui/components/SelectSource.tsx b/ui/components/SelectSource.tsx index eca7c37e61..4459d3643c 100644 --- a/ui/components/SelectSource.tsx +++ b/ui/components/SelectSource.tsx @@ -35,7 +35,8 @@ export default function SelectSource({ value === 'BIGQUERY' || value === 'S3' || value === 'CLICKHOUSE' || - value === 'KAFKA') + value === 'KAFKA' || + value === 'PUBSUB') ) .map((value) => ({ label: value, value })); diff --git a/ui/public/svgs/pubsub.svg b/ui/public/svgs/pubsub.svg new file mode 100644 index 0000000000..c3686f6e7b --- /dev/null +++ b/ui/public/svgs/pubsub.svg @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + Icon_24px_Pub-Sub_Color + + + + + + + + + + + + + + + \ No newline at end of file