diff --git a/.github/workflows/validate-generated-files.yml b/.github/workflows/validate-generated-files.yml index c29330d29..d4e86151b 100644 --- a/.github/workflows/validate-generated-files.yml +++ b/.github/workflows/validate-generated-files.yml @@ -25,7 +25,5 @@ jobs: - name: Check generated files run: | export PATH=$PATH:$(go env GOPATH)/bin - make install-tools - make generate - make proto-generate + make install-tools generate proto-generate git diff --exit-code --numstat diff --git a/.gitignore b/.gitignore index 85e65191b..db0fb3e42 100644 --- a/.gitignore +++ b/.gitignore @@ -82,4 +82,6 @@ dist/ /conduit.yaml # Escape Analysis Report -escape_analysis.txt \ No newline at end of file +escape_analysis.txt + +pkg/plugin/processor/standalone/test/wasm_processors/*/processor.wasm diff --git a/docs/connectors.md b/docs/connectors.md index 84a43c7bb..ef91afd0d 100644 --- a/docs/connectors.md +++ b/docs/connectors.md @@ -46,7 +46,7 @@ Once you have chosen a connector to be built-in, you can: - Download the new package and its dependencies: `go get "github.com/foo/conduit-connector-new"` - Import the Go module defining the connector -into the [builtin registry](https://github.com/ConduitIO/conduit/blob/main/pkg/plugin/builtin/registry.go) +into the [builtin registry](https://github.com/ConduitIO/conduit/blob/main/pkg/plugin/connector/builtin/registry.go) and add a new key to `DefaultDispenserFactories`: ```diff diff --git a/go.mod b/go.mod index 27ba6c847..e582862b7 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/conduitio/conduit -go 1.21 - -toolchain go1.21.1 +go 1.21.5 require ( buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.31.0-20231027202514-3f42134f4c56.2 @@ -10,6 +8,7 @@ require ( github.com/NYTimes/gziphandler v1.1.1 github.com/antchfx/jsonquery v1.3.3 github.com/bufbuild/buf v1.28.1 + github.com/conduitio/conduit-commons v0.0.0-20231222154604-f6dfae573b45 github.com/conduitio/conduit-connector-file v0.6.0 github.com/conduitio/conduit-connector-generator v0.5.0 github.com/conduitio/conduit-connector-kafka v0.7.0 @@ -18,6 +17,7 @@ require ( github.com/conduitio/conduit-connector-protocol v0.5.0 github.com/conduitio/conduit-connector-s3 v0.5.0 github.com/conduitio/conduit-connector-sdk v0.8.0 + github.com/conduitio/conduit-processor-sdk v0.0.0-20231228195815-5a4e9b618cc8 github.com/conduitio/yaml/v3 v3.3.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dop251/goja v0.0.0-20230531210528-d7324b2d74f7 @@ -27,7 +27,7 @@ require ( github.com/golang/mock v1.6.0 github.com/golangci/golangci-lint v1.55.2 github.com/google/go-cmp v0.6.0 - github.com/google/uuid v1.4.0 + github.com/google/uuid v1.5.0 github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 github.com/hamba/avro/v2 v2.18.0 @@ -44,15 +44,16 @@ require ( github.com/prometheus/client_model v0.5.0 github.com/prometheus/common v0.45.0 github.com/rs/zerolog v1.31.0 + github.com/tetratelabs/wazero v1.5.0 github.com/twmb/go-cache v1.2.0 go.uber.org/goleak v1.3.0 go.uber.org/mock v0.3.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa - golang.org/x/tools v0.16.0 + golang.org/x/tools v0.16.1 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 - google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 - google.golang.org/grpc v1.59.0 - google.golang.org/protobuf v1.31.1-0.20231027082548-f4a6c1f6e5c1 + google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 + google.golang.org/grpc v1.60.0 + google.golang.org/protobuf v1.32.0 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 ) @@ -298,7 +299,6 @@ require ( github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c // indirect github.com/tdakkota/asciicheck v0.2.0 // indirect github.com/tetafro/godot v1.4.15 // indirect - github.com/tetratelabs/wazero v1.5.0 // indirect github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 // indirect github.com/timonwong/loggercheck v0.9.4 // indirect github.com/tomarrell/wrapcheck/v2 v2.8.1 // indirect @@ -335,8 +335,8 @@ require ( golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.4.0 // indirect - google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231211222908-989df2bf70f3 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 22f63bb3e..0c5a4234b 100644 --- a/go.sum +++ b/go.sum @@ -1020,6 +1020,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= +github.com/conduitio/conduit-commons v0.0.0-20231222154604-f6dfae573b45 h1:GJhyDjoyswfJdqKyszQvevOw/xAz6ilNgN8q6T0lDeQ= +github.com/conduitio/conduit-commons v0.0.0-20231222154604-f6dfae573b45/go.mod h1:+bKjNjWYkbc/MMkBFSzND09FxpPD3GqjmPRQMndkvZ0= github.com/conduitio/conduit-connector-file v0.6.0 h1:8tsGeGhKvFwYQZztOOL5/tmOhVShsfo9lQ3b/0fX8kQ= github.com/conduitio/conduit-connector-file v0.6.0/go.mod h1:ju7PiB4kTJgqng4KVXDt/Gvw/53kFwSzi5Ez9EDXxNI= github.com/conduitio/conduit-connector-generator v0.5.0 h1:zpXHif89DCJ13nftKLv31uI2AJGicpY5H1V7SwldRNo= @@ -1036,6 +1038,8 @@ github.com/conduitio/conduit-connector-s3 v0.5.0 h1:nqLcf/foYnDLkXWYcWJX/5UHzTjW github.com/conduitio/conduit-connector-s3 v0.5.0/go.mod h1:I6oE37zz25RTjnQiUBz2rOASwXNqfaMW7gSlsKX6z8E= github.com/conduitio/conduit-connector-sdk v0.8.0 h1:gvchqoj5d3AQsBoIosx4i32L8Ex9+5BuAyHi/IM9VD4= github.com/conduitio/conduit-connector-sdk v0.8.0/go.mod h1:nOz4K3X6fD8YMe5CPbULwSEE18Eu02ZrpT6o6KwQfxs= +github.com/conduitio/conduit-processor-sdk v0.0.0-20231228195815-5a4e9b618cc8 h1:JaaDBQWmB/APPuJ3OQvvN8c8UHaNYCPvUXDs+w1NfUw= +github.com/conduitio/conduit-processor-sdk v0.0.0-20231228195815-5a4e9b618cc8/go.mod h1:vzR6okyqvGBHlnFvBMkeZ1oMj8VaGSPk7mE3eJnndKE= github.com/conduitio/yaml/v3 v3.3.0 h1:kbbaOSHcuH39gP4+rgbJGl6DSbLZcJgEaBvkEXJlCsI= github.com/conduitio/yaml/v3 v3.3.0/go.mod h1:JNgFMOX1t8W4YJuRZOh6GggVtSMsgP9XgTw+7dIenpc= github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= @@ -1340,8 +1344,8 @@ github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= @@ -2423,8 +2427,8 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2661,8 +2665,8 @@ google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02 google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= -google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 h1:I6WNifs6pF9tNdSob2W24JtyxIYjzFB9qDlpUC76q+U= -google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405/go.mod h1:3WDQMjmJk36UQhjQ89emUzb1mdaHcPeeAh4SCBKznB4= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 h1:1hfbdAfFbkmpg41000wDVqr7jUpK/Yo+LPnIxxGzmkg= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3/go.mod h1:5RBcpGRxr25RbDzY5w+dmaqpSEvl8Gwl1x2CICf60ic= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= @@ -2673,8 +2677,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go. google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:RdyHbowztCGQySiCvQPgWQWgWhGnouTdCflKoDBt32U= -google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= -google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= +google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 h1:s1w3X6gQxwrLEpxnLd/qXTVLgQE2yXwaOaoa6IlY/+o= +google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0/go.mod h1:CAny0tYF+0/9rmDB9fahA9YLzX3+AEVl1qXbv5hhj6c= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= @@ -2688,8 +2692,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20230920183334-c177e329c48b/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231211222908-989df2bf70f3 h1:kzJAXnzZoFbe5bhZd4zjUuHos/I31yH4thfMb/13oVY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231211222908-989df2bf70f3/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2736,8 +2740,8 @@ google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpX google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k= +google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2757,8 +2761,8 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.1-0.20231027082548-f4a6c1f6e5c1 h1:fk72uXZyuZiTtW5tgd63jyVK6582lF61nRC/kGv6vCA= -google.golang.org/protobuf v1.31.1-0.20231027082548-f4a6c1f6e5c1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/conduit/config.go b/pkg/conduit/config.go index 75ef704d0..d43ed1d9b 100644 --- a/pkg/conduit/config.go +++ b/pkg/conduit/config.go @@ -20,7 +20,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/database" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/plugin/builtin" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin" "github.com/conduitio/conduit/pkg/processor" "github.com/rs/zerolog" ) diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index fde35b61b..fa29c720f 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -44,8 +44,8 @@ import ( "github.com/conduitio/conduit/pkg/orchestrator" "github.com/conduitio/conduit/pkg/pipeline" "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/builtin" - "github.com/conduitio/conduit/pkg/plugin/standalone" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin" + "github.com/conduitio/conduit/pkg/plugin/connector/standalone" "github.com/conduitio/conduit/pkg/processor" "github.com/conduitio/conduit/pkg/provisioning" "github.com/conduitio/conduit/pkg/web/api" diff --git a/pkg/connector/destination.go b/pkg/connector/destination.go index 6d75fee23..e1bdce52d 100644 --- a/pkg/connector/destination.go +++ b/pkg/connector/destination.go @@ -21,14 +21,15 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/record" ) type Destination struct { Instance *Instance - dispenser plugin.Dispenser - plugin plugin.DestinationPlugin + dispenser connectorPlugin.Dispenser + plugin connectorPlugin.DestinationPlugin // errs is used to signal the node that the connector experienced an error // when it was processing something asynchronously (e.g. persisting state). @@ -37,7 +38,7 @@ type Destination struct { // stopStream is a function that closes the context of the stream stopStream context.CancelFunc - // wg tracks the number of in flight calls to the plugin. + // wg tracks the number of in flight calls to the connectorPlugin. wg sync.WaitGroup } diff --git a/pkg/connector/destination_test.go b/pkg/connector/destination_test.go index b4f1a5755..4343b811b 100644 --- a/pkg/connector/destination_test.go +++ b/pkg/connector/destination_test.go @@ -22,7 +22,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/database/inmemory" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/mock" + "github.com/conduitio/conduit/pkg/plugin/connector/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/connector/instance.go b/pkg/connector/instance.go index 2dfa7e334..656e4442b 100644 --- a/pkg/connector/instance.go +++ b/pkg/connector/instance.go @@ -24,7 +24,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/inspector" - "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" ) const ( @@ -86,7 +86,7 @@ type Connector interface { // PluginDispenserFetcher can fetch a plugin dispenser. type PluginDispenserFetcher interface { - NewDispenser(logger log.CtxLogger, name string) (plugin.Dispenser, error) + NewDispenser(logger log.CtxLogger, name string) (connectorPlugin.Dispenser, error) } func (i *Instance) Init(logger log.CtxLogger, persister *Persister) { diff --git a/pkg/connector/instance_test.go b/pkg/connector/instance_test.go index 16852543c..aa755daae 100644 --- a/pkg/connector/instance_test.go +++ b/pkg/connector/instance_test.go @@ -17,12 +17,13 @@ package connector import ( "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" ) // fakePluginFetcher fulfills the PluginFetcher interface. -type fakePluginFetcher map[string]plugin.Dispenser +type fakePluginFetcher map[string]connectorPlugin.Dispenser -func (fpf fakePluginFetcher) NewDispenser(_ log.CtxLogger, name string) (plugin.Dispenser, error) { +func (fpf fakePluginFetcher) NewDispenser(_ log.CtxLogger, name string) (connectorPlugin.Dispenser, error) { plug, ok := fpf[name] if !ok { return nil, plugin.ErrPluginNotFound diff --git a/pkg/connector/service_test.go b/pkg/connector/service_test.go index fa13ef166..3af066453 100644 --- a/pkg/connector/service_test.go +++ b/pkg/connector/service_test.go @@ -24,7 +24,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/database/inmemory" "github.com/conduitio/conduit/pkg/foundation/database/mock" "github.com/conduitio/conduit/pkg/foundation/log" - pmock "github.com/conduitio/conduit/pkg/plugin/mock" + pmock "github.com/conduitio/conduit/pkg/plugin/connector/mock" "github.com/conduitio/conduit/pkg/record" "github.com/google/uuid" "github.com/matryer/is" diff --git a/pkg/connector/source.go b/pkg/connector/source.go index 077571523..44119709e 100644 --- a/pkg/connector/source.go +++ b/pkg/connector/source.go @@ -21,14 +21,15 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/record" ) type Source struct { Instance *Instance - dispenser plugin.Dispenser - plugin plugin.SourcePlugin + dispenser connectorPlugin.Dispenser + plugin connectorPlugin.SourcePlugin // errs is used to signal the node that the connector experienced an error // when it was processing something asynchronously (e.g. persisting state). @@ -37,7 +38,7 @@ type Source struct { // stopStream is a function that closes the context of the stream stopStream context.CancelFunc - // wg tracks the number of in flight calls to the plugin. + // wg tracks the number of in flight calls to the connectorPlugin. wg sync.WaitGroup } diff --git a/pkg/connector/source_test.go b/pkg/connector/source_test.go index cea8c3bac..18d2fce6e 100644 --- a/pkg/connector/source_test.go +++ b/pkg/connector/source_test.go @@ -25,7 +25,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/database/inmemory" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/mock" + "github.com/conduitio/conduit/pkg/plugin/connector/mock" "github.com/conduitio/conduit/pkg/record" "github.com/matryer/is" "go.uber.org/mock/gomock" diff --git a/pkg/orchestrator/connectors.go b/pkg/orchestrator/connectors.go index 539044703..1e1c746be 100644 --- a/pkg/orchestrator/connectors.go +++ b/pkg/orchestrator/connectors.go @@ -203,16 +203,12 @@ func (c *ConnectorOrchestrator) Validate( plugin string, config connector.Config, ) error { - d, err := c.plugins.NewDispenser(c.logger, plugin) - if err != nil { - return cerrors.Errorf("couldn't get dispenser: %w", err) - } - + var err error switch t { case connector.TypeSource: - err = c.plugins.ValidateSourceConfig(ctx, d, config.Settings) + err = c.plugins.ValidateSourceConfig(ctx, plugin, config.Settings) case connector.TypeDestination: - err = c.plugins.ValidateDestinationConfig(ctx, d, config.Settings) + err = c.plugins.ValidateDestinationConfig(ctx, plugin, config.Settings) default: return cerrors.New("invalid connector type") } diff --git a/pkg/orchestrator/connectors_test.go b/pkg/orchestrator/connectors_test.go index 5cb81087c..176a3871c 100644 --- a/pkg/orchestrator/connectors_test.go +++ b/pkg/orchestrator/connectors_test.go @@ -24,7 +24,6 @@ import ( "github.com/conduitio/conduit/pkg/foundation/database/inmemory" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/pipeline" - pmock "github.com/conduitio/conduit/pkg/plugin/mock" "github.com/google/uuid" "github.com/matryer/is" "go.uber.org/mock/gomock" @@ -34,7 +33,6 @@ func TestConnectorOrchestrator_Create_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - ctrl := gomock.NewController(t) plsMock, consMock, procsMock, pluginMock := newMockServices(t) pl := &pipeline.Instance{ @@ -57,18 +55,13 @@ func TestConnectorOrchestrator_Create_Success(t *testing.T) { UpdatedAt: time.Now().UTC(), } - pluginDispenser := pmock.NewDispenser(ctrl) - plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - pluginMock.EXPECT(). - NewDispenser(gomock.Any(), want.Plugin). - Return(pluginDispenser, nil) pluginMock.EXPECT(). ValidateSourceConfig( gomock.AssignableToTypeOf(ctxType), - pluginDispenser, + want.Plugin, want.Config.Settings, ).Return(nil) consMock.EXPECT(). @@ -159,11 +152,8 @@ func TestConnectorOrchestrator_Create_CreateConnectorError(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - ctrl := gomock.NewController(t) plsMock, consMock, procsMock, pluginMock := newMockServices(t) - pluginDispenser := pmock.NewDispenser(ctrl) - pl := &pipeline.Instance{ ID: uuid.NewString(), Status: pipeline.StatusSystemStopped, @@ -173,13 +163,10 @@ func TestConnectorOrchestrator_Create_CreateConnectorError(t *testing.T) { plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - pluginMock.EXPECT(). - NewDispenser(gomock.Any(), "test-plugin"). - Return(pluginDispenser, nil) pluginMock.EXPECT(). ValidateSourceConfig( gomock.AssignableToTypeOf(ctxType), - pluginDispenser, + "test-plugin", config.Settings, ).Return(nil) consMock.EXPECT(). @@ -205,11 +192,8 @@ func TestConnectorOrchestrator_Create_AddConnectorError(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - ctrl := gomock.NewController(t) plsMock, consMock, procsMock, pluginMock := newMockServices(t) - pluginDispenser := pmock.NewDispenser(ctrl) - pl := &pipeline.Instance{ ID: uuid.NewString(), Status: pipeline.StatusSystemStopped, @@ -234,13 +218,10 @@ func TestConnectorOrchestrator_Create_AddConnectorError(t *testing.T) { plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - pluginMock.EXPECT(). - NewDispenser(gomock.Any(), conn.Plugin). - Return(pluginDispenser, nil) pluginMock.EXPECT(). ValidateSourceConfig( gomock.AssignableToTypeOf(ctxType), - pluginDispenser, + conn.Plugin, conn.Config.Settings, ).Return(nil) consMock.EXPECT(). @@ -458,11 +439,8 @@ func TestConnectorOrchestrator_Update_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - ctrl := gomock.NewController(t) plsMock, consMock, procsMock, pluginMock := newMockServices(t) - pluginDispenser := pmock.NewDispenser(ctrl) - pl := &pipeline.Instance{ ID: uuid.NewString(), Status: pipeline.StatusSystemStopped, @@ -495,10 +473,7 @@ func TestConnectorOrchestrator_Update_Success(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) pluginMock.EXPECT(). - NewDispenser(gomock.Any(), conn.Plugin). - Return(pluginDispenser, nil) - pluginMock.EXPECT(). - ValidateSourceConfig(gomock.Any(), pluginDispenser, newConfig.Settings). + ValidateSourceConfig(gomock.Any(), conn.Plugin, newConfig.Settings). Return(nil) consMock.EXPECT(). Update(gomock.AssignableToTypeOf(ctxType), conn.ID, newConfig). @@ -562,11 +537,8 @@ func TestConnectorOrchestrator_Update_Fail(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - ctrl := gomock.NewController(t) plsMock, consMock, procsMock, pluginMock := newMockServices(t) - pluginDispenser := pmock.NewDispenser(ctrl) - pl := &pipeline.Instance{ ID: uuid.NewString(), Status: pipeline.StatusSystemStopped, @@ -585,10 +557,7 @@ func TestConnectorOrchestrator_Update_Fail(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) pluginMock.EXPECT(). - NewDispenser(gomock.Any(), conn.Plugin). - Return(pluginDispenser, nil) - pluginMock.EXPECT(). - ValidateDestinationConfig(gomock.Any(), pluginDispenser, conn.Config.Settings). + ValidateDestinationConfig(gomock.Any(), conn.Plugin, conn.Config.Settings). Return(nil) consMock.EXPECT(). Update(gomock.AssignableToTypeOf(ctxType), conn.ID, connector.Config{}). diff --git a/pkg/orchestrator/mock/orchestrator.go b/pkg/orchestrator/mock/orchestrator.go index 98c5df6eb..4a64e9614 100644 --- a/pkg/orchestrator/mock/orchestrator.go +++ b/pkg/orchestrator/mock/orchestrator.go @@ -15,7 +15,7 @@ import ( connector "github.com/conduitio/conduit/pkg/connector" log "github.com/conduitio/conduit/pkg/foundation/log" pipeline "github.com/conduitio/conduit/pkg/pipeline" - plugin "github.com/conduitio/conduit/pkg/plugin" + connector0 "github.com/conduitio/conduit/pkg/plugin/connector" processor "github.com/conduitio/conduit/pkg/processor" gomock "go.uber.org/mock/gomock" ) @@ -464,26 +464,26 @@ func (m *PluginService) EXPECT() *PluginServiceMockRecorder { return m.recorder } -// List mocks base method. -func (m *PluginService) List(arg0 context.Context) (map[string]plugin.Specification, error) { +// ListConnectors mocks base method. +func (m *PluginService) ListConnectors(arg0 context.Context) (map[string]connector0.Specification, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "List", arg0) - ret0, _ := ret[0].(map[string]plugin.Specification) + ret := m.ctrl.Call(m, "ListConnectors", arg0) + ret0, _ := ret[0].(map[string]connector0.Specification) ret1, _ := ret[1].(error) return ret0, ret1 } -// List indicates an expected call of List. -func (mr *PluginServiceMockRecorder) List(arg0 any) *gomock.Call { +// ListConnectors indicates an expected call of ListConnectors. +func (mr *PluginServiceMockRecorder) ListConnectors(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*PluginService)(nil).List), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListConnectors", reflect.TypeOf((*PluginService)(nil).ListConnectors), arg0) } // NewDispenser mocks base method. -func (m *PluginService) NewDispenser(arg0 log.CtxLogger, arg1 string) (plugin.Dispenser, error) { +func (m *PluginService) NewDispenser(arg0 log.CtxLogger, arg1 string) (connector0.Dispenser, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NewDispenser", arg0, arg1) - ret0, _ := ret[0].(plugin.Dispenser) + ret0, _ := ret[0].(connector0.Dispenser) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -495,7 +495,7 @@ func (mr *PluginServiceMockRecorder) NewDispenser(arg0, arg1 any) *gomock.Call { } // ValidateDestinationConfig mocks base method. -func (m *PluginService) ValidateDestinationConfig(arg0 context.Context, arg1 plugin.Dispenser, arg2 map[string]string) error { +func (m *PluginService) ValidateDestinationConfig(arg0 context.Context, arg1 string, arg2 map[string]string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ValidateDestinationConfig", arg0, arg1, arg2) ret0, _ := ret[0].(error) @@ -509,7 +509,7 @@ func (mr *PluginServiceMockRecorder) ValidateDestinationConfig(arg0, arg1, arg2 } // ValidateSourceConfig mocks base method. -func (m *PluginService) ValidateSourceConfig(arg0 context.Context, arg1 plugin.Dispenser, arg2 map[string]string) error { +func (m *PluginService) ValidateSourceConfig(arg0 context.Context, arg1 string, arg2 map[string]string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ValidateSourceConfig", arg0, arg1, arg2) ret0, _ := ret[0].(error) diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index 8a6ecb09a..d08a4dc89 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -23,7 +23,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/database" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/pipeline" - "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" ) @@ -113,8 +113,8 @@ type ProcessorService interface { } type PluginService interface { - List(ctx context.Context) (map[string]plugin.Specification, error) - NewDispenser(logger log.CtxLogger, name string) (plugin.Dispenser, error) - ValidateSourceConfig(ctx context.Context, d plugin.Dispenser, settings map[string]string) error - ValidateDestinationConfig(ctx context.Context, d plugin.Dispenser, settings map[string]string) error + ListConnectors(ctx context.Context) (map[string]connectorPlugin.Specification, error) + NewDispenser(logger log.CtxLogger, name string) (connectorPlugin.Dispenser, error) + ValidateSourceConfig(ctx context.Context, name string, settings map[string]string) error + ValidateDestinationConfig(ctx context.Context, name string, settings map[string]string) error } diff --git a/pkg/orchestrator/orchestrator_test.go b/pkg/orchestrator/orchestrator_test.go index 74b687b84..e2bf95ed8 100644 --- a/pkg/orchestrator/orchestrator_test.go +++ b/pkg/orchestrator/orchestrator_test.go @@ -29,8 +29,8 @@ import ( "github.com/conduitio/conduit/pkg/orchestrator/mock" "github.com/conduitio/conduit/pkg/pipeline" "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/builtin" - "github.com/conduitio/conduit/pkg/plugin/standalone" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin" + "github.com/conduitio/conduit/pkg/plugin/connector/standalone" "github.com/conduitio/conduit/pkg/processor" "github.com/conduitio/conduit/pkg/processor/procbuiltin" "github.com/conduitio/conduit/pkg/record" diff --git a/pkg/orchestrator/pipelines_test.go b/pkg/orchestrator/pipelines_test.go index 4fca4d76c..e205c2d86 100644 --- a/pkg/orchestrator/pipelines_test.go +++ b/pkg/orchestrator/pipelines_test.go @@ -22,7 +22,6 @@ import ( "github.com/conduitio/conduit/pkg/foundation/database/inmemory" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/pipeline" - pmock "github.com/conduitio/conduit/pkg/plugin/mock" "github.com/google/uuid" "github.com/matryer/is" "go.uber.org/mock/gomock" @@ -314,11 +313,8 @@ func TestPipelineOrchestrator_UpdateDLQ_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - ctrl := gomock.NewController(t) plsMock, consMock, procsMock, pluginMock := newMockServices(t) - pluginDispenser := pmock.NewDispenser(ctrl) - plBefore := &pipeline.Instance{ ID: uuid.NewString(), Status: pipeline.StatusSystemStopped, @@ -348,10 +344,7 @@ func TestPipelineOrchestrator_UpdateDLQ_Success(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) pluginMock.EXPECT(). - NewDispenser(gomock.Any(), newDLQ.Plugin). - Return(pluginDispenser, nil) - pluginMock.EXPECT(). - ValidateDestinationConfig(gomock.Any(), pluginDispenser, newDLQ.Settings). + ValidateDestinationConfig(gomock.Any(), newDLQ.Plugin, newDLQ.Settings). Return(nil) plsMock.EXPECT(). UpdateDLQ(gomock.AssignableToTypeOf(ctxType), plBefore.ID, newDLQ). @@ -409,11 +402,8 @@ func TestConnectorOrchestrator_UpdateDLQ_InvalidConfig(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - ctrl := gomock.NewController(t) plsMock, consMock, procsMock, pluginMock := newMockServices(t) - pluginDispenser := pmock.NewDispenser(ctrl) - plBefore := &pipeline.Instance{ ID: uuid.NewString(), Status: pipeline.StatusSystemStopped, @@ -432,10 +422,7 @@ func TestConnectorOrchestrator_UpdateDLQ_InvalidConfig(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) pluginMock.EXPECT(). - NewDispenser(gomock.Any(), newDLQ.Plugin). - Return(pluginDispenser, nil) - pluginMock.EXPECT(). - ValidateDestinationConfig(gomock.Any(), pluginDispenser, newDLQ.Settings). + ValidateDestinationConfig(gomock.Any(), newDLQ.Plugin, newDLQ.Settings). Return(wantErr) got, err := orc.Pipelines.UpdateDLQ(ctx, plBefore.ID, newDLQ) diff --git a/pkg/orchestrator/plugins.go b/pkg/orchestrator/plugins.go index 65a775f73..789ac61c5 100644 --- a/pkg/orchestrator/plugins.go +++ b/pkg/orchestrator/plugins.go @@ -17,11 +17,11 @@ package orchestrator import ( "context" - "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" ) type PluginOrchestrator base -func (ps *PluginOrchestrator) List(ctx context.Context) (map[string]plugin.Specification, error) { - return ps.plugins.List(ctx) +func (ps *PluginOrchestrator) ListConnectors(ctx context.Context) (map[string]connector.Specification, error) { + return ps.plugins.ListConnectors(ctx) } diff --git a/pkg/pipeline/lifecycle.go b/pkg/pipeline/lifecycle.go index 335ffd7dd..0609111bf 100644 --- a/pkg/pipeline/lifecycle.go +++ b/pkg/pipeline/lifecycle.go @@ -30,7 +30,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/metrics/measure" "github.com/conduitio/conduit/pkg/foundation/multierror" "github.com/conduitio/conduit/pkg/pipeline/stream" - "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" "gopkg.in/tomb.v2" ) @@ -48,7 +48,7 @@ type ProcessorFetcher interface { // PluginDispenserFetcher can fetch a plugin. type PluginDispenserFetcher interface { - NewDispenser(logger log.CtxLogger, name string) (plugin.Dispenser, error) + NewDispenser(logger log.CtxLogger, name string) (connectorPlugin.Dispenser, error) } // Run runs pipelines that had the running state in store. diff --git a/pkg/pipeline/lifecycle_test.go b/pkg/pipeline/lifecycle_test.go index 63561d6ac..4e2d2cd28 100644 --- a/pkg/pipeline/lifecycle_test.go +++ b/pkg/pipeline/lifecycle_test.go @@ -30,7 +30,8 @@ import ( "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/pipeline/stream" "github.com/conduitio/conduit/pkg/plugin" - pmock "github.com/conduitio/conduit/pkg/plugin/mock" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" + pmock "github.com/conduitio/conduit/pkg/plugin/connector/mock" "github.com/conduitio/conduit/pkg/processor" "github.com/conduitio/conduit/pkg/record" "github.com/google/uuid" @@ -573,9 +574,9 @@ func (tpf testProcessorFetcher) Get(_ context.Context, id string) (*processor.In } // testPluginFetcher fulfills the PluginFetcher interface. -type testPluginFetcher map[string]plugin.Dispenser +type testPluginFetcher map[string]connectorPlugin.Dispenser -func (tpf testPluginFetcher) NewDispenser(_ log.CtxLogger, name string) (plugin.Dispenser, error) { +func (tpf testPluginFetcher) NewDispenser(_ log.CtxLogger, name string) (connectorPlugin.Dispenser, error) { plug, ok := tpf[name] if !ok { return nil, plugin.ErrPluginNotFound diff --git a/pkg/pipeline/stream/source_test.go b/pkg/pipeline/stream/source_test.go index bf0a7c466..c19134107 100644 --- a/pkg/pipeline/stream/source_test.go +++ b/pkg/pipeline/stream/source_test.go @@ -26,7 +26,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" "github.com/conduitio/conduit/pkg/pipeline/stream/mock" - "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/record" "github.com/google/uuid" "github.com/matryer/is" @@ -97,7 +97,7 @@ func TestSourceNode_Stop_ConcurrentFail(t *testing.T) { src.EXPECT().Read(gomock.Any()).DoAndReturn(func(ctx context.Context) (record.Record, error) { close(startRead) <-unblockRead - return record.Record{}, plugin.ErrStreamNotOpen + return record.Record{}, connectorPlugin.ErrStreamNotOpen }).Times(1) startStop := make(chan struct{}) unblockStop := make(chan struct{}) @@ -124,7 +124,7 @@ func TestSourceNode_Stop_ConcurrentFail(t *testing.T) { go func() { defer close(nodeDone) err := node.Run(ctx) - is.True(cerrors.Is(err, plugin.ErrStreamNotOpen)) + is.True(cerrors.Is(err, connectorPlugin.ErrStreamNotOpen)) }() _, ok, err := cchan.ChanOut[struct{}](startRead).RecvTimeout(ctx, time.Second) @@ -348,7 +348,7 @@ func newMockSource(ctrl *gomock.Controller, recordCount int, wantErr error) (*mo return record.Record{}, wantErr } <-teardown - return record.Record{}, plugin.ErrStreamNotOpen + return record.Record{}, connectorPlugin.ErrStreamNotOpen } r := records[position] position++ diff --git a/pkg/pipeline/stream/stream_test.go b/pkg/pipeline/stream/stream_test.go index 3afb80b48..349a3d06c 100644 --- a/pkg/pipeline/stream/stream_test.go +++ b/pkg/pipeline/stream/stream_test.go @@ -27,7 +27,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/metrics/noop" "github.com/conduitio/conduit/pkg/pipeline/stream" streammock "github.com/conduitio/conduit/pkg/pipeline/stream/mock" - "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" procmock "github.com/conduitio/conduit/pkg/processor/mock" "github.com/conduitio/conduit/pkg/record" @@ -343,7 +343,7 @@ func generatorSource(ctrl *gomock.Controller, logger log.CtxLogger, nodeID strin if position == recordCount { // block until Teardown is called <-teardown - return record.Record{}, plugin.ErrStreamNotOpen + return record.Record{}, connectorPlugin.ErrStreamNotOpen } position++ diff --git a/pkg/plugin/acceptance_testing.go b/pkg/plugin/connector/acceptance_testing.go similarity index 99% rename from pkg/plugin/acceptance_testing.go rename to pkg/plugin/connector/acceptance_testing.go index 0472a9819..e9ebece68 100644 --- a/pkg/plugin/acceptance_testing.go +++ b/pkg/plugin/connector/acceptance_testing.go @@ -1,4 +1,4 @@ -// Copyright © 2022 Meroxa, Inc. +// Copyright © 2023 Meroxa, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ // limitations under the License. //nolint:revive,dogsled // this is a test file -package plugin +package connector import ( "context" diff --git a/pkg/plugin/builtin/registry.go b/pkg/plugin/connector/builtin/registry.go similarity index 84% rename from pkg/plugin/builtin/registry.go rename to pkg/plugin/connector/builtin/registry.go index b2b404aa4..605fd3c8c 100644 --- a/pkg/plugin/builtin/registry.go +++ b/pkg/plugin/connector/builtin/registry.go @@ -29,7 +29,8 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/plugin" - builtinv1 "github.com/conduitio/conduit/pkg/plugin/builtin/v1" + "github.com/conduitio/conduit/pkg/plugin/connector" + builtinv1 "github.com/conduitio/conduit/pkg/plugin/connector/builtin/v1" ) var ( @@ -56,29 +57,29 @@ type Registry struct { type blueprint struct { fullName plugin.FullName - specification plugin.Specification + specification connector.Specification dispenserFactory DispenserFactory } -type DispenserFactory func(name plugin.FullName, logger log.CtxLogger) plugin.Dispenser +type DispenserFactory func(name plugin.FullName, logger log.CtxLogger) connector.Dispenser -func NewDispenserFactory(connector sdk.Connector) DispenserFactory { - if connector.NewSource == nil { - connector.NewSource = func() sdk.Source { return nil } +func NewDispenserFactory(conn sdk.Connector) DispenserFactory { + if conn.NewSource == nil { + conn.NewSource = func() sdk.Source { return nil } } - if connector.NewDestination == nil { - connector.NewDestination = func() sdk.Destination { return nil } + if conn.NewDestination == nil { + conn.NewDestination = func() sdk.Destination { return nil } } - return func(name plugin.FullName, logger log.CtxLogger) plugin.Dispenser { + return func(name plugin.FullName, logger log.CtxLogger) connector.Dispenser { return builtinv1.NewDispenser( name, logger, func() cpluginv1.SpecifierPlugin { - return sdk.NewSpecifierPlugin(connector.NewSpecification(), connector.NewSource(), connector.NewDestination()) + return sdk.NewSpecifierPlugin(conn.NewSpecification(), conn.NewSource(), conn.NewDestination()) }, - func() cpluginv1.SourcePlugin { return sdk.NewSourcePlugin(connector.NewSource()) }, - func() cpluginv1.DestinationPlugin { return sdk.NewDestinationPlugin(connector.NewDestination()) }, + func() cpluginv1.SourcePlugin { return sdk.NewSourcePlugin(conn.NewSource()) }, + func() cpluginv1.DestinationPlugin { return sdk.NewDestinationPlugin(conn.NewDestination()) }, ) } } @@ -135,15 +136,15 @@ func loadPlugins(buildInfo *debug.BuildInfo, factories map[string]DispenserFacto return plugins } -func getSpecification(moduleName string, factory DispenserFactory, buildInfo *debug.BuildInfo) (plugin.Specification, error) { +func getSpecification(moduleName string, factory DispenserFactory, buildInfo *debug.BuildInfo) (connector.Specification, error) { dispenser := factory("", log.CtxLogger{}) specPlugin, err := dispenser.DispenseSpecifier() if err != nil { - return plugin.Specification{}, cerrors.Errorf("could not dispense specifier for built in plugin: %w", err) + return connector.Specification{}, cerrors.Errorf("could not dispense specifier for built in plugin: %w", err) } specs, err := specPlugin.Specify() if err != nil { - return plugin.Specification{}, cerrors.Errorf("could not get specs for built in plugin: %w", err) + return connector.Specification{}, cerrors.Errorf("could not get specs for built in plugin: %w", err) } if version := getModuleVersion(buildInfo.Deps, moduleName); version != "" { @@ -170,7 +171,7 @@ func newFullName(pluginName, pluginVersion string) plugin.FullName { return plugin.NewFullName(plugin.PluginTypeBuiltin, pluginName, pluginVersion) } -func (r *Registry) NewDispenser(logger log.CtxLogger, fullName plugin.FullName) (plugin.Dispenser, error) { +func (r *Registry) NewDispenser(logger log.CtxLogger, fullName plugin.FullName) (connector.Dispenser, error) { versionMap, ok := r.plugins[fullName.PluginName()] if !ok { return nil, plugin.ErrPluginNotFound @@ -187,8 +188,8 @@ func (r *Registry) NewDispenser(logger log.CtxLogger, fullName plugin.FullName) return b.dispenserFactory(fullName, logger), nil } -func (r *Registry) List() map[plugin.FullName]plugin.Specification { - specs := make(map[plugin.FullName]plugin.Specification, len(r.plugins)) +func (r *Registry) List() map[plugin.FullName]connector.Specification { + specs := make(map[plugin.FullName]connector.Specification, len(r.plugins)) for _, versions := range r.plugins { for version, bp := range versions { if version == plugin.PluginVersionLatest { diff --git a/pkg/plugin/builtin/v1/acceptance_test.go b/pkg/plugin/connector/builtin/v1/acceptance_test.go similarity index 86% rename from pkg/plugin/builtin/v1/acceptance_test.go rename to pkg/plugin/connector/builtin/v1/acceptance_test.go index a5149a5a0..29506e78e 100644 --- a/pkg/plugin/builtin/v1/acceptance_test.go +++ b/pkg/plugin/connector/builtin/v1/acceptance_test.go @@ -17,9 +17,9 @@ package builtinv1 import ( "testing" - "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" ) func TestAcceptance(t *testing.T) { - plugin.AcceptanceTestV1(t, newTestDispenser) + connector.AcceptanceTestV1(t, newTestDispenser) } diff --git a/pkg/plugin/builtin/v1/destination.go b/pkg/plugin/connector/builtin/v1/destination.go similarity index 92% rename from pkg/plugin/builtin/v1/destination.go rename to pkg/plugin/connector/builtin/v1/destination.go index 27dfb140a..d020ea1b4 100644 --- a/pkg/plugin/builtin/v1/destination.go +++ b/pkg/plugin/connector/builtin/v1/destination.go @@ -20,9 +20,9 @@ import ( "github.com/conduitio/conduit-connector-protocol/cpluginv1" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/builtin/v1/internal/fromplugin" - "github.com/conduitio/conduit/pkg/plugin/builtin/v1/internal/toplugin" + "github.com/conduitio/conduit/pkg/plugin/connector" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin/v1/internal/fromplugin" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin/v1/internal/toplugin" "github.com/conduitio/conduit/pkg/record" "github.com/rs/zerolog" ) @@ -43,7 +43,7 @@ type destinationPluginAdapter struct { stream *stream[cpluginv1.DestinationRunRequest, cpluginv1.DestinationRunResponse] } -var _ plugin.DestinationPlugin = (*destinationPluginAdapter)(nil) +var _ connector.DestinationPlugin = (*destinationPluginAdapter)(nil) func newDestinationPluginAdapter(impl cpluginv1.DestinationPlugin, logger log.CtxLogger) *destinationPluginAdapter { return &destinationPluginAdapter{ @@ -83,7 +83,7 @@ func (s *destinationPluginAdapter) Start(ctx context.Context) error { s.logger.Err(ctx, err).Msg("stream already stopped") } } else { - s.stream.stop(plugin.ErrStreamNotOpen) + s.stream.stop(connector.ErrStreamNotOpen) } s.logger.Trace(ctx).Msg("Run stopped") }() @@ -93,7 +93,7 @@ func (s *destinationPluginAdapter) Start(ctx context.Context) error { func (s *destinationPluginAdapter) Write(ctx context.Context, r record.Record) error { if s.stream == nil { - return plugin.ErrStreamNotOpen + return connector.ErrStreamNotOpen } req, err := toplugin.DestinationRunRequest(r) @@ -112,7 +112,7 @@ func (s *destinationPluginAdapter) Write(ctx context.Context, r record.Record) e func (s *destinationPluginAdapter) Ack(ctx context.Context) (record.Position, error) { if s.stream == nil { - return nil, plugin.ErrStreamNotOpen + return nil, connector.ErrStreamNotOpen } s.logger.Trace(ctx).Msg("receiving ack") @@ -131,7 +131,7 @@ func (s *destinationPluginAdapter) Ack(ctx context.Context) (record.Position, er func (s *destinationPluginAdapter) Stop(ctx context.Context, lastPosition record.Position) error { if s.stream == nil { - return plugin.ErrStreamNotOpen + return connector.ErrStreamNotOpen } s.logger.Trace(ctx).Bytes(log.RecordPositionField, lastPosition).Msg("calling Stop") diff --git a/pkg/plugin/builtin/v1/dispenser.go b/pkg/plugin/connector/builtin/v1/dispenser.go similarity index 87% rename from pkg/plugin/builtin/v1/dispenser.go rename to pkg/plugin/connector/builtin/v1/dispenser.go index 31e4b9e32..2d320024b 100644 --- a/pkg/plugin/builtin/v1/dispenser.go +++ b/pkg/plugin/connector/builtin/v1/dispenser.go @@ -18,6 +18,7 @@ import ( "github.com/conduitio/conduit-connector-protocol/cpluginv1" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" ) type Dispenser struct { @@ -44,15 +45,15 @@ func NewDispenser( } } -func (d *Dispenser) DispenseSpecifier() (plugin.SpecifierPlugin, error) { +func (d *Dispenser) DispenseSpecifier() (connector.SpecifierPlugin, error) { return newSpecifierPluginAdapter(d.specifierPlugin()), nil } -func (d *Dispenser) DispenseSource() (plugin.SourcePlugin, error) { +func (d *Dispenser) DispenseSource() (connector.SourcePlugin, error) { return newSourcePluginAdapter(d.sourcePlugin(), d.pluginLogger("source")), nil } -func (d *Dispenser) DispenseDestination() (plugin.DestinationPlugin, error) { +func (d *Dispenser) DispenseDestination() (connector.DestinationPlugin, error) { return newDestinationPluginAdapter(d.destinationPlugin(), d.pluginLogger("destination")), nil } diff --git a/pkg/plugin/builtin/v1/dispenser_test.go b/pkg/plugin/connector/builtin/v1/dispenser_test.go similarity index 89% rename from pkg/plugin/builtin/v1/dispenser_test.go rename to pkg/plugin/connector/builtin/v1/dispenser_test.go index 1b626577d..0302ff98d 100644 --- a/pkg/plugin/builtin/v1/dispenser_test.go +++ b/pkg/plugin/connector/builtin/v1/dispenser_test.go @@ -20,12 +20,12 @@ import ( "github.com/conduitio/conduit-connector-protocol/cpluginv1" "github.com/conduitio/conduit-connector-protocol/cpluginv1/mock" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/golang/mock/gomock" //nolint:depguard // the protocol is using the old mock package, this will be changed when we update conduit-connector-protocol "github.com/rs/zerolog" ) -func newTestDispenser(t *testing.T) (plugin.Dispenser, *mock.SpecifierPlugin, *mock.SourcePlugin, *mock.DestinationPlugin) { +func newTestDispenser(t *testing.T) (connector.Dispenser, *mock.SpecifierPlugin, *mock.SourcePlugin, *mock.DestinationPlugin) { logger := log.InitLogger(zerolog.InfoLevel, log.FormatCLI) ctrl := gomock.NewController(t) diff --git a/pkg/plugin/builtin/v1/internal/fromplugin/destination.go b/pkg/plugin/connector/builtin/v1/internal/fromplugin/destination.go similarity index 100% rename from pkg/plugin/builtin/v1/internal/fromplugin/destination.go rename to pkg/plugin/connector/builtin/v1/internal/fromplugin/destination.go diff --git a/pkg/plugin/builtin/v1/internal/fromplugin/record.go b/pkg/plugin/connector/builtin/v1/internal/fromplugin/record.go similarity index 100% rename from pkg/plugin/builtin/v1/internal/fromplugin/record.go rename to pkg/plugin/connector/builtin/v1/internal/fromplugin/record.go diff --git a/pkg/plugin/builtin/v1/internal/fromplugin/source.go b/pkg/plugin/connector/builtin/v1/internal/fromplugin/source.go similarity index 100% rename from pkg/plugin/builtin/v1/internal/fromplugin/source.go rename to pkg/plugin/connector/builtin/v1/internal/fromplugin/source.go diff --git a/pkg/plugin/builtin/v1/internal/fromplugin/specifier.go b/pkg/plugin/connector/builtin/v1/internal/fromplugin/specifier.go similarity index 51% rename from pkg/plugin/builtin/v1/internal/fromplugin/specifier.go rename to pkg/plugin/connector/builtin/v1/internal/fromplugin/specifier.go index cec742214..b025c14a3 100644 --- a/pkg/plugin/builtin/v1/internal/fromplugin/specifier.go +++ b/pkg/plugin/connector/builtin/v1/internal/fromplugin/specifier.go @@ -16,37 +16,37 @@ package fromplugin import ( "github.com/conduitio/conduit-connector-protocol/cpluginv1" - "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" ) func _() { // An "invalid array index" compiler error signifies that the constant values have changed. var vTypes [1]struct{} - _ = vTypes[int(cpluginv1.ValidationTypeRequired)-int(plugin.ValidationTypeRequired)] - _ = vTypes[int(cpluginv1.ValidationTypeLessThan)-int(plugin.ValidationTypeLessThan)] - _ = vTypes[int(cpluginv1.ValidationTypeGreaterThan)-int(plugin.ValidationTypeGreaterThan)] - _ = vTypes[int(cpluginv1.ValidationTypeInclusion)-int(plugin.ValidationTypeInclusion)] - _ = vTypes[int(cpluginv1.ValidationTypeExclusion)-int(plugin.ValidationTypeExclusion)] - _ = vTypes[int(cpluginv1.ValidationTypeRegex)-int(plugin.ValidationTypeRegex)] + _ = vTypes[int(cpluginv1.ValidationTypeRequired)-int(connectorPlugin.ValidationTypeRequired)] + _ = vTypes[int(cpluginv1.ValidationTypeLessThan)-int(connectorPlugin.ValidationTypeLessThan)] + _ = vTypes[int(cpluginv1.ValidationTypeGreaterThan)-int(connectorPlugin.ValidationTypeGreaterThan)] + _ = vTypes[int(cpluginv1.ValidationTypeInclusion)-int(connectorPlugin.ValidationTypeInclusion)] + _ = vTypes[int(cpluginv1.ValidationTypeExclusion)-int(connectorPlugin.ValidationTypeExclusion)] + _ = vTypes[int(cpluginv1.ValidationTypeRegex)-int(connectorPlugin.ValidationTypeRegex)] // parameter types - _ = vTypes[int(cpluginv1.ParameterTypeString)-int(plugin.ParameterTypeString)] - _ = vTypes[int(cpluginv1.ParameterTypeInt)-int(plugin.ParameterTypeInt)] - _ = vTypes[int(cpluginv1.ParameterTypeFloat)-int(plugin.ParameterTypeFloat)] - _ = vTypes[int(cpluginv1.ParameterTypeBool)-int(plugin.ParameterTypeBool)] - _ = vTypes[int(cpluginv1.ParameterTypeFile)-int(plugin.ParameterTypeFile)] - _ = vTypes[int(cpluginv1.ParameterTypeDuration)-int(plugin.ParameterTypeDuration)] + _ = vTypes[int(cpluginv1.ParameterTypeString)-int(connectorPlugin.ParameterTypeString)] + _ = vTypes[int(cpluginv1.ParameterTypeInt)-int(connectorPlugin.ParameterTypeInt)] + _ = vTypes[int(cpluginv1.ParameterTypeFloat)-int(connectorPlugin.ParameterTypeFloat)] + _ = vTypes[int(cpluginv1.ParameterTypeBool)-int(connectorPlugin.ParameterTypeBool)] + _ = vTypes[int(cpluginv1.ParameterTypeFile)-int(connectorPlugin.ParameterTypeFile)] + _ = vTypes[int(cpluginv1.ParameterTypeDuration)-int(connectorPlugin.ParameterTypeDuration)] } -func SpecifierSpecifyResponse(in cpluginv1.SpecifierSpecifyResponse) (plugin.Specification, error) { - specMap := func(params map[string]cpluginv1.SpecifierParameter) map[string]plugin.Parameter { - out := make(map[string]plugin.Parameter) +func SpecifierSpecifyResponse(in cpluginv1.SpecifierSpecifyResponse) (connectorPlugin.Specification, error) { + specMap := func(params map[string]cpluginv1.SpecifierParameter) map[string]connectorPlugin.Parameter { + out := make(map[string]connectorPlugin.Parameter) for k, v := range params { out[k] = SpecifierParameter(v) } return out } - return plugin.Specification{ + return connectorPlugin.Specification{ Name: in.Name, Summary: in.Summary, Description: in.Description, @@ -57,13 +57,13 @@ func SpecifierSpecifyResponse(in cpluginv1.SpecifierSpecifyResponse) (plugin.Spe }, nil } -func SpecifierParameter(in cpluginv1.SpecifierParameter) plugin.Parameter { - validations := make([]plugin.Validation, len(in.Validations)) +func SpecifierParameter(in cpluginv1.SpecifierParameter) connectorPlugin.Parameter { + validations := make([]connectorPlugin.Validation, len(in.Validations)) requiredExists := false for i, v := range in.Validations { - validations[i] = plugin.Validation{ - Type: plugin.ValidationType(v.Type), + validations[i] = connectorPlugin.Validation{ + Type: connectorPlugin.ValidationType(v.Type), Value: v.Value, } if v.Type == cpluginv1.ValidationTypeRequired { @@ -73,12 +73,12 @@ func SpecifierParameter(in cpluginv1.SpecifierParameter) plugin.Parameter { // needed for backward compatibility, in.Required is converted to a validation of type ValidationTypeRequired // making sure not to duplicate the required validation if in.Required && !requiredExists { - validations = append(validations, plugin.Validation{ - Type: plugin.ValidationTypeRequired, + validations = append(validations, connectorPlugin.Validation{ + Type: connectorPlugin.ValidationTypeRequired, }) } - return plugin.Parameter{ + return connectorPlugin.Parameter{ Default: in.Default, Type: cpluginv1ParamTypeToPluginParamType(in.Type), Description: in.Description, @@ -86,10 +86,10 @@ func SpecifierParameter(in cpluginv1.SpecifierParameter) plugin.Parameter { } } -func cpluginv1ParamTypeToPluginParamType(t cpluginv1.ParameterType) plugin.ParameterType { +func cpluginv1ParamTypeToPluginParamType(t cpluginv1.ParameterType) connectorPlugin.ParameterType { // default type should be string if t == 0 { - return plugin.ParameterTypeString + return connectorPlugin.ParameterTypeString } - return plugin.ParameterType(t) + return connectorPlugin.ParameterType(t) } diff --git a/pkg/plugin/builtin/v1/internal/toplugin/destination.go b/pkg/plugin/connector/builtin/v1/internal/toplugin/destination.go similarity index 100% rename from pkg/plugin/builtin/v1/internal/toplugin/destination.go rename to pkg/plugin/connector/builtin/v1/internal/toplugin/destination.go diff --git a/pkg/plugin/builtin/v1/internal/toplugin/record.go b/pkg/plugin/connector/builtin/v1/internal/toplugin/record.go similarity index 100% rename from pkg/plugin/builtin/v1/internal/toplugin/record.go rename to pkg/plugin/connector/builtin/v1/internal/toplugin/record.go diff --git a/pkg/plugin/builtin/v1/internal/toplugin/source.go b/pkg/plugin/connector/builtin/v1/internal/toplugin/source.go similarity index 100% rename from pkg/plugin/builtin/v1/internal/toplugin/source.go rename to pkg/plugin/connector/builtin/v1/internal/toplugin/source.go diff --git a/pkg/plugin/builtin/v1/internal/toplugin/specifier.go b/pkg/plugin/connector/builtin/v1/internal/toplugin/specifier.go similarity index 100% rename from pkg/plugin/builtin/v1/internal/toplugin/specifier.go rename to pkg/plugin/connector/builtin/v1/internal/toplugin/specifier.go diff --git a/pkg/plugin/builtin/v1/metadata_test.go b/pkg/plugin/connector/builtin/v1/metadata_test.go similarity index 100% rename from pkg/plugin/builtin/v1/metadata_test.go rename to pkg/plugin/connector/builtin/v1/metadata_test.go diff --git a/pkg/plugin/builtin/v1/sandbox.go b/pkg/plugin/connector/builtin/v1/sandbox.go similarity index 100% rename from pkg/plugin/builtin/v1/sandbox.go rename to pkg/plugin/connector/builtin/v1/sandbox.go diff --git a/pkg/plugin/builtin/v1/sandbox_test.go b/pkg/plugin/connector/builtin/v1/sandbox_test.go similarity index 100% rename from pkg/plugin/builtin/v1/sandbox_test.go rename to pkg/plugin/connector/builtin/v1/sandbox_test.go diff --git a/pkg/plugin/builtin/v1/source.go b/pkg/plugin/connector/builtin/v1/source.go similarity index 92% rename from pkg/plugin/builtin/v1/source.go rename to pkg/plugin/connector/builtin/v1/source.go index e184fecb8..69a18342b 100644 --- a/pkg/plugin/builtin/v1/source.go +++ b/pkg/plugin/connector/builtin/v1/source.go @@ -20,9 +20,9 @@ import ( "github.com/conduitio/conduit-connector-protocol/cpluginv1" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/builtin/v1/internal/fromplugin" - "github.com/conduitio/conduit/pkg/plugin/builtin/v1/internal/toplugin" + "github.com/conduitio/conduit/pkg/plugin/connector" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin/v1/internal/fromplugin" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin/v1/internal/toplugin" "github.com/conduitio/conduit/pkg/record" "github.com/rs/zerolog" ) @@ -43,7 +43,7 @@ type sourcePluginAdapter struct { stream *stream[cpluginv1.SourceRunRequest, cpluginv1.SourceRunResponse] } -var _ plugin.SourcePlugin = (*sourcePluginAdapter)(nil) +var _ connector.SourcePlugin = (*sourcePluginAdapter)(nil) func newSourcePluginAdapter(impl cpluginv1.SourcePlugin, logger log.CtxLogger) *sourcePluginAdapter { return &sourcePluginAdapter{ @@ -86,7 +86,7 @@ func (s *sourcePluginAdapter) Start(ctx context.Context, p record.Position) erro s.logger.Err(ctx, err).Msg("stream already stopped") } } else { - s.stream.stop(plugin.ErrStreamNotOpen) + s.stream.stop(connector.ErrStreamNotOpen) } s.logger.Trace(ctx).Msg("Run stopped") }() @@ -96,7 +96,7 @@ func (s *sourcePluginAdapter) Start(ctx context.Context, p record.Position) erro func (s *sourcePluginAdapter) Read(ctx context.Context) (record.Record, error) { if s.stream == nil { - return record.Record{}, plugin.ErrStreamNotOpen + return record.Record{}, connector.ErrStreamNotOpen } s.logger.Trace(ctx).Msg("receiving record") @@ -115,7 +115,7 @@ func (s *sourcePluginAdapter) Read(ctx context.Context) (record.Record, error) { func (s *sourcePluginAdapter) Ack(ctx context.Context, p record.Position) error { if s.stream == nil { - return plugin.ErrStreamNotOpen + return connector.ErrStreamNotOpen } req := toplugin.SourceRunRequest(p) @@ -131,7 +131,7 @@ func (s *sourcePluginAdapter) Ack(ctx context.Context, p record.Position) error func (s *sourcePluginAdapter) Stop(ctx context.Context) (record.Position, error) { if s.stream == nil { - return nil, plugin.ErrStreamNotOpen + return nil, connector.ErrStreamNotOpen } s.logger.Trace(ctx).Msg("calling Stop") diff --git a/pkg/plugin/builtin/v1/specifier.go b/pkg/plugin/connector/builtin/v1/specifier.go similarity index 73% rename from pkg/plugin/builtin/v1/specifier.go rename to pkg/plugin/connector/builtin/v1/specifier.go index e2bc2ffc6..ae3cb093c 100644 --- a/pkg/plugin/builtin/v1/specifier.go +++ b/pkg/plugin/connector/builtin/v1/specifier.go @@ -18,9 +18,9 @@ import ( "context" "github.com/conduitio/conduit-connector-protocol/cpluginv1" - "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/builtin/v1/internal/fromplugin" - "github.com/conduitio/conduit/pkg/plugin/builtin/v1/internal/toplugin" + "github.com/conduitio/conduit/pkg/plugin/connector" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin/v1/internal/fromplugin" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin/v1/internal/toplugin" ) // TODO make sure a panic in a plugin doesn't crash Conduit @@ -28,7 +28,7 @@ type specifierPluginAdapter struct { impl cpluginv1.SpecifierPlugin } -var _ plugin.SpecifierPlugin = (*specifierPluginAdapter)(nil) +var _ connector.SpecifierPlugin = (*specifierPluginAdapter)(nil) func newSpecifierPluginAdapter(impl cpluginv1.SpecifierPlugin) *specifierPluginAdapter { return &specifierPluginAdapter{ @@ -36,15 +36,15 @@ func newSpecifierPluginAdapter(impl cpluginv1.SpecifierPlugin) *specifierPluginA } } -func (s *specifierPluginAdapter) Specify() (plugin.Specification, error) { +func (s *specifierPluginAdapter) Specify() (connector.Specification, error) { req := toplugin.SpecifierSpecifyRequest() resp, err := runSandbox(s.impl.Specify, context.Background(), req) if err != nil { - return plugin.Specification{}, err + return connector.Specification{}, err } out, err := fromplugin.SpecifierSpecifyResponse(resp) if err != nil { - return plugin.Specification{}, err + return connector.Specification{}, err } return out, nil } diff --git a/pkg/plugin/builtin/v1/stream.go b/pkg/plugin/connector/builtin/v1/stream.go similarity index 96% rename from pkg/plugin/builtin/v1/stream.go rename to pkg/plugin/connector/builtin/v1/stream.go index 541792eb7..0175a525e 100644 --- a/pkg/plugin/builtin/v1/stream.go +++ b/pkg/plugin/connector/builtin/v1/stream.go @@ -19,7 +19,7 @@ import ( "io" "sync" - "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" ) // stream mimics the behavior of a gRPC stream using channels. @@ -73,7 +73,7 @@ func (s *stream[REQ, RES]) sendInternal(req REQ) error { case <-s.ctx.Done(): return s.ctx.Err() case <-s.stopChan: - return plugin.ErrStreamNotOpen + return connector.ErrStreamNotOpen case s.reqChan <- req: return nil } diff --git a/pkg/plugin/connector/errors.go b/pkg/plugin/connector/errors.go new file mode 100644 index 000000000..fe6c425b6 --- /dev/null +++ b/pkg/plugin/connector/errors.go @@ -0,0 +1,19 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connector + +import "github.com/conduitio/conduit/pkg/foundation/cerrors" + +var ErrStreamNotOpen = cerrors.New("stream not open") diff --git a/pkg/plugin/mock/plugin.go b/pkg/plugin/connector/mock/plugin.go similarity index 95% rename from pkg/plugin/mock/plugin.go rename to pkg/plugin/connector/mock/plugin.go index c6f665216..460f5f9a2 100644 --- a/pkg/plugin/mock/plugin.go +++ b/pkg/plugin/connector/mock/plugin.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/plugin (interfaces: Dispenser,DestinationPlugin,SourcePlugin,SpecifierPlugin) +// Source: github.com/conduitio/conduit/pkg/plugin/connector (interfaces: Dispenser,DestinationPlugin,SourcePlugin,SpecifierPlugin) // // Generated by this command: // @@ -12,7 +12,7 @@ import ( context "context" reflect "reflect" - plugin "github.com/conduitio/conduit/pkg/plugin" + connector "github.com/conduitio/conduit/pkg/plugin/connector" record "github.com/conduitio/conduit/pkg/record" gomock "go.uber.org/mock/gomock" ) @@ -41,10 +41,10 @@ func (m *Dispenser) EXPECT() *DispenserMockRecorder { } // DispenseDestination mocks base method. -func (m *Dispenser) DispenseDestination() (plugin.DestinationPlugin, error) { +func (m *Dispenser) DispenseDestination() (connector.DestinationPlugin, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DispenseDestination") - ret0, _ := ret[0].(plugin.DestinationPlugin) + ret0, _ := ret[0].(connector.DestinationPlugin) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -56,10 +56,10 @@ func (mr *DispenserMockRecorder) DispenseDestination() *gomock.Call { } // DispenseSource mocks base method. -func (m *Dispenser) DispenseSource() (plugin.SourcePlugin, error) { +func (m *Dispenser) DispenseSource() (connector.SourcePlugin, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DispenseSource") - ret0, _ := ret[0].(plugin.SourcePlugin) + ret0, _ := ret[0].(connector.SourcePlugin) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -71,10 +71,10 @@ func (mr *DispenserMockRecorder) DispenseSource() *gomock.Call { } // DispenseSpecifier mocks base method. -func (m *Dispenser) DispenseSpecifier() (plugin.SpecifierPlugin, error) { +func (m *Dispenser) DispenseSpecifier() (connector.SpecifierPlugin, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DispenseSpecifier") - ret0, _ := ret[0].(plugin.SpecifierPlugin) + ret0, _ := ret[0].(connector.SpecifierPlugin) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -410,10 +410,10 @@ func (m *SpecifierPlugin) EXPECT() *SpecifierPluginMockRecorder { } // Specify mocks base method. -func (m *SpecifierPlugin) Specify() (plugin.Specification, error) { +func (m *SpecifierPlugin) Specify() (connector.Specification, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Specify") - ret0, _ := ret[0].(plugin.Specification) + ret0, _ := ret[0].(connector.Specification) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/pkg/plugin/plugin.go b/pkg/plugin/connector/plugin.go similarity index 84% rename from pkg/plugin/plugin.go rename to pkg/plugin/connector/plugin.go index 578f4980b..9948ebffe 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/connector/plugin.go @@ -1,4 +1,4 @@ -// Copyright © 2022 Meroxa, Inc. +// Copyright © 2023 Meroxa, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,13 +14,11 @@ //go:generate mockgen -destination=mock/plugin.go -package=mock -mock_names=Dispenser=Dispenser,SourcePlugin=SourcePlugin,DestinationPlugin=DestinationPlugin,SpecifierPlugin=SpecifierPlugin . Dispenser,DestinationPlugin,SourcePlugin,SpecifierPlugin -package plugin +package connector import ( "context" - "strings" - "github.com/Masterminds/semver/v3" "github.com/conduitio/conduit/pkg/record" ) @@ -208,75 +206,3 @@ const ( ParameterTypeFile ParameterTypeDuration ) - -const ( - PluginTypeBuiltin = "builtin" - PluginTypeStandalone = "standalone" - PluginTypeAny = "any" - - PluginVersionLatest = "latest" -) - -type FullName string - -func NewFullName(pluginType, pluginName, pluginVersion string) FullName { - if pluginType != "" { - pluginType += ":" - } - if pluginVersion != "" { - pluginVersion = "@" + pluginVersion - } - return FullName(pluginType + pluginName + pluginVersion) -} - -func (fn FullName) PluginType() string { - tokens := strings.SplitN(string(fn), ":", 2) - if len(tokens) > 1 { - return tokens[0] - } - return PluginTypeAny // default -} - -func (fn FullName) PluginName() string { - name := string(fn) - - tokens := strings.SplitN(name, ":", 2) - if len(tokens) > 1 { - name = tokens[1] - } - - tokens = strings.SplitN(name, "@", 2) - if len(tokens) > 1 { - name = tokens[0] - } - - return name -} - -func (fn FullName) PluginVersion() string { - tokens := strings.SplitN(string(fn), "@", 2) - if len(tokens) > 1 { - return tokens[len(tokens)-1] - } - return PluginVersionLatest // default -} - -func (fn FullName) PluginVersionGreaterThan(other FullName) bool { - leftVersion := fn.PluginVersion() - rightVersion := other.PluginVersion() - - leftSemver, err := semver.NewVersion(leftVersion) - if err != nil { - return false // left is an invalid semver, right is greater either way - } - rightSemver, err := semver.NewVersion(rightVersion) - if err != nil { - return true // left is a valid semver, right is not, left is greater - } - - return leftSemver.GreaterThan(rightSemver) -} - -func (fn FullName) String() string { - return fn.PluginType() + ":" + fn.PluginName() + "@" + fn.PluginVersion() -} diff --git a/pkg/plugin/standalone/registry.go b/pkg/plugin/connector/standalone/registry.go similarity index 94% rename from pkg/plugin/standalone/registry.go rename to pkg/plugin/connector/standalone/registry.go index 6d3cd975d..a3a2a5068 100644 --- a/pkg/plugin/standalone/registry.go +++ b/pkg/plugin/connector/standalone/registry.go @@ -24,7 +24,8 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/plugin" - standalonev1 "github.com/conduitio/conduit/pkg/plugin/standalone/v1" + "github.com/conduitio/conduit/pkg/plugin/connector" + standalonev1 "github.com/conduitio/conduit/pkg/plugin/connector/standalone/v1" "github.com/rs/zerolog" ) @@ -41,7 +42,7 @@ type Registry struct { type blueprint struct { fullName plugin.FullName - specification plugin.Specification + specification connector.Specification path string // TODO store hash of plugin binary and compare before running the binary to // ensure someone can't switch the plugin after we registered it @@ -174,7 +175,7 @@ func (r *Registry) loadPlugins(ctx context.Context, pluginDir string) map[string return plugins } -func (r *Registry) NewDispenser(logger log.CtxLogger, fullName plugin.FullName) (plugin.Dispenser, error) { +func (r *Registry) NewDispenser(logger log.CtxLogger, fullName plugin.FullName) (connector.Dispenser, error) { r.m.RLock() defer r.m.RUnlock() @@ -194,11 +195,11 @@ func (r *Registry) NewDispenser(logger log.CtxLogger, fullName plugin.FullName) return standalonev1.NewDispenser(logger.ZerologWithComponent(), bp.path) } -func (r *Registry) List() map[plugin.FullName]plugin.Specification { +func (r *Registry) List() map[plugin.FullName]connector.Specification { r.m.RLock() defer r.m.RUnlock() - specs := make(map[plugin.FullName]plugin.Specification, len(r.plugins)) + specs := make(map[plugin.FullName]connector.Specification, len(r.plugins)) for _, versions := range r.plugins { for version, bp := range versions { if version == plugin.PluginVersionLatest { diff --git a/pkg/plugin/standalone/registry_test.go b/pkg/plugin/connector/standalone/registry_test.go similarity index 76% rename from pkg/plugin/standalone/registry_test.go rename to pkg/plugin/connector/standalone/registry_test.go index 9e91ef2f8..8be043811 100644 --- a/pkg/plugin/standalone/registry_test.go +++ b/pkg/plugin/connector/standalone/registry_test.go @@ -22,10 +22,11 @@ import ( "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/matryer/is" ) -// These constants need to match the constants in pkg/plugin/standalone/test/testplugin/main.go +// These constants need to match the constants in pkg/plugin/connector/standalone/test/testplugin/main.go const ( testPluginDir = "./test" @@ -54,68 +55,68 @@ func testPluginBlueprint() blueprint { return blueprint{ fullName: plugin.FullName(fmt.Sprintf("standalone:%v@%v", testPluginName, testPluginVersion)), path: path.Join(testPluginDir, "testplugin.sh"), - specification: plugin.Specification{ + specification: connector.Specification{ Name: testPluginName, Summary: testPluginSummary, Description: testPluginDescription, Version: testPluginVersion, Author: testPluginAuthor, - SourceParams: map[string]plugin.Parameter{ + SourceParams: map[string]connector.Parameter{ testPluginSourceParam1: { Default: testPluginSourceParam1Default, - Type: plugin.ParameterTypeString, // default type + Type: connector.ParameterTypeString, // default type Description: testPluginSourceParam1Description, - Validations: []plugin.Validation{ + Validations: []connector.Validation{ { - Type: plugin.ValidationTypeRequired, + Type: connector.ValidationTypeRequired, Value: "", }, { - Type: plugin.ValidationTypeInclusion, + Type: connector.ValidationTypeInclusion, Value: "one,two", }, }, }, testPluginSourceParam2: { Default: testPluginSourceParam2Default, - Type: plugin.ParameterTypeInt, + Type: connector.ParameterTypeInt, Description: testPluginSourceParam2Description, - Validations: []plugin.Validation{ + Validations: []connector.Validation{ { - Type: plugin.ValidationTypeExclusion, + Type: connector.ValidationTypeExclusion, Value: "3,4", }, { - Type: plugin.ValidationTypeGreaterThan, + Type: connector.ValidationTypeGreaterThan, Value: "1", }, }, }, }, - DestinationParams: map[string]plugin.Parameter{ + DestinationParams: map[string]connector.Parameter{ testPluginDestinationParam1: { Default: testPluginDestinationParam1Default, - Type: plugin.ParameterTypeInt, + Type: connector.ParameterTypeInt, Description: testPluginDestinationParam1Description, - Validations: []plugin.Validation{ + Validations: []connector.Validation{ { - Type: plugin.ValidationTypeLessThan, + Type: connector.ValidationTypeLessThan, Value: "10", }, { - Type: plugin.ValidationTypeRegex, + Type: connector.ValidationTypeRegex, Value: "[1-9]", }, { - Type: plugin.ValidationTypeRequired, + Type: connector.ValidationTypeRequired, }, }, }, testPluginDestinationParam2: { Default: testPluginDestinationParam2Default, - Type: plugin.ParameterTypeDuration, + Type: connector.ParameterTypeDuration, Description: testPluginDestinationParam2Description, - Validations: []plugin.Validation{}, + Validations: []connector.Validation{}, }, }, }, @@ -145,7 +146,7 @@ func TestRegistry_List(t *testing.T) { got := r.List() bp := testPluginBlueprint() - want := map[plugin.FullName]plugin.Specification{ + want := map[plugin.FullName]connector.Specification{ bp.fullName: bp.specification, } is.Equal(got, want) diff --git a/pkg/plugin/standalone/test/testplugin.sh b/pkg/plugin/connector/standalone/test/testplugin.sh similarity index 100% rename from pkg/plugin/standalone/test/testplugin.sh rename to pkg/plugin/connector/standalone/test/testplugin.sh diff --git a/pkg/plugin/standalone/test/testplugin/main.go b/pkg/plugin/connector/standalone/test/testplugin/main.go similarity index 97% rename from pkg/plugin/standalone/test/testplugin/main.go rename to pkg/plugin/connector/standalone/test/testplugin/main.go index 2024fd4d4..91955e88c 100644 --- a/pkg/plugin/standalone/test/testplugin/main.go +++ b/pkg/plugin/connector/standalone/test/testplugin/main.go @@ -23,7 +23,7 @@ import ( "github.com/conduitio/conduit-connector-protocol/cpluginv1/server" ) -// These constants need to match the constants in pkg/plugin/standalone/registry_test.go +// These constants need to match the constants in pkg/plugin/connector/standalone/registry_test.go const ( testPluginName = "test-plugin" testPluginSummary = "My test plugin summary" diff --git a/pkg/plugin/standalone/v1/acceptance_test.go b/pkg/plugin/connector/standalone/v1/acceptance_test.go similarity index 81% rename from pkg/plugin/standalone/v1/acceptance_test.go rename to pkg/plugin/connector/standalone/v1/acceptance_test.go index 8286df925..7514c7b55 100644 --- a/pkg/plugin/standalone/v1/acceptance_test.go +++ b/pkg/plugin/connector/standalone/v1/acceptance_test.go @@ -18,13 +18,13 @@ import ( "testing" "github.com/conduitio/conduit-connector-protocol/cpluginv1/mock" - "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/rs/zerolog" ) func TestAcceptance(t *testing.T) { logger := zerolog.Nop() - plugin.AcceptanceTestV1(t, func(t *testing.T) (plugin.Dispenser, *mock.SpecifierPlugin, *mock.SourcePlugin, *mock.DestinationPlugin) { + connector.AcceptanceTestV1(t, func(t *testing.T) (connector.Dispenser, *mock.SpecifierPlugin, *mock.SourcePlugin, *mock.DestinationPlugin) { return newTestDispenser(t, logger) }) } diff --git a/pkg/plugin/standalone/v1/client.go b/pkg/plugin/connector/standalone/v1/client.go similarity index 100% rename from pkg/plugin/standalone/v1/client.go rename to pkg/plugin/connector/standalone/v1/client.go diff --git a/pkg/plugin/standalone/v1/destination.go b/pkg/plugin/connector/standalone/v1/destination.go similarity index 90% rename from pkg/plugin/standalone/v1/destination.go rename to pkg/plugin/connector/standalone/v1/destination.go index b8d44b6d5..2e0dd25d2 100644 --- a/pkg/plugin/standalone/v1/destination.go +++ b/pkg/plugin/connector/standalone/v1/destination.go @@ -20,9 +20,9 @@ import ( connectorv1 "github.com/conduitio/conduit-connector-protocol/proto/connector/v1" "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/standalone/v1/internal/fromproto" - "github.com/conduitio/conduit/pkg/plugin/standalone/v1/internal/toproto" + "github.com/conduitio/conduit/pkg/plugin/connector" + "github.com/conduitio/conduit/pkg/plugin/connector/standalone/v1/internal/fromproto" + "github.com/conduitio/conduit/pkg/plugin/connector/standalone/v1/internal/toproto" "github.com/conduitio/conduit/pkg/record" goplugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" @@ -49,7 +49,7 @@ type destinationPluginClient struct { stream connectorv1.DestinationPlugin_RunClient } -var _ plugin.DestinationPlugin = (*destinationPluginClient)(nil) +var _ connector.DestinationPlugin = (*destinationPluginClient)(nil) func (s *destinationPluginClient) Configure(ctx context.Context, cfg map[string]string) error { protoReq := toproto.DestinationConfigureRequest(cfg) @@ -77,7 +77,7 @@ func (s *destinationPluginClient) Start(ctx context.Context) error { func (s *destinationPluginClient) Write(_ context.Context, r record.Record) error { if s.stream == nil { - return plugin.ErrStreamNotOpen + return connector.ErrStreamNotOpen } protoReq, err := toproto.DestinationRunRequest(r) @@ -89,7 +89,7 @@ func (s *destinationPluginClient) Write(_ context.Context, r record.Record) erro if err != nil { if err == io.EOF { // stream was gracefully closed - return plugin.ErrStreamNotOpen + return connector.ErrStreamNotOpen } return unwrapGRPCError(err) } @@ -98,13 +98,13 @@ func (s *destinationPluginClient) Write(_ context.Context, r record.Record) erro func (s *destinationPluginClient) Ack(_ context.Context) (record.Position, error) { if s.stream == nil { - return nil, plugin.ErrStreamNotOpen + return nil, connector.ErrStreamNotOpen } resp, err := s.stream.Recv() if err != nil { if err == io.EOF { - return nil, plugin.ErrStreamNotOpen + return nil, connector.ErrStreamNotOpen } return nil, unwrapGRPCError(err) } @@ -119,7 +119,7 @@ func (s *destinationPluginClient) Ack(_ context.Context) (record.Position, error func (s *destinationPluginClient) Stop(ctx context.Context, lastPosition record.Position) error { if s.stream == nil { - return plugin.ErrStreamNotOpen + return connector.ErrStreamNotOpen } protoReq := toproto.DestinationStopRequest(lastPosition) diff --git a/pkg/plugin/standalone/v1/dispenser.go b/pkg/plugin/connector/standalone/v1/dispenser.go similarity index 85% rename from pkg/plugin/standalone/v1/dispenser.go rename to pkg/plugin/connector/standalone/v1/dispenser.go index 7c3f9524d..cd26f0f3f 100644 --- a/pkg/plugin/standalone/v1/dispenser.go +++ b/pkg/plugin/connector/standalone/v1/dispenser.go @@ -19,7 +19,7 @@ import ( "sync" "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" goplugin "github.com/hashicorp/go-plugin" "github.com/rs/zerolog" ) @@ -91,7 +91,7 @@ func (d *Dispenser) teardown() { d.dispensed = false } -func (d *Dispenser) DispenseSpecifier() (plugin.SpecifierPlugin, error) { +func (d *Dispenser) DispenseSpecifier() (connector.SpecifierPlugin, error) { err := d.dispense() if err != nil { return nil, err @@ -106,7 +106,7 @@ func (d *Dispenser) DispenseSpecifier() (plugin.SpecifierPlugin, error) { return nil, err } - specifier, ok := raw.(plugin.SpecifierPlugin) + specifier, ok := raw.(connector.SpecifierPlugin) if !ok { return nil, cerrors.Errorf("plugin did not dispense specifier, got type: %T", raw) } @@ -114,7 +114,7 @@ func (d *Dispenser) DispenseSpecifier() (plugin.SpecifierPlugin, error) { return specifierPluginDispenserSignaller{specifier, d}, nil } -func (d *Dispenser) DispenseSource() (plugin.SourcePlugin, error) { +func (d *Dispenser) DispenseSource() (connector.SourcePlugin, error) { err := d.dispense() if err != nil { return nil, err @@ -129,7 +129,7 @@ func (d *Dispenser) DispenseSource() (plugin.SourcePlugin, error) { return nil, err } - source, ok := raw.(plugin.SourcePlugin) + source, ok := raw.(connector.SourcePlugin) if !ok { return nil, cerrors.Errorf("plugin did not dispense source, got type: %T", raw) } @@ -137,7 +137,7 @@ func (d *Dispenser) DispenseSource() (plugin.SourcePlugin, error) { return sourcePluginDispenserSignaller{source, d}, nil } -func (d *Dispenser) DispenseDestination() (plugin.DestinationPlugin, error) { +func (d *Dispenser) DispenseDestination() (connector.DestinationPlugin, error) { err := d.dispense() if err != nil { return nil, err @@ -152,7 +152,7 @@ func (d *Dispenser) DispenseDestination() (plugin.DestinationPlugin, error) { return nil, err } - destination, ok := raw.(plugin.DestinationPlugin) + destination, ok := raw.(connector.DestinationPlugin) if !ok { return nil, cerrors.Errorf("plugin did not dispense destination, got type: %T", raw) } @@ -161,17 +161,17 @@ func (d *Dispenser) DispenseDestination() (plugin.DestinationPlugin, error) { } type specifierPluginDispenserSignaller struct { - plugin.SpecifierPlugin + connector.SpecifierPlugin d *Dispenser } -func (s specifierPluginDispenserSignaller) Specify() (plugin.Specification, error) { +func (s specifierPluginDispenserSignaller) Specify() (connector.Specification, error) { defer s.d.teardown() return s.SpecifierPlugin.Specify() } type sourcePluginDispenserSignaller struct { - plugin.SourcePlugin + connector.SourcePlugin d *Dispenser } @@ -181,7 +181,7 @@ func (s sourcePluginDispenserSignaller) Teardown(ctx context.Context) error { } type destinationPluginDispenserSignaller struct { - plugin.DestinationPlugin + connector.DestinationPlugin d *Dispenser } diff --git a/pkg/plugin/standalone/v1/dispenser_test.go b/pkg/plugin/connector/standalone/v1/dispenser_test.go similarity index 96% rename from pkg/plugin/standalone/v1/dispenser_test.go rename to pkg/plugin/connector/standalone/v1/dispenser_test.go index 684fdbc51..74b0aaca7 100644 --- a/pkg/plugin/standalone/v1/dispenser_test.go +++ b/pkg/plugin/connector/standalone/v1/dispenser_test.go @@ -22,14 +22,14 @@ import ( "github.com/conduitio/conduit-connector-protocol/cpluginv1" "github.com/conduitio/conduit-connector-protocol/cpluginv1/mock" "github.com/conduitio/conduit-connector-protocol/cpluginv1/server" - "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/golang/mock/gomock" //nolint:depguard // the protocol is using the old mock package, this will be changed when we update conduit-connector-protocol goplugin "github.com/hashicorp/go-plugin" "github.com/rs/zerolog" ) func newTestDispenser(t *testing.T, logger zerolog.Logger) ( - plugin.Dispenser, + connector.Dispenser, *mock.SpecifierPlugin, *mock.SourcePlugin, *mock.DestinationPlugin, diff --git a/pkg/plugin/standalone/v1/internal/fromproto/destination.go b/pkg/plugin/connector/standalone/v1/internal/fromproto/destination.go similarity index 100% rename from pkg/plugin/standalone/v1/internal/fromproto/destination.go rename to pkg/plugin/connector/standalone/v1/internal/fromproto/destination.go diff --git a/pkg/plugin/standalone/v1/internal/fromproto/record.go b/pkg/plugin/connector/standalone/v1/internal/fromproto/record.go similarity index 100% rename from pkg/plugin/standalone/v1/internal/fromproto/record.go rename to pkg/plugin/connector/standalone/v1/internal/fromproto/record.go diff --git a/pkg/plugin/standalone/v1/internal/fromproto/source.go b/pkg/plugin/connector/standalone/v1/internal/fromproto/source.go similarity index 100% rename from pkg/plugin/standalone/v1/internal/fromproto/source.go rename to pkg/plugin/connector/standalone/v1/internal/fromproto/source.go diff --git a/pkg/plugin/standalone/v1/internal/fromproto/specifier.go b/pkg/plugin/connector/standalone/v1/internal/fromproto/specifier.go similarity index 52% rename from pkg/plugin/standalone/v1/internal/fromproto/specifier.go rename to pkg/plugin/connector/standalone/v1/internal/fromproto/specifier.go index 71f3670a5..c3af4b737 100644 --- a/pkg/plugin/standalone/v1/internal/fromproto/specifier.go +++ b/pkg/plugin/connector/standalone/v1/internal/fromproto/specifier.go @@ -18,30 +18,30 @@ import ( "fmt" connectorv1 "github.com/conduitio/conduit-connector-protocol/proto/connector/v1" - "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" ) func _() { // An "invalid array index" compiler error signifies that the constant values have changed. var vTypes [1]struct{} - _ = vTypes[int(plugin.ValidationTypeRequired)-int(connectorv1.Specifier_Parameter_Validation_TYPE_REQUIRED)] - _ = vTypes[int(plugin.ValidationTypeLessThan)-int(connectorv1.Specifier_Parameter_Validation_TYPE_LESS_THAN)] - _ = vTypes[int(plugin.ValidationTypeGreaterThan)-int(connectorv1.Specifier_Parameter_Validation_TYPE_GREATER_THAN)] - _ = vTypes[int(plugin.ValidationTypeInclusion)-int(connectorv1.Specifier_Parameter_Validation_TYPE_INCLUSION)] - _ = vTypes[int(plugin.ValidationTypeExclusion)-int(connectorv1.Specifier_Parameter_Validation_TYPE_EXCLUSION)] - _ = vTypes[int(plugin.ValidationTypeRegex)-int(connectorv1.Specifier_Parameter_Validation_TYPE_REGEX)] + _ = vTypes[int(connector.ValidationTypeRequired)-int(connectorv1.Specifier_Parameter_Validation_TYPE_REQUIRED)] + _ = vTypes[int(connector.ValidationTypeLessThan)-int(connectorv1.Specifier_Parameter_Validation_TYPE_LESS_THAN)] + _ = vTypes[int(connector.ValidationTypeGreaterThan)-int(connectorv1.Specifier_Parameter_Validation_TYPE_GREATER_THAN)] + _ = vTypes[int(connector.ValidationTypeInclusion)-int(connectorv1.Specifier_Parameter_Validation_TYPE_INCLUSION)] + _ = vTypes[int(connector.ValidationTypeExclusion)-int(connectorv1.Specifier_Parameter_Validation_TYPE_EXCLUSION)] + _ = vTypes[int(connector.ValidationTypeRegex)-int(connectorv1.Specifier_Parameter_Validation_TYPE_REGEX)] // parameter types - _ = vTypes[int(plugin.ParameterTypeString)-int(connectorv1.Specifier_Parameter_TYPE_STRING)] - _ = vTypes[int(plugin.ParameterTypeInt)-int(connectorv1.Specifier_Parameter_TYPE_INT)] - _ = vTypes[int(plugin.ParameterTypeFloat)-int(connectorv1.Specifier_Parameter_TYPE_FLOAT)] - _ = vTypes[int(plugin.ParameterTypeBool)-int(connectorv1.Specifier_Parameter_TYPE_BOOL)] - _ = vTypes[int(plugin.ParameterTypeFile)-int(connectorv1.Specifier_Parameter_TYPE_FILE)] - _ = vTypes[int(plugin.ParameterTypeDuration)-int(connectorv1.Specifier_Parameter_TYPE_DURATION)] + _ = vTypes[int(connector.ParameterTypeString)-int(connectorv1.Specifier_Parameter_TYPE_STRING)] + _ = vTypes[int(connector.ParameterTypeInt)-int(connectorv1.Specifier_Parameter_TYPE_INT)] + _ = vTypes[int(connector.ParameterTypeFloat)-int(connectorv1.Specifier_Parameter_TYPE_FLOAT)] + _ = vTypes[int(connector.ParameterTypeBool)-int(connectorv1.Specifier_Parameter_TYPE_BOOL)] + _ = vTypes[int(connector.ParameterTypeFile)-int(connectorv1.Specifier_Parameter_TYPE_FILE)] + _ = vTypes[int(connector.ParameterTypeDuration)-int(connectorv1.Specifier_Parameter_TYPE_DURATION)] } -func SpecifierSpecifyResponse(in *connectorv1.Specifier_Specify_Response) (plugin.Specification, error) { - specMap := func(in map[string]*connectorv1.Specifier_Parameter) (map[string]plugin.Parameter, error) { - out := make(map[string]plugin.Parameter, len(in)) +func SpecifierSpecifyResponse(in *connectorv1.Specifier_Specify_Response) (connector.Specification, error) { + specMap := func(in map[string]*connectorv1.Specifier_Parameter) (map[string]connector.Parameter, error) { + out := make(map[string]connector.Parameter, len(in)) var err error for k, v := range in { out[k], err = SpecifierParameter(v) @@ -54,15 +54,15 @@ func SpecifierSpecifyResponse(in *connectorv1.Specifier_Specify_Response) (plugi sourceParams, err := specMap(in.SourceParams) if err != nil { - return plugin.Specification{}, fmt.Errorf("error converting SourceSpec: %w", err) + return connector.Specification{}, fmt.Errorf("error converting SourceSpec: %w", err) } destinationParams, err := specMap(in.DestinationParams) if err != nil { - return plugin.Specification{}, fmt.Errorf("error converting DestinationSpec: %w", err) + return connector.Specification{}, fmt.Errorf("error converting DestinationSpec: %w", err) } - out := plugin.Specification{ + out := connector.Specification{ Name: in.Name, Summary: in.Summary, Description: in.Description, @@ -74,13 +74,13 @@ func SpecifierSpecifyResponse(in *connectorv1.Specifier_Specify_Response) (plugi return out, nil } -func SpecifierParameter(in *connectorv1.Specifier_Parameter) (plugin.Parameter, error) { - validations := make([]plugin.Validation, len(in.Validations)) +func SpecifierParameter(in *connectorv1.Specifier_Parameter) (connector.Parameter, error) { + validations := make([]connector.Validation, len(in.Validations)) requiredExists := false for i, v := range in.Validations { - validations[i] = plugin.Validation{ - Type: plugin.ValidationType(v.Type), + validations[i] = connector.Validation{ + Type: connector.ValidationType(v.Type), Value: v.Value, } if v.Type == connectorv1.Specifier_Parameter_Validation_TYPE_REQUIRED { @@ -90,12 +90,12 @@ func SpecifierParameter(in *connectorv1.Specifier_Parameter) (plugin.Parameter, // needed for backward compatibility, in.Required is converted to a validation of type ValidationTypeRequired // making sure not to duplicate the required validation if in.Required && !requiredExists { //nolint: staticcheck // required is still supported for now - validations = append(validations, plugin.Validation{ //nolint: makezero // list is full so need to append - Type: plugin.ValidationTypeRequired, + validations = append(validations, connector.Validation{ //nolint: makezero // list is full so need to append + Type: connector.ValidationTypeRequired, }) } - out := plugin.Parameter{ + out := connector.Parameter{ Default: in.Default, Description: in.Description, Type: connectorv1ParamTypeToPluginParamType(in.Type), @@ -104,10 +104,10 @@ func SpecifierParameter(in *connectorv1.Specifier_Parameter) (plugin.Parameter, return out, nil } -func connectorv1ParamTypeToPluginParamType(t connectorv1.Specifier_Parameter_Type) plugin.ParameterType { +func connectorv1ParamTypeToPluginParamType(t connectorv1.Specifier_Parameter_Type) connector.ParameterType { // default type should be string if t == connectorv1.Specifier_Parameter_TYPE_UNSPECIFIED { - return plugin.ParameterTypeString + return connector.ParameterTypeString } - return plugin.ParameterType(t) + return connector.ParameterType(t) } diff --git a/pkg/plugin/standalone/v1/internal/toproto/destination.go b/pkg/plugin/connector/standalone/v1/internal/toproto/destination.go similarity index 100% rename from pkg/plugin/standalone/v1/internal/toproto/destination.go rename to pkg/plugin/connector/standalone/v1/internal/toproto/destination.go diff --git a/pkg/plugin/standalone/v1/internal/toproto/record.go b/pkg/plugin/connector/standalone/v1/internal/toproto/record.go similarity index 100% rename from pkg/plugin/standalone/v1/internal/toproto/record.go rename to pkg/plugin/connector/standalone/v1/internal/toproto/record.go diff --git a/pkg/plugin/standalone/v1/internal/toproto/source.go b/pkg/plugin/connector/standalone/v1/internal/toproto/source.go similarity index 100% rename from pkg/plugin/standalone/v1/internal/toproto/source.go rename to pkg/plugin/connector/standalone/v1/internal/toproto/source.go diff --git a/pkg/plugin/standalone/v1/internal/toproto/specifier.go b/pkg/plugin/connector/standalone/v1/internal/toproto/specifier.go similarity index 100% rename from pkg/plugin/standalone/v1/internal/toproto/specifier.go rename to pkg/plugin/connector/standalone/v1/internal/toproto/specifier.go diff --git a/pkg/plugin/standalone/v1/logger.go b/pkg/plugin/connector/standalone/v1/logger.go similarity index 100% rename from pkg/plugin/standalone/v1/logger.go rename to pkg/plugin/connector/standalone/v1/logger.go diff --git a/pkg/plugin/standalone/v1/logger_test.go b/pkg/plugin/connector/standalone/v1/logger_test.go similarity index 100% rename from pkg/plugin/standalone/v1/logger_test.go rename to pkg/plugin/connector/standalone/v1/logger_test.go diff --git a/pkg/plugin/standalone/v1/metadata_test.go b/pkg/plugin/connector/standalone/v1/metadata_test.go similarity index 100% rename from pkg/plugin/standalone/v1/metadata_test.go rename to pkg/plugin/connector/standalone/v1/metadata_test.go diff --git a/pkg/plugin/standalone/v1/source.go b/pkg/plugin/connector/standalone/v1/source.go similarity index 89% rename from pkg/plugin/standalone/v1/source.go rename to pkg/plugin/connector/standalone/v1/source.go index a69612856..ec5f4a680 100644 --- a/pkg/plugin/standalone/v1/source.go +++ b/pkg/plugin/connector/standalone/v1/source.go @@ -20,9 +20,9 @@ import ( connectorv1 "github.com/conduitio/conduit-connector-protocol/proto/connector/v1" "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/standalone/v1/internal/fromproto" - "github.com/conduitio/conduit/pkg/plugin/standalone/v1/internal/toproto" + "github.com/conduitio/conduit/pkg/plugin/connector" + "github.com/conduitio/conduit/pkg/plugin/connector/standalone/v1/internal/fromproto" + "github.com/conduitio/conduit/pkg/plugin/connector/standalone/v1/internal/toproto" "github.com/conduitio/conduit/pkg/record" goplugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" @@ -49,7 +49,7 @@ type sourcePluginClient struct { stream connectorv1.SourcePlugin_RunClient } -var _ plugin.SourcePlugin = (*sourcePluginClient)(nil) +var _ connector.SourcePlugin = (*sourcePluginClient)(nil) func (s *sourcePluginClient) Configure(ctx context.Context, cfg map[string]string) error { protoReq := toproto.SourceConfigureRequest(cfg) @@ -77,13 +77,13 @@ func (s *sourcePluginClient) Start(ctx context.Context, p record.Position) error func (s *sourcePluginClient) Read(context.Context) (record.Record, error) { if s.stream == nil { - return record.Record{}, plugin.ErrStreamNotOpen + return record.Record{}, connector.ErrStreamNotOpen } protoResp, err := s.stream.Recv() if err != nil { if err == io.EOF { - return record.Record{}, plugin.ErrStreamNotOpen + return record.Record{}, connector.ErrStreamNotOpen } return record.Record{}, unwrapGRPCError(err) } @@ -96,7 +96,7 @@ func (s *sourcePluginClient) Read(context.Context) (record.Record, error) { func (s *sourcePluginClient) Ack(_ context.Context, p record.Position) error { if s.stream == nil { - return plugin.ErrStreamNotOpen + return connector.ErrStreamNotOpen } protoReq := toproto.SourceRunRequest(p) @@ -104,7 +104,7 @@ func (s *sourcePluginClient) Ack(_ context.Context, p record.Position) error { if err != nil { if err == io.EOF { // stream was gracefully closed - return plugin.ErrStreamNotOpen + return connector.ErrStreamNotOpen } return unwrapGRPCError(err) } @@ -113,7 +113,7 @@ func (s *sourcePluginClient) Ack(_ context.Context, p record.Position) error { func (s *sourcePluginClient) Stop(ctx context.Context) (record.Position, error) { if s.stream == nil { - return nil, plugin.ErrStreamNotOpen + return nil, connector.ErrStreamNotOpen } protoReq := toproto.SourceStopRequest() diff --git a/pkg/plugin/standalone/v1/specifier.go b/pkg/plugin/connector/standalone/v1/specifier.go similarity index 79% rename from pkg/plugin/standalone/v1/specifier.go rename to pkg/plugin/connector/standalone/v1/specifier.go index e1be59f19..5abc749e5 100644 --- a/pkg/plugin/standalone/v1/specifier.go +++ b/pkg/plugin/connector/standalone/v1/specifier.go @@ -19,9 +19,9 @@ import ( connectorv1 "github.com/conduitio/conduit-connector-protocol/proto/connector/v1" "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/standalone/v1/internal/fromproto" - "github.com/conduitio/conduit/pkg/plugin/standalone/v1/internal/toproto" + "github.com/conduitio/conduit/pkg/plugin/connector" + "github.com/conduitio/conduit/pkg/plugin/connector/standalone/v1/internal/fromproto" + "github.com/conduitio/conduit/pkg/plugin/connector/standalone/v1/internal/toproto" goplugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" ) @@ -46,17 +46,17 @@ type specifierPluginClient struct { grpcClient connectorv1.SpecifierPluginClient } -var _ plugin.SpecifierPlugin = (*specifierPluginClient)(nil) +var _ connector.SpecifierPlugin = (*specifierPluginClient)(nil) -func (s *specifierPluginClient) Specify() (plugin.Specification, error) { +func (s *specifierPluginClient) Specify() (connector.Specification, error) { protoReq := toproto.SpecifierSpecifyRequest() protoResp, err := s.grpcClient.Specify(context.Background(), protoReq) if err != nil { - return plugin.Specification{}, unwrapGRPCError(err) + return connector.Specification{}, unwrapGRPCError(err) } specs, err := fromproto.SpecifierSpecifyResponse(protoResp) if err != nil { - return plugin.Specification{}, err + return connector.Specification{}, err } return specs, nil } diff --git a/pkg/plugin/errors.go b/pkg/plugin/errors.go index 98671018d..77e2ba4ab 100644 --- a/pkg/plugin/errors.go +++ b/pkg/plugin/errors.go @@ -21,7 +21,6 @@ import ( ) var ( - ErrStreamNotOpen = cerrors.New("stream not open") ErrPluginNotFound = cerrors.New("plugin not found") ErrPluginNotRunning = cerrors.New("plugin is not running") ErrUnimplemented = cerrors.New("method not implemented") diff --git a/pkg/plugin/name.go b/pkg/plugin/name.go new file mode 100644 index 000000000..f6a31fb71 --- /dev/null +++ b/pkg/plugin/name.go @@ -0,0 +1,93 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plugin + +import ( + "strings" + + "github.com/Masterminds/semver/v3" +) + +const ( + PluginTypeBuiltin = "builtin" + PluginTypeStandalone = "standalone" + PluginTypeAny = "any" + + PluginVersionLatest = "latest" +) + +type FullName string + +func NewFullName(pluginType, pluginName, pluginVersion string) FullName { + if pluginType != "" { + pluginType += ":" + } + if pluginVersion != "" { + pluginVersion = "@" + pluginVersion + } + return FullName(pluginType + pluginName + pluginVersion) +} + +func (fn FullName) PluginType() string { + tokens := strings.SplitN(string(fn), ":", 2) + if len(tokens) > 1 { + return tokens[0] + } + return PluginTypeAny // default +} + +func (fn FullName) PluginName() string { + name := string(fn) + + tokens := strings.SplitN(name, ":", 2) + if len(tokens) > 1 { + name = tokens[1] + } + + tokens = strings.SplitN(name, "@", 2) + if len(tokens) > 1 { + name = tokens[0] + } + + return name +} + +func (fn FullName) PluginVersion() string { + tokens := strings.SplitN(string(fn), "@", 2) + if len(tokens) > 1 { + return tokens[len(tokens)-1] + } + return PluginVersionLatest // default +} + +func (fn FullName) PluginVersionGreaterThan(other FullName) bool { + leftVersion := fn.PluginVersion() + rightVersion := other.PluginVersion() + + leftSemver, err := semver.NewVersion(leftVersion) + if err != nil { + return false // left is an invalid semver, right is greater either way + } + rightSemver, err := semver.NewVersion(rightVersion) + if err != nil { + return true // left is a valid semver, right is not, left is greater + } + + return leftSemver.GreaterThan(rightSemver) +} + +func (fn FullName) String() string { + return fn.PluginType() + ":" + fn.PluginName() + "@" + fn.PluginVersion() +} diff --git a/pkg/plugin/plugin_test.go b/pkg/plugin/name_test.go similarity index 100% rename from pkg/plugin/plugin_test.go rename to pkg/plugin/name_test.go diff --git a/pkg/plugin/processor/builtin/registry.go b/pkg/plugin/processor/builtin/registry.go new file mode 100644 index 000000000..563b17f01 --- /dev/null +++ b/pkg/plugin/processor/builtin/registry.go @@ -0,0 +1,174 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builtin + +import ( + "context" + "reflect" + "runtime/debug" + + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/plugin" +) + +type Registry struct { + logger log.CtxLogger + + // plugins stores plugin blueprints in a 2D map, first key is the plugin + // name, the second key is the plugin version + plugins map[string]map[string]blueprint +} + +type blueprint struct { + fullName plugin.FullName + specification sdk.Specification + constructor ProcessorPluginConstructor +} + +type ProcessorPluginConstructor func(log.CtxLogger) sdk.Processor + +func NewRegistry(logger log.CtxLogger, constructors map[string]ProcessorPluginConstructor) *Registry { + logger = logger.WithComponent("builtin.Registry") + buildInfo, ok := debug.ReadBuildInfo() + if !ok { + // we are using modules, build info should always be available, we are staying on the safe side + logger.Warn(context.Background()).Msg("build info not available, built-in plugin versions may not be read correctly") + buildInfo = &debug.BuildInfo{} // prevent nil pointer exceptions + } + + r := &Registry{ + plugins: loadPlugins(buildInfo, constructors), + logger: logger, + } + logger.Info(context.Background()).Int("count", len(r.List())).Msg("builtin plugins initialized") + return r +} + +func NewProcessorPluginConstructor(processorPlugin sdk.Processor) ProcessorPluginConstructor { + procType := reflect.TypeOf(processorPlugin) + for procType.Kind() != reflect.Struct { + procType.Elem() + } + + f := func(logger log.CtxLogger) sdk.Processor { + // TODO create processor plugin wrapper that injects logger into context + // before forwarding the call to the plugin + newProcValue := reflect.New(procType) + return newProcValue.Interface().(sdk.Processor) + } + + // try out f, to catch any panic early + f(log.CtxLogger{}) + + return f +} + +func loadPlugins(buildInfo *debug.BuildInfo, constructors map[string]ProcessorPluginConstructor) map[string]map[string]blueprint { + plugins := make(map[string]map[string]blueprint, len(constructors)) + for moduleName, constructor := range constructors { + specs, err := getSpecification(moduleName, constructor, buildInfo) + if err != nil { + // stop initialization if a built-in plugin is misbehaving + panic(err) + } + + versionMap := plugins[specs.Name] + if versionMap == nil { + versionMap = make(map[string]blueprint) + plugins[specs.Name] = versionMap + } + + fullName := newFullName(specs.Name, specs.Version) + if _, ok := versionMap[specs.Version]; ok { + panic(cerrors.Errorf("plugin %q already registered", fullName)) + } + + bp := blueprint{ + fullName: fullName, + constructor: constructor, + specification: specs, + } + versionMap[specs.Version] = bp + + latestBp, ok := versionMap[plugin.PluginVersionLatest] + if !ok || fullName.PluginVersionGreaterThan(latestBp.fullName) { + versionMap[plugin.PluginVersionLatest] = bp + } + } + return plugins +} + +func getSpecification(moduleName string, constructor ProcessorPluginConstructor, buildInfo *debug.BuildInfo) (sdk.Specification, error) { + procPlugin := constructor(log.CtxLogger{}) + specs, err := procPlugin.Specification() + if err != nil { + return sdk.Specification{}, err + } + + if version := getModuleVersion(buildInfo.Deps, moduleName); version != "" { + // overwrite version with the import version + specs.Version = version + } + + return specs, nil +} + +func getModuleVersion(deps []*debug.Module, moduleName string) string { + for _, dep := range deps { + if dep.Path == moduleName { + if dep.Replace != nil { + return dep.Replace.Version + } + return dep.Version + } + } + return "" +} + +func newFullName(pluginName, pluginVersion string) plugin.FullName { + return plugin.NewFullName(plugin.PluginTypeBuiltin, pluginName, pluginVersion) +} + +func (r *Registry) NewProcessorPlugin(logger log.CtxLogger, fullName plugin.FullName) (sdk.Processor, error) { + versionMap, ok := r.plugins[fullName.PluginName()] + if !ok { + return nil, plugin.ErrPluginNotFound + } + b, ok := versionMap[fullName.PluginVersion()] + if !ok { + availableVersions := make([]string, 0, len(versionMap)) + for k := range versionMap { + availableVersions = append(availableVersions, k) + } + return nil, cerrors.Errorf("could not find builtin plugin %q, only found versions %v: %w", fullName, availableVersions, plugin.ErrPluginNotFound) + } + + return b.constructor(logger), nil +} + +func (r *Registry) List() map[plugin.FullName]sdk.Specification { + specs := make(map[plugin.FullName]sdk.Specification, len(r.plugins)) + for _, versions := range r.plugins { + for version, bp := range versions { + if version == plugin.PluginVersionLatest { + continue // skip latest versions + } + specs[bp.fullName] = bp.specification + } + } + return specs +} diff --git a/pkg/plugin/processor/builtin/registry_test.go b/pkg/plugin/processor/builtin/registry_test.go new file mode 100644 index 000000000..3d9a1a744 --- /dev/null +++ b/pkg/plugin/processor/builtin/registry_test.go @@ -0,0 +1,50 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builtin + +import ( + "testing" + + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit-processor-sdk/mock" + "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/plugin" + "github.com/matryer/is" + "go.uber.org/mock/gomock" +) + +func TestRegistry_List(t *testing.T) { + is := is.New(t) + logger := log.Nop() + + ctrl := gomock.NewController(t) + procPlugin := mock.NewProcessor(ctrl) + + procSpec := sdk.Specification{ + Name: "test-processor", + Version: "v0.1.2", + } + procPlugin.EXPECT().Specification().Return(procSpec, nil) + procConstructor := func(log.CtxLogger) sdk.Processor { return procPlugin } + + wantList := map[plugin.FullName]sdk.Specification{ + "builtin:test-processor@v0.1.2": procSpec, + } + + reg := NewRegistry(logger, map[string]ProcessorPluginConstructor{procSpec.Name: procConstructor}) + + got := reg.List() + is.Equal(got, wantList) +} diff --git a/pkg/plugin/processor/standalone/registry.go b/pkg/plugin/processor/standalone/registry.go new file mode 100644 index 000000000..0e1bb7075 --- /dev/null +++ b/pkg/plugin/processor/standalone/registry.go @@ -0,0 +1,68 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package standalone + +import ( + "context" + "fmt" + + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/plugin" + "github.com/rs/zerolog" +) + +type Registry struct { + *plugin.Registry +} + +type spec struct { + sdk.Specification +} + +func (s *spec) GetName() string { + return s.Name +} + +func (s *spec) GetVersion() string { + return s.Version +} + +func NewRegistry(logger log.CtxLogger, pluginDir string) *Registry { + return &Registry{ + Registry: plugin.NewRegistry( + logger, + pluginDir, + func(ctx context.Context, logger zerolog.Logger, path string) (plugin.Spec, error) { + p, err := NewWASMProcessor(ctx, logger, path) + if err != nil { + return nil, fmt.Errorf("failed creating a new WASM processor: %w", err) + } + defer func() { + err := p.Teardown(ctx) + if err != nil { + logger.Warn().Err(err).Msg("processor teardown failed") + } + }() + + s, err := p.Specification() + if err != nil { + return nil, err + } + return &spec{Specification: s}, nil + }, + ), + } +} diff --git a/pkg/plugin/processor/standalone/registry_test.go b/pkg/plugin/processor/standalone/registry_test.go new file mode 100644 index 000000000..a9c389afe --- /dev/null +++ b/pkg/plugin/processor/standalone/registry_test.go @@ -0,0 +1,92 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate ./test/build-test-processors.sh + +package standalone + +import ( + "testing" + + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/matryer/is" +) + +var testPluginDir = "./test/wasm_processors/" + +func TestRegistry_List(t *testing.T) { + is := is.New(t) + + underTest := NewRegistry(log.Nop(), testPluginDir+"chaos_processor/") + list := underTest.List() + is.Equal(1, len(list)) + got, ok := list["standalone:chaos-processor@v1.3.5"] + is.True(ok) // expected spec for standalone:test-processor@v1.3.5 + + param := sdk.Parameter{ + Default: "success", + Type: sdk.ParameterTypeString, + Description: "prefix", + Validations: []sdk.Validation{ + { + Type: sdk.ValidationTypeInclusion, + Value: "success,error,panic", + }, + }, + } + wantSpec := sdk.Specification{ + Name: "chaos-processor", + Summary: "chaos processor summary", + Description: "chaos processor description", + Version: "v1.3.5", + Author: "Meroxa, Inc.", + Parameters: map[string]sdk.Parameter{ + "configure": param, + "open": param, + "process.prefix": { + Default: "", + Type: sdk.ParameterTypeString, + Description: "prefix to be added to the payload's after", + Validations: []sdk.Validation{ + { + Type: sdk.ValidationTypeRequired, + }, + }, + }, + "process": param, + "teardown": param, + }, + } + is.Equal( + got, + &spec{Specification: wantSpec}, + ) +} + +func TestRegistry_MalformedProcessor(t *testing.T) { + is := is.New(t) + + underTest := NewRegistry(log.Nop(), testPluginDir+"malformed_processor/") + list := underTest.List() + is.Equal(0, len(list)) +} + +func TestRegistry_SpecifyError(t *testing.T) { + is := is.New(t) + + underTest := NewRegistry(log.Nop(), testPluginDir+"specify_error/") + list := underTest.List() + is.Equal(0, len(list)) +} diff --git a/pkg/plugin/processor/standalone/test/build-test-processors.sh b/pkg/plugin/processor/standalone/test/build-test-processors.sh new file mode 100755 index 000000000..847170af9 --- /dev/null +++ b/pkg/plugin/processor/standalone/test/build-test-processors.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +WASM_PROCESSORS_DIR="$SCRIPT_DIR/wasm_processors" + +for dir in "$WASM_PROCESSORS_DIR"/*/; do + # Check if the directory contains a .go file + if [ -e "${dir}processor.go" ]; then + cd "$dir" || exit + + GOOS=wasip1 GOARCH=wasm go build -tags wasm -o processor.wasm processor.go + + cd "$WASM_PROCESSORS_DIR" || exit + fi +done diff --git a/pkg/plugin/processor/standalone/test/wasm_processors/chaos_processor/processor.go b/pkg/plugin/processor/standalone/test/wasm_processors/chaos_processor/processor.go new file mode 100644 index 000000000..ce386e419 --- /dev/null +++ b/pkg/plugin/processor/standalone/test/wasm_processors/chaos_processor/processor.go @@ -0,0 +1,133 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + + "github.com/conduitio/conduit-commons/opencdc" + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit-processor-sdk/run" + "github.com/conduitio/conduit/pkg/foundation/cerrors" +) + +func main() { + run.Run(&chaosProcessor{}) +} + +type chaosProcessor struct { + cfg map[string]string +} + +func (p *chaosProcessor) Specification() (sdk.Specification, error) { + param := sdk.Parameter{ + Default: "success", + Type: sdk.ParameterTypeString, + Description: "prefix", + Validations: []sdk.Validation{ + { + Type: sdk.ValidationTypeInclusion, + Value: "success,error,panic", + }, + }, + } + return sdk.Specification{ + Name: "chaos-processor", + Summary: "chaos processor summary", + Description: "chaos processor description", + Version: "v1.3.5", + Author: "Meroxa, Inc.", + Parameters: map[string]sdk.Parameter{ + "configure": param, + "open": param, + "process.prefix": { + Default: "", + Type: sdk.ParameterTypeString, + Description: "prefix to be added to the payload's after", + Validations: []sdk.Validation{ + { + Type: sdk.ValidationTypeRequired, + }, + }, + }, + "process": param, + "teardown": param, + }, + }, nil +} + +func (p *chaosProcessor) Configure(_ context.Context, cfg map[string]string) error { + p.cfg = cfg + + err := p.methodBehavior("configure") + if err != nil { + return err + } + + _, ok := cfg["process.prefix"] + if !ok { + return cerrors.New("missing prefix") + } + + return nil +} + +func (p *chaosProcessor) Open(context.Context) error { + return p.methodBehavior("open") +} + +func (p *chaosProcessor) methodBehavior(name string) error { + switch p.cfg[name] { + case "error": + return cerrors.New(name + " error") + case "panic": + panic(name + " panic") + case "", "success": + return nil + default: + panic("unknown mode: " + p.cfg[name]) + } +} + +func (p *chaosProcessor) Process(_ context.Context, records []opencdc.Record) []sdk.ProcessedRecord { + err := p.methodBehavior("process") + if err != nil { + return p.makeErrorRecords(len(records), err) + } + + out := make([]sdk.ProcessedRecord, len(records)) + for i, record := range records { + outRec := record.Clone() + original := outRec.Payload.After.(opencdc.RawData) + outRec.Payload.After = opencdc.RawData(p.cfg["process.prefix"] + string(original.Bytes())) + + out[i] = sdk.SingleRecord(outRec) + } + + return out +} + +func (p *chaosProcessor) makeErrorRecords(num int, err error) []sdk.ProcessedRecord { + out := make([]sdk.ProcessedRecord, num) + for i := 0; i < num; i++ { + out[i] = sdk.ErrorRecord{Err: err} + } + + return out +} + +func (p *chaosProcessor) Teardown(context.Context) error { + return p.methodBehavior("teardown") +} diff --git a/pkg/plugin/processor/standalone/test/wasm_processors/specify_error/processor.go b/pkg/plugin/processor/standalone/test/wasm_processors/specify_error/processor.go new file mode 100644 index 000000000..0f80f95e3 --- /dev/null +++ b/pkg/plugin/processor/standalone/test/wasm_processors/specify_error/processor.go @@ -0,0 +1,54 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + + "github.com/conduitio/conduit-commons/opencdc" + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit-processor-sdk/run" + "github.com/conduitio/conduit/pkg/foundation/cerrors" +) + +func main() { + run.Run(&testProcessor{}) +} + +type testProcessor struct { +} + +func (p *testProcessor) Specification() (sdk.Specification, error) { + return sdk.Specification{}, cerrors.New("boom") +} + +func (p *testProcessor) Configure(context.Context, map[string]string) error { + // TODO implement me + panic("implement me") +} + +func (p *testProcessor) Open(context.Context) error { + // TODO implement me + panic("implement me") +} + +func (p *testProcessor) Process(context.Context, []opencdc.Record) []sdk.ProcessedRecord { + // TODO implement me + panic("implement me") +} + +func (p *testProcessor) Teardown(context.Context) error { + return nil +} diff --git a/pkg/plugin/processor/standalone/wasm_processor.go b/pkg/plugin/processor/standalone/wasm_processor.go new file mode 100644 index 000000000..7a99fdf60 --- /dev/null +++ b/pkg/plugin/processor/standalone/wasm_processor.go @@ -0,0 +1,307 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package standalone + +import ( + "context" + "fmt" + "os" + + "github.com/conduitio/conduit-commons/opencdc" + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit-processor-sdk/serde" + "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/rs/zerolog" + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" +) + +type wasmProcessor struct { + logger zerolog.Logger + + runtime wazero.Runtime + runtimeCancel context.CancelFunc + + // commandCh is used to send commands to + // the actual processor (the WASM module) + // and through exported host functions. + commandCh chan sdk.Command + // replyErr is used to communicate replies between + // the actual processor (the WASM module) and wasmProcessor + replyCh chan sdk.CommandResponse + // replyErr is used to communicate errors between + // the actual processor (the WASM module) and wasmProcessor + replyErr chan error + // runModStopped is used to know when the run module stopped + runModStopped chan struct{} +} + +func NewWASMProcessor(ctx context.Context, logger zerolog.Logger, wasmPath string) (sdk.Processor, error) { + runtimeCtx, runtimeCancel := context.WithCancel(ctx) + r := wazero.NewRuntimeWithConfig( + runtimeCtx, + wazero.NewRuntimeConfig().WithCloseOnContextDone(true), + ) + wasi_snapshot_preview1.MustInstantiate(ctx, r) + + p := &wasmProcessor{ + logger: logger, + runtime: r, + runtimeCancel: runtimeCancel, + commandCh: make(chan sdk.Command), + replyCh: make(chan sdk.CommandResponse), + replyErr: make(chan error), + runModStopped: make(chan struct{}), + } + + err := p.exportFunctions(ctx) + if err != nil { + return nil, fmt.Errorf("failed exporting processor functions: %w", err) + } + + err = p.run(ctx, wasmPath) + if err != nil { + return nil, fmt.Errorf("failed running WASM module: %w", err) + } + + return p, nil +} + +func (p *wasmProcessor) exportFunctions(ctx context.Context) error { + // Build a host module, called `env`, which will expose + // functions which WASM processor module can use, + envBuilder := p.runtime.NewHostModuleBuilder("env") + envBuilder. + NewFunctionBuilder(). + WithFunc(p.nextCommand). + Export("nextCommand") + envBuilder. + NewFunctionBuilder(). + WithFunc(p.reply). + Export("reply") + + _, err := envBuilder.Instantiate(ctx) + return err +} + +func (p *wasmProcessor) nextCommand(ctx context.Context, m api.Module, ptr, allocSize uint32) uint32 { + p.logger.Trace().Msg("getting next command") + + cmd, ok := <-p.commandCh + if !ok { + p.logger.Info().Msg("command channel closed") + return sdk.ErrCodeNoMoreCommands + } + + bytes, err := serde.MarshalCommand(cmd) + if err != nil { + p.logger.Err(err).Msg("failed marshaling command") + p.replyErr <- fmt.Errorf("failed marshaling command: %w", err) + + return sdk.ErrCodeFailedGettingCommand + } + + return p.write(ctx, m, ptr, allocSize, bytes) +} + +func (p *wasmProcessor) write(_ context.Context, mod api.Module, ptr uint32, sizeAllocated uint32, bytes []byte) uint32 { + p.logger.Trace(). + Int("total_bytes", len(bytes)). + Uint32("allocated_size", sizeAllocated). + Msgf("writing command to module memory") + + if sizeAllocated < uint32(len(bytes)) { + p.logger.Error(). + Int("total_bytes", len(bytes)). + Uint32("allocated_size", sizeAllocated). + Msgf("insufficient memory") + + p.replyErr <- fmt.Errorf( + "insufficient memory allocated for reply, needed %v, allocated %v", + len(bytes), + sizeAllocated, + ) + return sdk.ErrCodeInsufficientSize + } + + bytesWritten := uint64(len(bytes)) + // The pointer is a linear memory offset, which is where we write the name. + if !mod.Memory().Write(ptr, bytes) { + p.logger.Error(). + Uint32("pointer", ptr). + Int("total_bytes", len(bytes)). + Uint32("wasm_module_mem_size", mod.Memory().Size()). + Msg("WASM module memory write is out of range") + + p.replyErr <- cerrors.New("WASM module memory write is out of range") + return sdk.ErrCodeMemoryOutOfRange + } + + p.logger.Trace().Msgf("bytes written: %v", bytesWritten) + return uint32(bytesWritten) +} + +func (p *wasmProcessor) reply(_ context.Context, m api.Module, ptr, size uint32) { + bytes, b := m.Memory().Read(ptr, size) + if !b { + p.logger.Error().Msg("failed reading reply") + p.replyErr <- cerrors.New("failed reading reply") + return + } + + cr, err := serde.UnmarshalCommandResponse(bytes) + if err != nil { + p.replyErr <- fmt.Errorf("failed deserializing command response: %w", err) + return + } + + p.replyCh <- cr +} + +// run instantiates a new module from the given path. +// Blocks as long as the module's start function is running. +func (p *wasmProcessor) run(ctx context.Context, path string) error { + p.logger. + Debug(). + Str("path", path). + Msg("running WASM processor") + + wasmBytes, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed reading WASM file %s: %w", path, err) + } + + // Compiling a module helps check for some errors early on + p.logger. + Debug(). + Str("path", path). + Msg("compiling module") + mod, err := p.runtime.CompileModule(ctx, wasmBytes) + if err != nil { + return fmt.Errorf("failed compiling WASM module: %w", err) + } + + // Needs to run in a goroutine because + // the WASM module is blocking as long as + // the "main" function is running + go func() { + p.logger.Debug().Msg("instantiating module") + _, err = p.runtime.InstantiateModule( + ctx, + mod, + wazero.NewModuleConfig(). + WithName("run-module"). + WithStdout(p.logger). + WithStderr(p.logger), + ) + p.runModStopped <- struct{}{} + if err != nil { + p.logger.Err(err).Msg("WASM module not instantiated or stopped") + } + }() + + return nil +} + +func (p *wasmProcessor) Specification() (sdk.Specification, error) { + p.commandCh <- &sdk.SpecifyCmd{} + + select { + case cr := <-p.replyCh: + resp, ok := cr.(*sdk.SpecifyResponse) + if !ok { + return sdk.Specification{}, fmt.Errorf("unexpected response type: %T", cr) + } + + return resp.Specification, resp.Error() + case err := <-p.replyErr: + return sdk.Specification{}, fmt.Errorf("reply error: %w", err) + } +} + +func (p *wasmProcessor) Configure(_ context.Context, cfg map[string]string) error { + p.commandCh <- &sdk.ConfigureCmd{ + ConfigMap: cfg, + } + + select { + case cr := <-p.replyCh: + resp, ok := cr.(*sdk.ConfigureResponse) + if !ok { + return fmt.Errorf("unexpected response type: %T", cr) + } + + return resp.Error() + case err := <-p.replyErr: + return fmt.Errorf("reply error: %w", err) + } +} + +func (p *wasmProcessor) Open(context.Context) error { + p.commandCh <- &sdk.OpenCmd{} + + select { + case cr := <-p.replyCh: + resp, ok := cr.(*sdk.OpenResponse) + if !ok { + return fmt.Errorf("unexpected response type: %T", cr) + } + + return resp.Error() + case err := <-p.replyErr: + return fmt.Errorf("reply error: %w", err) + } +} + +func (p *wasmProcessor) Process(_ context.Context, records []opencdc.Record) []sdk.ProcessedRecord { + p.commandCh <- &sdk.ProcessCmd{ + Records: records, + } + + select { + case cr := <-p.replyCh: + resp, ok := cr.(*sdk.ProcessResponse) + if !ok { + return p.makeErrorRecords(len(records), fmt.Errorf("unexpected response type: %T", cr)) + } + if resp.Error() != nil { + return p.makeErrorRecords(len(records), resp.Error()) + } + + return resp.Records + case err := <-p.replyErr: + return p.makeErrorRecords(len(records), fmt.Errorf("reply error: %w", err)) + } +} + +func (p *wasmProcessor) Teardown(context.Context) error { + // Closing the command channel will send ErrCodeFailedGettingCommand + // to the WASM module, which will exit. + // TODO handle case when the WASM module is unresponsive. + close(p.commandCh) + <-p.runModStopped + p.runtimeCancel() + return nil +} + +func (p *wasmProcessor) makeErrorRecords(num int, err error) []sdk.ProcessedRecord { + out := make([]sdk.ProcessedRecord, num) + for i := 0; i < num; i++ { + out[i] = sdk.ErrorRecord{Err: err} + } + + return out +} diff --git a/pkg/plugin/processor/standalone/wasm_processor_test.go b/pkg/plugin/processor/standalone/wasm_processor_test.go new file mode 100644 index 000000000..1998a2502 --- /dev/null +++ b/pkg/plugin/processor/standalone/wasm_processor_test.go @@ -0,0 +1,161 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package standalone + +import ( + "context" + "errors" //nolint:depguard // needed to test external error + "fmt" + "os" + "os/exec" + "testing" + + "github.com/conduitio/conduit-commons/opencdc" + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/matryer/is" + "github.com/rs/zerolog" +) + +func TestMain(m *testing.M) { + cmd := exec.Command("bash", "./test/build-test-processors.sh") + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + // Run the command + err := cmd.Run() + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "error executing bash script: %v", err) + os.Exit(1) + } + + os.Exit(m.Run()) +} + +func TestWASMProcessor_MalformedProcessor(t *testing.T) { + is := is.New(t) + ctx := context.Background() + + _, err := NewWASMProcessor(ctx, zerolog.Nop(), "./wasm_processor_test.go") + is.True(err != nil) + is.Equal(err.Error(), "failed running WASM module: failed compiling WASM module: invalid magic number") +} + +func TestWASMProcessor_SpecifyError(t *testing.T) { + is := is.New(t) + ctx := context.Background() + + underTest, err := NewWASMProcessor(ctx, zerolog.Nop(), testPluginDir+"specify_error/processor.wasm") + is.NoErr(err) + + _, err = underTest.Specification() + is.Equal(err, errors.New("boom")) +} + +func TestWASMProcessor_Specify(t *testing.T) { + is := is.New(t) + ctx := context.Background() + + underTest, err := NewWASMProcessor( + ctx, + zerolog.New(zerolog.NewTestWriter(t)), + testPluginDir+"chaos_processor/processor.wasm", + ) + is.NoErr(err) + + gotSpec, err := underTest.Specification() + is.NoErr(err) + + param := sdk.Parameter{ + Default: "success", + Type: sdk.ParameterTypeString, + Description: "prefix", + Validations: []sdk.Validation{ + { + Type: sdk.ValidationTypeInclusion, + Value: "success,error,panic", + }, + }, + } + wantSpec := sdk.Specification{ + Name: "chaos-processor", + Summary: "chaos processor summary", + Description: "chaos processor description", + Version: "v1.3.5", + Author: "Meroxa, Inc.", + Parameters: map[string]sdk.Parameter{ + "configure": param, + "open": param, + "process.prefix": { + Default: "", + Type: sdk.ParameterTypeString, + Description: "prefix to be added to the payload's after", + Validations: []sdk.Validation{ + { + Type: sdk.ValidationTypeRequired, + }, + }, + }, + "process": param, + "teardown": param, + }, + } + is.Equal( + gotSpec, + wantSpec, + ) + + is.NoErr(underTest.Teardown(ctx)) +} + +func TestWASMProcessor_Process(t *testing.T) { + is := is.New(t) + ctx := context.Background() + + underTest, err := NewWASMProcessor( + ctx, + zerolog.New(zerolog.NewTestWriter(t)), + testPluginDir+"chaos_processor/processor.wasm", + ) + is.NoErr(err) + + is.NoErr(underTest.Configure(ctx, map[string]string{"process.prefix": "hello!\n\n"})) + + is.NoErr(underTest.Open(ctx)) + + processed := underTest.Process(ctx, nil) + is.Equal(0, len(processed)) + + processed = underTest.Process(ctx, []opencdc.Record{}) + is.Equal(0, len(processed)) + + input := opencdc.Record{ + Position: opencdc.Position("first left then right"), + Operation: opencdc.OperationCreate, + Metadata: map[string]string{ + "street": "23rd", + }, + Key: opencdc.RawData("broken"), + Payload: opencdc.Change{ + After: opencdc.RawData("oranges"), + }, + } + want := sdk.SingleRecord(input.Clone()) + want.Payload.After = opencdc.RawData("hello!\n\n" + string(want.Payload.After.Bytes())) + processed = underTest.Process(ctx, []opencdc.Record{input}) + is.Equal(1, len(processed)) + is.Equal(want, processed[0]) + + is.NoErr(underTest.Teardown(ctx)) +} diff --git a/pkg/plugin/registry.go b/pkg/plugin/registry.go new file mode 100644 index 000000000..0339a2d75 --- /dev/null +++ b/pkg/plugin/registry.go @@ -0,0 +1,194 @@ +// Copyright © 2023 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plugin + +import ( + "context" + "os" + "path" + "path/filepath" + "sync" + + "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/rs/zerolog" +) + +type Spec interface { + GetName() string + GetVersion() string +} + +type GetSpecFn func(ctx context.Context, logger zerolog.Logger, path string) (Spec, error) + +// Registry is a generic directory registry of plugins, +// organized by plugin type, name and version. +// Every file in the specified directory is considered a plugin +// (directories are skipped). +// A registry is instantiated with a function, which can +// get specifications from a plugin, given its path on the file system. +type Registry struct { + logger log.CtxLogger + getSpecs GetSpecFn + + pluginDir string + // plugins stores plugin blueprints in a 2D map, first key is the plugin + // name, the second key is the plugin version + plugins map[string]map[string]blueprint + // m guards plugins from being concurrently accessed + m sync.RWMutex +} + +type blueprint struct { + fullName FullName + specification Spec + path string + // TODO store hash of plugin binary and compare before running the binary to + // ensure someone can't switch the plugin after we registered it +} + +func NewRegistry( + logger log.CtxLogger, + pluginDir string, + getSpecs GetSpecFn, +) *Registry { + r := &Registry{ + logger: logger.WithComponent("standalone.Registry"), + getSpecs: getSpecs, + } + + if pluginDir != "" { + // extract absolute path to make it clearer in the logs what directory is used + absPluginDir, err := filepath.Abs(pluginDir) + if err != nil { + r.logger.Warn(context.Background()).Err(err).Msg("could not extract absolute plugins path") + } else { + r.pluginDir = absPluginDir // store plugin dir for hot reloads + r.reloadPlugins() + } + } + + r.logger.Info(context.Background()). + Str(log.PluginPathField, r.pluginDir). + Int("count", len(r.List())). + Msg("standalone plugins initialized") + + return r +} + +func newFullName(pluginName, pluginVersion string) FullName { + return NewFullName(PluginTypeStandalone, pluginName, pluginVersion) +} + +func (r *Registry) reloadPlugins() { + plugins := r.loadPlugins(context.Background(), r.pluginDir) + r.m.Lock() + r.plugins = plugins + r.m.Unlock() +} + +func (r *Registry) loadPlugins(ctx context.Context, pluginDir string) map[string]map[string]blueprint { + r.logger.Debug(ctx).Msgf("loading plugins from directory %v", pluginDir) + plugins := make(map[string]map[string]blueprint) + + dirEntries, err := os.ReadDir(pluginDir) + if err != nil { + r.logger.Warn(ctx).Err(err).Msg("could not read plugin directory") + return plugins // return empty map + } + warn := func(ctx context.Context, err error, pluginPath string) { + r.logger.Warn(ctx). + Err(err). + Str(log.PluginPathField, pluginPath). + Msgf("could not load standalone plugin") + } + + for _, dirEntry := range dirEntries { + if dirEntry.IsDir() { + // skip directories + continue + } + + pluginPath := path.Join(pluginDir, dirEntry.Name()) + + // create dispenser without a logger to not spam logs on refresh + specs, err := r.getSpecs(ctx, zerolog.Nop(), pluginPath) + if err != nil { + err = cerrors.Errorf("failed to get specs: %w", err) + warn(ctx, err, pluginPath) + continue + } + + versionMap := plugins[specs.GetName()] + if versionMap == nil { + versionMap = make(map[string]blueprint) + plugins[specs.GetName()] = versionMap + } + + fullName := newFullName(specs.GetName(), specs.GetVersion()) + if conflict, ok := versionMap[specs.GetVersion()]; ok { + err = cerrors.Errorf("conflict detected, plugin %v already registered, please remove either %v or %v, these plugins won't be usable until that happens", fullName, conflict.path, pluginPath) + warn(ctx, err, pluginPath) + // delete plugin from map at the end so that further duplicates can + // still be found + defer func() { + delete(versionMap, specs.GetVersion()) + if len(versionMap) == 0 { + delete(plugins, specs.GetName()) + } + }() + continue + } + + bp := blueprint{ + fullName: fullName, + specification: specs, + path: pluginPath, + } + versionMap[specs.GetVersion()] = bp + + latestFullName := versionMap[PluginVersionLatest].fullName + if fullName.PluginVersionGreaterThan(latestFullName) { + versionMap[PluginVersionLatest] = bp + r.logger.Debug(ctx). + Str(log.PluginPathField, pluginPath). + Str(log.PluginNameField, string(bp.fullName)). + Msg("set plugin as latest") + } + + r.logger.Debug(ctx). + Str(log.PluginPathField, pluginPath). + Str(log.PluginNameField, string(bp.fullName)). + Msg("loaded standalone plugin") + } + + return plugins +} + +func (r *Registry) List() map[FullName]Spec { + r.m.RLock() + defer r.m.RUnlock() + + specs := make(map[FullName]Spec, len(r.plugins)) + for _, versions := range r.plugins { + for version, bp := range versions { + if version == PluginVersionLatest { + continue // skip latest versions + } + specs[bp.fullName] = bp.specification + } + } + return specs +} diff --git a/pkg/plugin/service.go b/pkg/plugin/service.go index 385010127..377f370ce 100644 --- a/pkg/plugin/service.go +++ b/pkg/plugin/service.go @@ -19,6 +19,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/plugin/connector" ) // registry is an object that can create new plugin dispensers. We need to use @@ -32,8 +33,8 @@ import ( // separate process and communicates with it via gRPC. These plugins are // compiled independently of Conduit and can be included at runtime. type registry interface { - NewDispenser(logger log.CtxLogger, name FullName) (Dispenser, error) - List() map[FullName]Specification + NewDispenser(logger log.CtxLogger, name FullName) (connector.Dispenser, error) + List() map[FullName]connector.Specification } type Service struct { @@ -51,11 +52,11 @@ func NewService(logger log.CtxLogger, builtin registry, standalone registry) *Se } } -func (s *Service) Check(_ context.Context) error { +func (s *Service) Check(context.Context) error { return nil } -func (s *Service) NewDispenser(logger log.CtxLogger, name string) (Dispenser, error) { +func (s *Service) NewDispenser(logger log.CtxLogger, name string) (connector.Dispenser, error) { logger = logger.WithComponent("plugin") fullName := FullName(name) @@ -76,11 +77,11 @@ func (s *Service) NewDispenser(logger log.CtxLogger, name string) (Dispenser, er } } -func (s *Service) List(context.Context) (map[string]Specification, error) { +func (s *Service) ListConnectors(context.Context) (map[string]connector.Specification, error) { builtinSpecs := s.builtin.List() standaloneSpecs := s.standalone.List() - specs := make(map[string]Specification, len(builtinSpecs)+len(standaloneSpecs)) + specs := make(map[string]connector.Specification, len(builtinSpecs)+len(standaloneSpecs)) for k, v := range builtinSpecs { specs[string(k)] = v } @@ -91,7 +92,12 @@ func (s *Service) List(context.Context) (map[string]Specification, error) { return specs, nil } -func (s *Service) ValidateSourceConfig(ctx context.Context, d Dispenser, settings map[string]string) (err error) { +func (s *Service) ValidateSourceConfig(ctx context.Context, name string, settings map[string]string) (err error) { + d, err := s.NewDispenser(s.logger, name) + if err != nil { + return cerrors.Errorf("couldn't get dispenser: %w", err) + } + src, err := d.DispenseSource() if err != nil { return cerrors.Errorf("could not dispense source: %w", err) @@ -112,7 +118,12 @@ func (s *Service) ValidateSourceConfig(ctx context.Context, d Dispenser, setting return nil } -func (s *Service) ValidateDestinationConfig(ctx context.Context, d Dispenser, settings map[string]string) (err error) { +func (s *Service) ValidateDestinationConfig(ctx context.Context, name string, settings map[string]string) (err error) { + d, err := s.NewDispenser(s.logger, name) + if err != nil { + return cerrors.Errorf("couldn't get dispenser: %w", err) + } + dest, err := d.DispenseDestination() if err != nil { return cerrors.Errorf("could not dispense destination: %w", err) diff --git a/pkg/provisioning/interfaces.go b/pkg/provisioning/interfaces.go index eb055269c..bc3b62c0f 100644 --- a/pkg/provisioning/interfaces.go +++ b/pkg/provisioning/interfaces.go @@ -20,7 +20,7 @@ import ( "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/pipeline" - "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" ) @@ -61,5 +61,5 @@ type ProcessorService interface { } type PluginService interface { - NewDispenser(ctx log.CtxLogger, name string) (plugin.Dispenser, error) + NewDispenser(ctx log.CtxLogger, name string) (connectorPlugin.Dispenser, error) } diff --git a/pkg/provisioning/mock/provisioning.go b/pkg/provisioning/mock/provisioning.go index eaa29d920..1b2c98728 100644 --- a/pkg/provisioning/mock/provisioning.go +++ b/pkg/provisioning/mock/provisioning.go @@ -15,7 +15,7 @@ import ( connector "github.com/conduitio/conduit/pkg/connector" log "github.com/conduitio/conduit/pkg/foundation/log" pipeline "github.com/conduitio/conduit/pkg/pipeline" - plugin "github.com/conduitio/conduit/pkg/plugin" + connector0 "github.com/conduitio/conduit/pkg/plugin/connector" processor "github.com/conduitio/conduit/pkg/processor" gomock "go.uber.org/mock/gomock" ) @@ -437,10 +437,10 @@ func (m *PluginService) EXPECT() *PluginServiceMockRecorder { } // NewDispenser mocks base method. -func (m *PluginService) NewDispenser(arg0 log.CtxLogger, arg1 string) (plugin.Dispenser, error) { +func (m *PluginService) NewDispenser(arg0 log.CtxLogger, arg1 string) (connector0.Dispenser, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NewDispenser", arg0, arg1) - ret0, _ := ret[0].(plugin.Dispenser) + ret0, _ := ret[0].(connector0.Dispenser) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/pkg/provisioning/service_test.go b/pkg/provisioning/service_test.go index f036b9709..2a5c3049a 100644 --- a/pkg/provisioning/service_test.go +++ b/pkg/provisioning/service_test.go @@ -27,8 +27,8 @@ import ( "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/pipeline" "github.com/conduitio/conduit/pkg/plugin" - "github.com/conduitio/conduit/pkg/plugin/builtin" - "github.com/conduitio/conduit/pkg/plugin/standalone" + "github.com/conduitio/conduit/pkg/plugin/connector/builtin" + "github.com/conduitio/conduit/pkg/plugin/connector/standalone" "github.com/conduitio/conduit/pkg/processor" "github.com/conduitio/conduit/pkg/processor/procbuiltin" p1 "github.com/conduitio/conduit/pkg/provisioning/test/pipelines1" diff --git a/pkg/web/api/mock/plugin.go b/pkg/web/api/mock/plugin.go index 7120171bb..2f78249c3 100644 --- a/pkg/web/api/mock/plugin.go +++ b/pkg/web/api/mock/plugin.go @@ -12,7 +12,7 @@ import ( context "context" reflect "reflect" - plugin "github.com/conduitio/conduit/pkg/plugin" + connector "github.com/conduitio/conduit/pkg/plugin/connector" gomock "go.uber.org/mock/gomock" ) @@ -39,17 +39,17 @@ func (m *PluginOrchestrator) EXPECT() *PluginOrchestratorMockRecorder { return m.recorder } -// List mocks base method. -func (m *PluginOrchestrator) List(arg0 context.Context) (map[string]plugin.Specification, error) { +// ListConnectors mocks base method. +func (m *PluginOrchestrator) ListConnectors(arg0 context.Context) (map[string]connector.Specification, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "List", arg0) - ret0, _ := ret[0].(map[string]plugin.Specification) + ret := m.ctrl.Call(m, "ListConnectors", arg0) + ret0, _ := ret[0].(map[string]connector.Specification) ret1, _ := ret[1].(error) return ret0, ret1 } -// List indicates an expected call of List. -func (mr *PluginOrchestratorMockRecorder) List(arg0 any) *gomock.Call { +// ListConnectors indicates an expected call of ListConnectors. +func (mr *PluginOrchestratorMockRecorder) ListConnectors(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*PluginOrchestrator)(nil).List), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListConnectors", reflect.TypeOf((*PluginOrchestrator)(nil).ListConnectors), arg0) } diff --git a/pkg/web/api/plugin_v1.go b/pkg/web/api/plugin_v1.go index afb131746..f5e9cbea0 100644 --- a/pkg/web/api/plugin_v1.go +++ b/pkg/web/api/plugin_v1.go @@ -19,7 +19,7 @@ import ( "regexp" "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/web/api/status" "github.com/conduitio/conduit/pkg/web/api/toproto" apiv1 "github.com/conduitio/conduit/proto/api/v1" @@ -30,8 +30,8 @@ import ( // PluginOrchestrator defines a CRUD interface that manages the Plugin resource. type PluginOrchestrator interface { - // List will return all plugins' specs. - List(ctx context.Context) (map[string]plugin.Specification, error) + // ListConnectors will return all connector plugins' specs. + ListConnectors(ctx context.Context) (map[string]connectorPlugin.Specification, error) } type PluginAPIv1 struct { @@ -60,7 +60,7 @@ func (p *PluginAPIv1) ListPlugins( } } - mp, err := p.ps.List(ctx) + mp, err := p.ps.ListConnectors(ctx) if err != nil { return nil, status.PluginError(err) } diff --git a/pkg/web/api/plugin_v1_test.go b/pkg/web/api/plugin_v1_test.go index 68bc4e892..8bb03bda0 100644 --- a/pkg/web/api/plugin_v1_test.go +++ b/pkg/web/api/plugin_v1_test.go @@ -19,7 +19,7 @@ import ( "sort" "testing" - "github.com/conduitio/conduit/pkg/plugin" + connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/web/api/mock" "github.com/conduitio/conduit/pkg/web/api/toproto" apiv1 "github.com/conduitio/conduit/proto/api/v1" @@ -37,31 +37,31 @@ func TestPluginAPIv1_ListPluginByName(t *testing.T) { names := []string{"do-not-want-this-plugin", "want-p1", "want-p2", "skip", "another-skipped"} - plsMap := make(map[string]plugin.Specification) - pls := make([]plugin.Specification, 0) + plsMap := make(map[string]connectorPlugin.Specification) + pls := make([]connectorPlugin.Specification, 0) for _, name := range names { - ps := plugin.Specification{ + ps := connectorPlugin.Specification{ Name: name, Description: "desc", Version: "v1.0", Author: "Aaron", - SourceParams: map[string]plugin.Parameter{ + SourceParams: map[string]connectorPlugin.Parameter{ "param": { - Type: plugin.ParameterTypeString, - Validations: []plugin.Validation{{ - Type: plugin.ValidationTypeRequired, + Type: connectorPlugin.ParameterTypeString, + Validations: []connectorPlugin.Validation{{ + Type: connectorPlugin.ValidationTypeRequired, }}, }, }, - DestinationParams: map[string]plugin.Parameter{}, + DestinationParams: map[string]connectorPlugin.Parameter{}, } pls = append(pls, ps) plsMap[name] = ps } psMock.EXPECT(). - List(ctx). + ListConnectors(ctx). Return(plsMap, nil). Times(1) diff --git a/pkg/web/api/toproto/plugin.go b/pkg/web/api/toproto/plugin.go index 2dc7c1092..16d3042c6 100644 --- a/pkg/web/api/toproto/plugin.go +++ b/pkg/web/api/toproto/plugin.go @@ -15,29 +15,29 @@ package toproto import ( - "github.com/conduitio/conduit/pkg/plugin" + "github.com/conduitio/conduit/pkg/plugin/connector" apiv1 "github.com/conduitio/conduit/proto/api/v1" ) func _() { // An "invalid array index" compiler error signifies that the constant values have changed. var vTypes [1]struct{} - _ = vTypes[int(plugin.ValidationTypeRequired)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_REQUIRED)] - _ = vTypes[int(plugin.ValidationTypeGreaterThan)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_GREATER_THAN)] - _ = vTypes[int(plugin.ValidationTypeLessThan)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_LESS_THAN)] - _ = vTypes[int(plugin.ValidationTypeInclusion)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_INCLUSION)] - _ = vTypes[int(plugin.ValidationTypeExclusion)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_EXCLUSION)] - _ = vTypes[int(plugin.ValidationTypeRegex)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_REGEX)] + _ = vTypes[int(connector.ValidationTypeRequired)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_REQUIRED)] + _ = vTypes[int(connector.ValidationTypeGreaterThan)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_GREATER_THAN)] + _ = vTypes[int(connector.ValidationTypeLessThan)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_LESS_THAN)] + _ = vTypes[int(connector.ValidationTypeInclusion)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_INCLUSION)] + _ = vTypes[int(connector.ValidationTypeExclusion)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_EXCLUSION)] + _ = vTypes[int(connector.ValidationTypeRegex)-int(apiv1.PluginSpecifications_Parameter_Validation_TYPE_REGEX)] - _ = vTypes[int(plugin.ParameterTypeString)-int(apiv1.PluginSpecifications_Parameter_TYPE_STRING)] - _ = vTypes[int(plugin.ParameterTypeInt)-int(apiv1.PluginSpecifications_Parameter_TYPE_INT)] - _ = vTypes[int(plugin.ParameterTypeFloat)-int(apiv1.PluginSpecifications_Parameter_TYPE_FLOAT)] - _ = vTypes[int(plugin.ParameterTypeFile)-int(apiv1.PluginSpecifications_Parameter_TYPE_FILE)] - _ = vTypes[int(plugin.ParameterTypeBool)-int(apiv1.PluginSpecifications_Parameter_TYPE_BOOL)] - _ = vTypes[int(plugin.ParameterTypeDuration)-int(apiv1.PluginSpecifications_Parameter_TYPE_DURATION)] + _ = vTypes[int(connector.ParameterTypeString)-int(apiv1.PluginSpecifications_Parameter_TYPE_STRING)] + _ = vTypes[int(connector.ParameterTypeInt)-int(apiv1.PluginSpecifications_Parameter_TYPE_INT)] + _ = vTypes[int(connector.ParameterTypeFloat)-int(apiv1.PluginSpecifications_Parameter_TYPE_FLOAT)] + _ = vTypes[int(connector.ParameterTypeFile)-int(apiv1.PluginSpecifications_Parameter_TYPE_FILE)] + _ = vTypes[int(connector.ParameterTypeBool)-int(apiv1.PluginSpecifications_Parameter_TYPE_BOOL)] + _ = vTypes[int(connector.ParameterTypeDuration)-int(apiv1.PluginSpecifications_Parameter_TYPE_DURATION)] } -func Plugin(name string, in plugin.Specification) *apiv1.PluginSpecifications { +func Plugin(name string, in connector.Specification) *apiv1.PluginSpecifications { return &apiv1.PluginSpecifications{ Name: name, Summary: in.Summary, @@ -49,7 +49,7 @@ func Plugin(name string, in plugin.Specification) *apiv1.PluginSpecifications { } } -func PluginParamsMap(in map[string]plugin.Parameter) map[string]*apiv1.PluginSpecifications_Parameter { +func PluginParamsMap(in map[string]connector.Parameter) map[string]*apiv1.PluginSpecifications_Parameter { out := make(map[string]*apiv1.PluginSpecifications_Parameter) for k, v := range in { out[k] = &apiv1.PluginSpecifications_Parameter{ @@ -62,7 +62,7 @@ func PluginParamsMap(in map[string]plugin.Parameter) map[string]*apiv1.PluginSpe return out } -func PluginParamValidations(in []plugin.Validation) []*apiv1.PluginSpecifications_Parameter_Validation { +func PluginParamValidations(in []connector.Validation) []*apiv1.PluginSpecifications_Parameter_Validation { // we need an empty slice here so that the returned JSON would be "validations":[] instead of "validations":null out := make([]*apiv1.PluginSpecifications_Parameter_Validation, 0) for _, v := range in { @@ -74,6 +74,6 @@ func PluginParamValidations(in []plugin.Validation) []*apiv1.PluginSpecification return out } -func ValidationType(in plugin.ValidationType) apiv1.PluginSpecifications_Parameter_Validation_Type { +func ValidationType(in connector.ValidationType) apiv1.PluginSpecifications_Parameter_Validation_Type { return apiv1.PluginSpecifications_Parameter_Validation_Type(in) }