diff --git a/cmd/export_external_data.go b/cmd/export_external_data.go new file mode 100644 index 00000000..7239f1f3 --- /dev/null +++ b/cmd/export_external_data.go @@ -0,0 +1,84 @@ +package cmd + +import ( + "fmt" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/stellar/go/utils/apiclient" + "github.com/stellar/stellar-etl/internal/input" + "github.com/stellar/stellar-etl/internal/transform" + "github.com/stellar/stellar-etl/internal/utils" +) + +var externalDataCmd = &cobra.Command{ + Use: "export_external_data", + Short: "Exports external data updated over a specified timestamp range", + Long: "Exports external data updated over a specified timestamp range to an output file.", + Run: func(cmd *cobra.Command, args []string) { + cmdLogger.SetLevel(logrus.InfoLevel) + timestampArgs := utils.MustTimestampRangeFlags(cmd.Flags(), cmdLogger) + path := utils.MustBucketFlags(cmd.Flags(), cmdLogger) + provider := utils.MustProviderFlags(cmd.Flags(), cmdLogger) + cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger) + isTest, err := cmd.Flags().GetBool("testnet") + if err != nil { + cmdLogger.Fatal("could not get testnet boolean: ", err) + } + + outFile := mustOutFile(path) + numFailures := 0 + totalNumBytes := 0 + + switch provider { + case "retool": + var client *apiclient.APIClient + if isTest { + client = input.GetMockClient(provider) + } else { + client = nil + } + entities, err := input.GetEntityData[utils.RetoolEntityDataTransformInput](client, provider, timestampArgs.StartTime, timestampArgs.EndTime) + if err != nil { + cmdLogger.Fatal("could not read entity data: ", err) + } + + for _, transformInput := range entities { + transformed, err := transform.TransformRetoolEntityData(transformInput) + if err != nil { + numFailures += 1 + continue + } + + numBytes, err := exportEntry(transformed, outFile, nil) + if err != nil { + cmdLogger.LogError(fmt.Errorf("could not export entity data: %v", err)) + numFailures += 1 + continue + } + totalNumBytes += numBytes + } + outFile.Close() + cmdLogger.Info("Number of bytes written: ", totalNumBytes) + + printTransformStats(len(entities), numFailures) + + default: + panic("unsupported provider: " + provider) + } + + maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path) + }, +} + +func init() { + rootCmd.AddCommand(externalDataCmd) + utils.AddArchiveFlags("entity", externalDataCmd.Flags()) + utils.AddCloudStorageFlags(externalDataCmd.Flags()) + utils.AddTimestampRangeFlags(externalDataCmd.Flags()) + utils.AddProviderFlags(externalDataCmd.Flags()) + utils.AddTestFlags(externalDataCmd.Flags()) + externalDataCmd.MarkFlagRequired("provider") + externalDataCmd.MarkFlagRequired("start-time") + externalDataCmd.MarkFlagRequired("end-time") +} diff --git a/cmd/export_external_data_test.go b/cmd/export_external_data_test.go new file mode 100644 index 00000000..85797543 --- /dev/null +++ b/cmd/export_external_data_test.go @@ -0,0 +1,20 @@ +package cmd + +import ( + "testing" +) + +func TestExportExternalData(t *testing.T) { + tests := []cliTest{ + { + name: "external data from retool", + args: []string{"export_external_data", "--provider", "retool", "--start-time", "", "--end-time", "", "-o", gotTestDir(t, "external_data_retool.txt"), "--testnet"}, + golden: "external_data_retool.golden", + wantErr: nil, + }, + } + + for _, test := range tests { + runCLITest(t, test, "testdata/external_data/") + } +} diff --git a/go.mod b/go.mod index 718ca4e8..66b6b680 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.17.0 - github.com/stellar/go v0.0.0-20240905180041-acfaa0686213 + github.com/stellar/go v0.0.0-20241119162825-0112a1d34a9a github.com/stretchr/testify v1.9.0 github.com/xitongsys/parquet-go v1.6.2 github.com/xitongsys/parquet-go-source v0.0.0-20240122235623-d6294584ab18 @@ -29,6 +29,8 @@ require ( cloud.google.com/go/iam v1.1.8 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f // indirect + github.com/andybalholm/brotli v1.0.4 // indirect github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect github.com/apache/thrift v0.14.2 // indirect github.com/aws/aws-sdk-go v1.51.24 // indirect @@ -37,14 +39,17 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/djherbis/fscache v0.10.1 // indirect + github.com/fatih/structs v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gavv/monotime v0.0.0-20161010190848-47d58efa6955 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect @@ -52,7 +57,9 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.3 // indirect + github.com/imkira/go-interpol v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect github.com/klauspost/compress v1.17.6 // indirect @@ -60,6 +67,7 @@ require ( github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moul/http2curl v0.0.0-20161031194548-4e24498b31db // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.8 // indirect @@ -71,12 +79,22 @@ require ( github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 // indirect + github.com/sergi/go-diff v0.0.0-20161205080420-83532ca1c1ca // indirect + github.com/smartystreets/goconvey v1.8.1 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.10.0 // indirect github.com/spf13/cast v1.5.1 // indirect github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.34.0 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/yalp/jsonpath v0.0.0-20150812003900-31a79c7593bb // indirect + github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d // indirect + github.com/yudai/golcs v0.0.0-20150405163532-d1c525dea8ce // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.50.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 // indirect @@ -99,10 +117,11 @@ require ( google.golang.org/genproto v0.0.0-20240528184218-531527333157 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/grpc v1.64.0 // indirect + google.golang.org/grpc v1.64.1 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/djherbis/atime.v1 v1.0.0 // indirect gopkg.in/djherbis/stream.v1 v1.3.1 // indirect + gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d16de7d9..832d6e24 100644 --- a/go.sum +++ b/go.sum @@ -419,12 +419,14 @@ github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/Oth github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/schema v1.2.0 h1:YufUaxZYCKGFuAq3c96BOhjgd5nmXiOY9NGzF247Tsc= -github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= +github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E= +github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/guregu/null v4.0.0+incompatible h1:4zw0ckM7ECd6FNNddc3Fu4aty9nTlpkkzH7dPn4/4Gw= @@ -500,10 +502,13 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= @@ -543,10 +548,14 @@ github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739 h1:ykXz+pRRTibcSj github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739/go.mod h1:zUx1mhth20V3VKgL5jbd1BSQcW4Fy6Qs4PZvQwRFwzM= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= @@ -627,6 +636,10 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -641,8 +654,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI= github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI= -github.com/stellar/go v0.0.0-20240905180041-acfaa0686213 h1:224VUCwV1xmmeTru1zCmTHxvi2RECoHdfdWgd9ni518= -github.com/stellar/go v0.0.0-20240905180041-acfaa0686213/go.mod h1:rrFK7a8i2h9xad9HTfnSN/dTNEqXVHKAbkFeR7UxAgs= +github.com/stellar/go v0.0.0-20241119162825-0112a1d34a9a h1:ymP2bH0UqTRZ3dpNu3VCHnhbQjmUo9e8WsJogGOu+/I= +github.com/stellar/go v0.0.0-20241119162825-0112a1d34a9a/go.mod h1:2jxuLI6d8tmmeauAgFYApGrBen5x/FlRfCdatzgRJ7s= github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 h1:OzCVd0SV5qE3ZcDeSFCmOWLZfEWZ3Oe8KtmSOYKEVWE= github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -673,6 +686,7 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/xdrpp/goxdr v0.1.1 h1:E1B2c6E8eYhOVyd7yEpOyopzTPirUeF6mVOfXfGyJyc= github.com/xdrpp/goxdr v0.1.1/go.mod h1:dXo1scL/l6s7iME1gxHWo2XCppbHEKZS7m/KyYWkNzA= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= @@ -694,6 +708,8 @@ github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d h1:yJIizrfO599ot2 github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20150405163532-d1c525dea8ce h1:888GrqRxabUce7lj4OaoShPxodm3kXOMpSa85wdYzfY= github.com/yudai/golcs v0.0.0-20150405163532-d1c525dea8ce/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -760,6 +776,7 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -1259,8 +1276,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/internal/input/external_data.go b/internal/input/external_data.go new file mode 100644 index 00000000..57b28fe3 --- /dev/null +++ b/internal/input/external_data.go @@ -0,0 +1,177 @@ +package input + +import ( + "fmt" + "log" + "net/http" + "net/url" + + "github.com/stellar/go/support/http/httptest" + "github.com/stellar/go/utils/apiclient" + "github.com/stellar/stellar-etl/internal/utils" +) + +type ProviderConfig struct { + BaseURL string + AuthType string + AuthKeyEnv string + AuthHeaders map[string]interface{} + Endpoint string + QueryParams url.Values + RequestType string +} + +func GetProviderConfig(provider string) ProviderConfig { + var providerConfig ProviderConfig + switch provider { + case "retool": + providerConfig = ProviderConfig{ + BaseURL: "https://xrri-vvsg-obfa.n7c.xano.io/api:glgSAjxV", + AuthType: "api_key", + AuthHeaders: map[string]interface{}{"api_key": utils.GetEnv("RETOOL_API_KEY", "test-api-key")}, + RequestType: "GET", + Endpoint: "apps_details", + QueryParams: url.Values{}, + } + default: + panic("unsupported provider: " + provider) + } + return providerConfig +} + +func GetEntityData[T any](client *apiclient.APIClient, provider string, startTime string, endTime string) ([]T, error) { + providerConfig := GetProviderConfig(provider) + + if client == nil { + client = &apiclient.APIClient{ + BaseURL: providerConfig.BaseURL, + AuthType: providerConfig.AuthType, + AuthHeaders: providerConfig.AuthHeaders, + } + } + reqParams := apiclient.RequestParams{ + RequestType: providerConfig.RequestType, + Endpoint: providerConfig.Endpoint, + QueryParams: providerConfig.QueryParams, + } + result, err := client.CallAPI(reqParams) + if err != nil { + return nil, fmt.Errorf("failed to call API: %w", err) + } + + // Assert that the result is a slice of interfaces + resultSlice, ok := result.([]interface{}) + if !ok { + return nil, fmt.Errorf("Result is not a slice of interface") + } + dataSlice := []T{} + + for i, item := range resultSlice { + if itemMap, ok := item.(map[string]interface{}); ok { + var resp T + err := utils.MapToStruct(itemMap, &resp) + if err != nil { + log.Printf("Error converting map to struct: %v", err) + continue + } + dataSlice = append(dataSlice, resp) + } else { + fmt.Printf("Item %d is not a map\n", i) + } + } + + return dataSlice, nil +} + +func GetMockClient(provider string) *apiclient.APIClient { + var mockResponses []httptest.ResponseData + switch provider { + case "retool": + mockResponses = []httptest.ResponseData{ + { + Status: http.StatusOK, + Body: `[ + { + "id": 16, + "created_at": 1706749912776, + "updated_at": null, + "custodial": true, + "non_custodial": true, + "home_domains_id": 240, + "name": "El Dorado", + "description": "", + "website_url": "", + "sdp_enabled": false, + "soroban_enabled": false, + "notes": "", + "verified": false, + "fee_sponsor": false, + "account_sponsor": false, + "live": true, + "status": "live", + "_home_domain": { + "id": 240, + "created_at": 1706749903897, + "home_domain": "eldorado.io", + "updated_at": 1706749903897 + }, + "_app_geographies_details": [ + { + "id": 39, + "apps_id": 16, + "created_at": 1707887845605, + "geographies_id": [ + { + "id": 176, + "created_at": 1691020699576, + "updated_at": 1706650713745, + "name": "Argentina", + "official_name": "The Argentine Republic" + }, + { + "id": 273, + "created_at": 1691020699834, + "updated_at": 1706650708355, + "name": "Brazil", + "official_name": "The Federative Republic of Brazil" + } + ], + "retail": false, + "enterprise": false + } + ], + "_app_to_ramps_integrations": [ + { + "id": 18, + "created_at": 1707617027154, + "anchors_id": 28, + "apps_id": 16, + "_anchor": { + "id": 28, + "created_at": 1705423531705, + "name": "MoneyGram", + "updated_at": 1706596979487, + "home_domains_id": 203 + } + } + ] + } + ]`, + Header: nil, + }, + } + default: + panic("unsupported provider: " + provider) + } + hmock := httptest.NewClient() + providerConfig := GetProviderConfig(provider) + + hmock.On("GET", fmt.Sprintf("%s/%s", providerConfig.BaseURL, providerConfig.Endpoint)). + ReturnMultipleResults(mockResponses) + + mockClient := &apiclient.APIClient{ + BaseURL: providerConfig.BaseURL, + HTTP: hmock, + } + return mockClient +} diff --git a/internal/input/external_data_test.go b/internal/input/external_data_test.go new file mode 100644 index 00000000..f4db8fe8 --- /dev/null +++ b/internal/input/external_data_test.go @@ -0,0 +1,88 @@ +package input + +import ( + "testing" + + "github.com/stellar/stellar-etl/internal/utils" + "github.com/stretchr/testify/assert" +) + +func getEntityDataHelper[T any](t *testing.T, provider string, expected []T) { + mockClient := GetMockClient(provider) + result, err := GetEntityData[T](mockClient, provider, "", "") + if err != nil { + t.Fatalf("Error calling GetEntityData: %v", err) + } + + assert.Equal(t, expected, result) +} + +func TestGetEntityDataForRetool(t *testing.T) { + expected := []utils.RetoolEntityDataTransformInput{ + { + ID: 16, + Name: "El Dorado", + Status: "live", + CreatedAt: 1706749912776, + UpdatedAt: nil, + Custodial: true, + NonCustodial: true, + HomeDomainsID: 240, + Description: "", + WebsiteURL: "", + SdpEnabled: false, + SorobanEnabled: false, + Notes: "", + Verified: false, + FeeSponsor: false, + AccountSponsor: false, + Live: true, + HomeDomain: utils.HomeDomain{ + ID: 240, + CreatedAt: 1706749903897, + UpdatedAt: 1706749903897, + HomeDomain: "eldorado.io", + }, + AppGeographiesDetails: []utils.AppGeographyDetail{ + { + ID: 39, + AppsID: 16, + CreatedAt: 1707887845605, + GeographiesID: []utils.Geography{ + { + ID: 176, + CreatedAt: 1691020699576, + UpdatedAt: 1706650713745, + Name: "Argentina", + OfficialName: "The Argentine Republic", + }, + { + ID: 273, + CreatedAt: 1691020699834, + UpdatedAt: 1706650708355, + Name: "Brazil", + OfficialName: "The Federative Republic of Brazil", + }, + }, + Retail: false, + Enterprise: false, + }, + }, + AppToRampsIntegrations: []utils.AppToRampIntegration{ + { + ID: 18, + CreatedAt: 1707617027154, + AnchorsID: 28, + AppsID: 16, + Anchor: utils.Anchor{ + ID: 28, + CreatedAt: 1705423531705, + UpdatedAt: 1706596979487, + Name: "MoneyGram", + }, + }, + }, + }, + } + getEntityDataHelper[utils.RetoolEntityDataTransformInput](t, "retool", expected) +} diff --git a/internal/transform/retool_entity_data.go b/internal/transform/retool_entity_data.go new file mode 100644 index 00000000..d964b4a6 --- /dev/null +++ b/internal/transform/retool_entity_data.go @@ -0,0 +1,50 @@ +package transform + +import ( + "github.com/stellar/stellar-etl/internal/utils" +) + +func TransformRetoolEntityData(entityData utils.RetoolEntityDataTransformInput) (EntityDataTransformOutput, error) { + transformedRetoolEntityData := EntityDataTransformOutput{ + ID: entityData.ID, + Name: entityData.Name, + HomeDomain: entityData.HomeDomain.HomeDomain, + Status: entityData.Status, + CreatedAt: entityData.CreatedAt, + UpdatedAt: entityData.UpdatedAt, + Custodial: entityData.Custodial, + NonCustodial: entityData.NonCustodial, + HomeDomainsID: entityData.HomeDomainsID, + Description: entityData.Description, + WebsiteURL: entityData.WebsiteURL, + SdpEnabled: entityData.SdpEnabled, + SorobanEnabled: entityData.SorobanEnabled, + Notes: entityData.Notes, + Verified: entityData.Verified, + FeeSponsor: entityData.FeeSponsor, + AccountSponsor: entityData.AccountSponsor, + Live: entityData.Live, + AppGeographies: mapGeographies(entityData.AppGeographiesDetails), + Ramps: mapRamps(entityData.AppToRampsIntegrations), + } + + return transformedRetoolEntityData, nil +} + +func mapGeographies(appGeographies []utils.AppGeographyDetail) []string { + var geographies []string + for _, geoDetail := range appGeographies { + for _, geo := range geoDetail.GeographiesID { + geographies = append(geographies, geo.Name) + } + } + return geographies +} + +func mapRamps(appRamps []utils.AppToRampIntegration) []string { + var ramps []string + for _, ramp := range appRamps { + ramps = append(ramps, ramp.Anchor.Name) + } + return ramps +} diff --git a/internal/transform/retool_entity_data_test.go b/internal/transform/retool_entity_data_test.go new file mode 100644 index 00000000..13630e31 --- /dev/null +++ b/internal/transform/retool_entity_data_test.go @@ -0,0 +1,99 @@ +package transform + +import ( + "testing" + + "github.com/stellar/stellar-etl/internal/utils" + "github.com/stretchr/testify/assert" +) + +func TestTransformRetoolEntityData(t *testing.T) { + output, _ := TransformRetoolEntityData(utils.RetoolEntityDataTransformInput{ + ID: 16, + Name: "El Dorado", + Status: "live", + CreatedAt: 1706749912776, + UpdatedAt: nil, + Custodial: true, + NonCustodial: true, + HomeDomainsID: 240, + Description: "", + WebsiteURL: "", + SdpEnabled: false, + SorobanEnabled: false, + Notes: "", + Verified: false, + FeeSponsor: false, + AccountSponsor: false, + Live: true, + HomeDomain: utils.HomeDomain{ + ID: 240, + CreatedAt: 1706749903897, + UpdatedAt: 1706749903897, + HomeDomain: "eldorado.io", + }, + AppGeographiesDetails: []utils.AppGeographyDetail{ + { + ID: 39, + AppsID: 16, + CreatedAt: 1707887845605, + GeographiesID: []utils.Geography{ + { + ID: 176, + CreatedAt: 1691020699576, + UpdatedAt: 1706650713745, + Name: "Argentina", + OfficialName: "The Argentine Republic", + }, + { + ID: 273, + CreatedAt: 1691020699834, + UpdatedAt: 1706650708355, + Name: "Brazil", + OfficialName: "The Federative Republic of Brazil", + }, + }, + Retail: false, + Enterprise: false, + }, + }, + AppToRampsIntegrations: []utils.AppToRampIntegration{ + { + ID: 18, + CreatedAt: 1707617027154, + AnchorsID: 28, + AppsID: 16, + Anchor: utils.Anchor{ + ID: 28, + CreatedAt: 1705423531705, + UpdatedAt: 1706596979487, + Name: "MoneyGram", + }, + }, + }, + }, + ) + expectedOutput := EntityDataTransformOutput{ + ID: 16, + Name: "El Dorado", + Status: "live", + CreatedAt: 1706749912776, + UpdatedAt: nil, + Custodial: true, + NonCustodial: true, + HomeDomainsID: 240, + Description: "", + WebsiteURL: "", + SdpEnabled: false, + SorobanEnabled: false, + Notes: "", + Verified: false, + FeeSponsor: false, + AccountSponsor: false, + Live: true, + HomeDomain: "eldorado.io", + AppGeographies: []string{"Argentina", "Brazil"}, + Ramps: []string{"MoneyGram"}, + } + assert.Equal(t, expectedOutput, output) +} diff --git a/internal/transform/schema.go b/internal/transform/schema.go index 1bcce561..cf990ba0 100644 --- a/internal/transform/schema.go +++ b/internal/transform/schema.go @@ -629,3 +629,26 @@ type ContractEventOutput struct { DataDecoded map[string]string `json:"data_decoded"` ContractEventXDR string `json:"contract_event_xdr"` } + +type EntityDataTransformOutput struct { + ID int `json:"id"` + Name string `json:"name"` + HomeDomain string `json:"home_domain"` + Status string `json:"status"` + CreatedAt int64 `json:"created_at"` + UpdatedAt *int64 `json:"updated_at"` + Custodial bool `json:"custodial"` + NonCustodial bool `json:"non_custodial"` + HomeDomainsID int `json:"home_domains_id"` + Description string `json:"description"` + WebsiteURL string `json:"website_url"` + SdpEnabled bool `json:"sdp_enabled"` + SorobanEnabled bool `json:"soroban_enabled"` + Notes string `json:"notes"` + Verified bool `json:"verified"` + FeeSponsor bool `json:"fee_sponsor"` + AccountSponsor bool `json:"account_sponsor"` + Live bool `json:"live"` + AppGeographies []string `json:"app_geographies"` + Ramps []string `json:"ramps"` +} diff --git a/internal/utils/main.go b/internal/utils/main.go index 4456f222..0c50c464 100644 --- a/internal/utils/main.go +++ b/internal/utils/main.go @@ -3,9 +3,11 @@ package utils import ( "context" "encoding/hex" + "encoding/json" "errors" "fmt" "math/big" + "os" "time" "github.com/spf13/pflag" @@ -287,6 +289,19 @@ func AddExportTypeFlags(flags *pflag.FlagSet) { flags.BoolP("export-ttl", "", false, "set in order to export ttl changes") } +func AddTimestampRangeFlags(flags *pflag.FlagSet) { + flags.String("start-time", "", "Start timestamp of the range") + flags.String("end-time", "", "End timestamp of the range") +} + +func AddProviderFlags(flags *pflag.FlagSet) { + flags.StringP("provider", "p", "", "Third party provider name. Example: retool, github") +} + +func AddTestFlags(flags *pflag.FlagSet) { + flags.Bool("testnet", false, "If set, will connect to Testnet instead of Mainnet.") +} + // TODO: https://stellarorg.atlassian.net/browse/HUBBLE-386 better flags/params // Some flags should be named better type FlagValues struct { @@ -452,6 +467,11 @@ type CommonFlagValues struct { WriteParquet bool } +type TimestampRangeFlagValues struct { + StartTime string + EndTime string +} + // MustCommonFlags gets the values of the the flags common to all commands: end-ledger and strict-export. // If any do not exist, it stops the program fatally using the logger func MustCommonFlags(flags *pflag.FlagSet, logger *EtlLogger) CommonFlagValues { @@ -650,6 +670,31 @@ func MustExportTypeFlags(flags *pflag.FlagSet, logger *EtlLogger) map[string]boo return exports } +func MustTimestampRangeFlags(flags *pflag.FlagSet, logger *EtlLogger) TimestampRangeFlagValues { + startTime, err := flags.GetString("start-time") + if err != nil { + logger.Fatal("could not get start time of the range: ", err) + } + + endTime, err := flags.GetString("end-time") + if err != nil { + logger.Fatal("could not get end time of the range: ", err) + } + + return TimestampRangeFlagValues{ + StartTime: startTime, + EndTime: endTime, + } +} + +func MustProviderFlags(flags *pflag.FlagSet, logger *EtlLogger) string { + provider, err := flags.GetString("provider") + if err != nil { + logger.Fatal("could not get provider: ", err) + } + return provider +} + type historyArchiveBackend struct { client historyarchive.ArchiveInterface ledgers map[uint32]*historyarchive.Ledger @@ -1101,3 +1146,18 @@ type HistoryArchiveLedgerAndLCM struct { Ledger historyarchive.Ledger LCM xdr.LedgerCloseMeta } + +func MapToStruct(data map[string]interface{}, result interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + return json.Unmarshal(jsonData, result) +} + +func GetEnv(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} diff --git a/internal/utils/retool.go b/internal/utils/retool.go new file mode 100644 index 00000000..20d65213 --- /dev/null +++ b/internal/utils/retool.go @@ -0,0 +1,64 @@ +package utils + +// RetoolEntityDataTransformInput is a representation of the input for the TransformRetoolEntityData function +type RetoolEntityDataTransformInput struct { + ID int `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + CreatedAt int64 `json:"created_at"` + UpdatedAt *int64 `json:"updated_at"` + Custodial bool `json:"custodial"` + NonCustodial bool `json:"non_custodial"` + HomeDomainsID int `json:"home_domains_id"` + Description string `json:"description"` + WebsiteURL string `json:"website_url"` + SdpEnabled bool `json:"sdp_enabled"` + SorobanEnabled bool `json:"soroban_enabled"` + Notes string `json:"notes"` + Verified bool `json:"verified"` + FeeSponsor bool `json:"fee_sponsor"` + AccountSponsor bool `json:"account_sponsor"` + Live bool `json:"live"` + HomeDomain HomeDomain `json:"_home_domain"` + AppGeographiesDetails []AppGeographyDetail `json:"_app_geographies_details"` + AppToRampsIntegrations []AppToRampIntegration `json:"_app_to_ramps_integrations"` +} + +type HomeDomain struct { + ID int64 `json:"id"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` + HomeDomain string `json:"home_domain"` +} + +type AppGeographyDetail struct { + ID int64 `json:"id"` + AppsID int64 `json:"apps_id"` + CreatedAt int64 `json:"created_at"` + GeographiesID []Geography `json:"geographies_id"` + Retail bool `json:"retail"` + Enterprise bool `json:"enterprise"` +} + +type Geography struct { + ID int64 `json:"id"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` + Name string `json:"name"` + OfficialName string `json:"official_name"` +} + +type AppToRampIntegration struct { + ID int64 `json:"id"` + CreatedAt int64 `json:"created_at"` + AnchorsID int64 `json:"anchors_id"` + AppsID int64 `json:"apps_id"` + Anchor Anchor `json:"_anchor"` +} + +type Anchor struct { + ID int64 `json:"id"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` + Name string `json:"name"` +} diff --git a/testdata/external_data/external_data_retool.golden b/testdata/external_data/external_data_retool.golden new file mode 100644 index 00000000..fbb876a0 --- /dev/null +++ b/testdata/external_data/external_data_retool.golden @@ -0,0 +1 @@ +{"account_sponsor":false,"app_geographies":["Argentina","Brazil"],"created_at":1706749912776,"custodial":true,"description":"","fee_sponsor":false,"home_domain":"eldorado.io","home_domains_id":240,"id":16,"live":true,"name":"El Dorado","non_custodial":true,"notes":"","ramps":["MoneyGram"],"sdp_enabled":false,"soroban_enabled":false,"status":"live","updated_at":null,"verified":false,"website_url":""}