diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index 1c3a385d3f..66cb684937 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -40,7 +40,12 @@ jobs: - name: install gotestsum run: | go install gotest.tools/gotestsum@latest - + + - name: install lib-geos + run: | + sudo apt-get update + sudo apt-get install libgeos-dev + - name: download go modules run: | go mod download diff --git a/flow/connectors/postgres/cdc.go b/flow/connectors/postgres/cdc.go index b48143150b..6130094cd7 100644 --- a/flow/connectors/postgres/cdc.go +++ b/flow/connectors/postgres/cdc.go @@ -527,9 +527,18 @@ func (p *PostgresCDCSource) decodeColumnData(data []byte, dataType uint32, forma } return retVal, nil } + typeName, ok := p.customTypeMapping[dataType] if ok { - return &qvalue.QValue{Kind: customTypeToQKind(typeName), + customQKind := customTypeToQKind(typeName) + if customQKind == qvalue.QValueKindGeography || customQKind == qvalue.QValueKindGeometry { + err := GeoValidate(string(data)) + if err != nil { + return &qvalue.QValue{Kind: customQKind, + Value: nil}, nil + } + } + return &qvalue.QValue{Kind: customQKind, Value: string(data)}, nil } diff --git a/flow/connectors/postgres/qrep_query_executor.go b/flow/connectors/postgres/qrep_query_executor.go index 94943132bb..42ed6a1ada 100644 --- a/flow/connectors/postgres/qrep_query_executor.go +++ b/flow/connectors/postgres/qrep_query_executor.go @@ -433,8 +433,16 @@ func mapRowToQRecord(row pgx.Rows, fds []pgconn.FieldDescription, } record.Set(i, *tmp) } else { + customQKind := customTypeToQKind(typeName) + if customQKind == qvalue.QValueKindGeography || customQKind == qvalue.QValueKindGeometry { + wkbString, ok := values[i].(string) + err := GeoValidate(wkbString) + if err != nil || !ok { + values[i] = nil + } + } customTypeVal := qvalue.QValue{ - Kind: customTypeToQKind(typeName), + Kind: customQKind, Value: values[i], } record.Set(i, customTypeVal) diff --git a/flow/connectors/postgres/qvalue_convert.go b/flow/connectors/postgres/qvalue_convert.go index 21f802f6c0..cc35db7cad 100644 --- a/flow/connectors/postgres/qvalue_convert.go +++ b/flow/connectors/postgres/qvalue_convert.go @@ -1,6 +1,7 @@ package connpostgres import ( + "encoding/hex" "encoding/json" "errors" "fmt" @@ -13,6 +14,9 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/lib/pq/oid" log "github.com/sirupsen/logrus" + + //nolint:all + geom "github.com/twpayne/go-geos" ) func postgresOIDToQValueKind(recvOID uint32) qvalue.QValueKind { @@ -402,3 +406,25 @@ func customTypeToQKind(typeName string) qvalue.QValueKind { } return qValueKind } + +func GeoValidate(hexWkb string) error { + log.Infof("Validating geometry shape %s", hexWkb) + // Decode the WKB hex string into binary + wkb, hexErr := hex.DecodeString(hexWkb) + if hexErr != nil { + log.Warnf("Ignoring invalid WKB: %s", hexWkb) + return hexErr + } + // UnmarshalWKB performs geometry validation along with WKB parsing + geometryObject, geoErr := geom.NewGeomFromWKB(wkb) + if geoErr != nil { + log.Warnf("Ignoring invalid geometry WKB %s: %v", hexWkb, geoErr) + return geoErr + } + invalidReason := geometryObject.IsValidReason() + if invalidReason != "Valid Geometry" { + log.Warnf("Ignoring invalid geometry shape %s: %s", hexWkb, invalidReason) + return errors.New(invalidReason) + } + return nil +} diff --git a/flow/connectors/snowflake/snowflake.go b/flow/connectors/snowflake/snowflake.go index 4e6a913d38..6a05bf9832 100644 --- a/flow/connectors/snowflake/snowflake.go +++ b/flow/connectors/snowflake/snowflake.go @@ -961,9 +961,14 @@ func (c *SnowflakeConnector) generateAndExecuteMergeStatement( case qvalue.QValueKindBytes, qvalue.QValueKindBit: flattenedCastsSQLArray = append(flattenedCastsSQLArray, fmt.Sprintf("BASE64_DECODE_BINARY(%s:\"%s\") "+ "AS %s,", toVariantColumnName, columnName, targetColumnName)) - case qvalue.QValueKindGeography, qvalue.QValueKindGeometry, qvalue.QValueKindPoint: - flattenedCastsSQLArray = append(flattenedCastsSQLArray, fmt.Sprintf("CAST(%s:\"%s\" AS STRING) AS %s,", - toVariantColumnName, columnName, targetColumnName)) + case qvalue.QValueKindGeography: + flattenedCastsSQLArray = append(flattenedCastsSQLArray, + fmt.Sprintf("TO_GEOGRAPHY(CAST(%s:\"%s\" AS STRING),true) AS %s,", + toVariantColumnName, columnName, targetColumnName)) + case qvalue.QValueKindGeometry: + flattenedCastsSQLArray = append(flattenedCastsSQLArray, + fmt.Sprintf("TO_GEOMETRY(CAST(%s:\"%s\" AS STRING),true) AS %s,", + toVariantColumnName, columnName, targetColumnName)) // TODO: https://github.com/PeerDB-io/peerdb/issues/189 - handle time types and interval types // case model.ColumnTypeTime: // flattenedCastsSQLArray = append(flattenedCastsSQLArray, fmt.Sprintf("TIME_FROM_PARTS(0,0,0,%s:%s:"+ diff --git a/flow/connectors/sql/query_executor.go b/flow/connectors/sql/query_executor.go index 5d4c154cbb..29291b40ff 100644 --- a/flow/connectors/sql/query_executor.go +++ b/flow/connectors/sql/query_executor.go @@ -124,6 +124,16 @@ func (g *GenericSQLQueryExecutor) CountRows(schemaName string, tableName string) return count, err } +func (g *GenericSQLQueryExecutor) CountNonNullRows( + schemaName string, + tableName string, + columnName string) (int64, error) { + var count int64 + err := g.db.QueryRowx("SELECT COUNT(CASE WHEN " + columnName + + " IS NOT NULL THEN 1 END) AS non_null_count FROM " + schemaName + "." + tableName).Scan(&count) + return count, err +} + func (g *GenericSQLQueryExecutor) columnTypeToQField(ct *sql.ColumnType) (*model.QField, error) { qvKind, ok := g.dbtypeToQValueKind[ct.DatabaseTypeName()] if !ok { diff --git a/flow/e2e/snowflake/peer_flow_sf_test.go b/flow/e2e/snowflake/peer_flow_sf_test.go index f335b3c78f..9f92496eb3 100644 --- a/flow/e2e/snowflake/peer_flow_sf_test.go +++ b/flow/e2e/snowflake/peer_flow_sf_test.go @@ -231,6 +231,93 @@ func (s *PeerFlowE2ETestSuiteSF) Test_Complete_Simple_Flow_SF_Avro_CDC() { env.AssertExpectations(s.T()) } +func (s *PeerFlowE2ETestSuiteSF) Test_Invalid_Geo_SF_Avro_CDC() { + env := s.NewTestWorkflowEnvironment() + e2e.RegisterWorkflowsAndActivities(env) + + srcTableName := s.attachSchemaSuffix("test_invalid_geo_sf_avro_cdc") + dstTableName := fmt.Sprintf("%s.%s", s.sfHelper.testSchemaName, "test_invalid_geo_sf_avro_cdc") + + _, err := s.pool.Exec(context.Background(), fmt.Sprintf(` + CREATE TABLE %s ( + id SERIAL PRIMARY KEY, + line GEOMETRY(LINESTRING) NOT NULL, + poly GEOGRAPHY(POLYGON) NOT NULL + ); + `, srcTableName)) + s.NoError(err) + + connectionGen := e2e.FlowConnectionGenerationConfig{ + FlowJobName: s.attachSuffix("test_invalid_geo_sf_avro_cdc"), + TableNameMapping: map[string]string{srcTableName: dstTableName}, + PostgresPort: e2e.PostgresPort, + Destination: s.sfHelper.Peer, + CDCSyncMode: protos.QRepSyncMode_QREP_SYNC_MODE_STORAGE_AVRO, + } + + flowConnConfig, err := connectionGen.GenerateFlowConnectionConfigs() + s.NoError(err) + + limits := peerflow.CDCFlowLimits{ + TotalSyncFlows: 2, + MaxBatchSize: 100, + } + + // in a separate goroutine, wait for PeerFlowStatusQuery to finish setup + // and then insert 10 rows into the source table + go func() { + e2e.SetupCDCFlowStatusQuery(env, connectionGen) + // insert 10 rows into the source table + for i := 0; i < 4; i++ { + _, err = s.pool.Exec(context.Background(), fmt.Sprintf(` + INSERT INTO %s (line,poly) VALUES ($1,$2) + `, srcTableName), "010200000001000000000000000000F03F0000000000000040", + "0103000020e6100000010000000c0000001a8361d35dc64140afdb8d2b1bc3c9bf1b8ed4685fc641405ba64c"+ + "579dc2c9bf6a6ad95a5fc64140cd82767449c2c9bf9570fbf85ec641408a07944db9c2c9bf729a18a55ec6414021b8b748c7c2c9bfba46de4c"+ + "5fc64140f2567052abc2c9bf2df9c5925fc641409394e16573c2c9bf2df9c5925fc6414049eceda9afc1c9bfdd1cc1a05fc64140fe43faedebc0"+ + "c9bf4694f6065fc64140fe43faedebc0c9bfffe7305f5ec641406693d6f2ddc0c9bf1a8361d35dc64140afdb8d2b1bc3c9bf", + ) + s.NoError(err) + } + fmt.Println("Inserted 4 invalid geography rows into the source table") + for i := 4; i < 10; i++ { + _, err = s.pool.Exec(context.Background(), fmt.Sprintf(` + INSERT INTO %s (line,poly) VALUES ($1,$2) + `, srcTableName), "010200000002000000000000000000F03F000000000000004000000000000008400000000000001040", + "010300000001000000050000000000000000000000000000000000000000000000"+ + "00000000000000000000f03f000000000000f03f000000000000f03f0000000000"+ + "00f03f000000000000000000000000000000000000000000000000") + s.NoError(err) + } + fmt.Println("Inserted 6 valid geography rows and 10 total rows into source") + }() + + env.ExecuteWorkflow(peerflow.CDCFlowWorkflowWithConfig, flowConnConfig, &limits, nil) + + // Verify workflow completes without error + s.True(env.IsWorkflowCompleted()) + err = env.GetWorkflowError() + + // allow only continue as new error + s.Error(err) + s.Contains(err.Error(), "continue as new") + + // We inserted 4 invalid shapes in each. + // They should have filtered out as null on destination + lineCount, err := s.sfHelper.CountNonNullRows("test_invalid_geo_sf_avro_cdc", "line") + s.NoError(err) + s.Equal(6, lineCount) + + polyCount, err := s.sfHelper.CountNonNullRows("test_invalid_geo_sf_avro_cdc", "poly") + s.NoError(err) + s.Equal(6, polyCount) + + // TODO: verify that the data is correctly synced to the destination table + // on the bigquery side + + env.AssertExpectations(s.T()) +} + func (s *PeerFlowE2ETestSuiteSF) Test_Toast_SF() { env := s.NewTestWorkflowEnvironment() e2e.RegisterWorkflowsAndActivities(env) diff --git a/flow/e2e/snowflake/snowflake_helper.go b/flow/e2e/snowflake/snowflake_helper.go index 2a39daf93e..8a96b1ad91 100644 --- a/flow/e2e/snowflake/snowflake_helper.go +++ b/flow/e2e/snowflake/snowflake_helper.go @@ -118,6 +118,16 @@ func (s *SnowflakeTestHelper) CountRows(tableName string) (int, error) { return int(res), nil } +// CountRows(tableName) returns the non-null number of rows in the given table. +func (s *SnowflakeTestHelper) CountNonNullRows(tableName string, columnName string) (int, error) { + res, err := s.testClient.CountNonNullRows(s.testSchemaName, tableName, columnName) + if err != nil { + return 0, err + } + + return int(res), nil +} + func (s *SnowflakeTestHelper) CheckNull(tableName string, colNames []string) (bool, error) { return s.testClient.CheckNull(s.testSchemaName, tableName, colNames) } diff --git a/flow/e2e/test_utils.go b/flow/e2e/test_utils.go index 29b7a66ed0..4ea032bf3d 100644 --- a/flow/e2e/test_utils.go +++ b/flow/e2e/test_utils.go @@ -196,12 +196,10 @@ func PopulateSourceTable(pool *pgxpool.Pool, suffix string, tableName string, ro ids = append(ids, id) geoValues := "" if strings.Contains(tableName, "sf") { - // geo types geoValues = `,'POINT(1 2)','POINT(40.7128 -74.0060)', 'LINESTRING(0 0, 1 1, 2 2)', 'LINESTRING(-74.0060 40.7128, -73.9352 40.7306, -73.9123 40.7831)', - 'POLYGON((0 0, 0 1, 1 1, 1 0, 0 0))', - 'POLYGON((-74.0060 40.7128, -73.9352 40.7306, -73.9123 40.7831, -74.0060 40.7128))'` + 'POLYGON((0 0, 0 1, 1 1, 1 0, 0 0))','POLYGON((0 0, 0 1, 1 1, 1 0, 0 0))'` } row := fmt.Sprintf(` ( diff --git a/flow/go.mod b/flow/go.mod index 88150adf52..c1dd4013e2 100644 --- a/flow/go.mod +++ b/flow/go.mod @@ -43,6 +43,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/twmb/murmur3 v1.1.8 // indirect + github.com/twpayne/go-geos v0.13.2 // indirect ) require ( diff --git a/flow/go.sum b/flow/go.sum index b6a1ba5b98..1f643a26f4 100644 --- a/flow/go.sum +++ b/flow/go.sum @@ -1009,6 +1009,7 @@ github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpx github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= @@ -1029,6 +1030,7 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/gogo/status v1.1.1 h1:DuHXlSFHNKqTQ+/ACf5Vs6r4X/dH2EgIzR9Vr+H65kg= github.com/gogo/status v1.1.1/go.mod h1:jpG3dM5QPcqu19Hg8lkUhBFBa3TcLs1DG7+2Jqci7oU= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= @@ -1216,6 +1218,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -1227,6 +1230,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= @@ -1277,6 +1281,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -1285,8 +1291,13 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= +github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s= +github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/peterstace/simplefeatures v0.45.1 h1:V615OgtN8ZXYuM9h1UTtuQFn9AgAC1XQT2ipQyYZWVE= +github.com/peterstace/simplefeatures v0.45.1/go.mod h1:nosSwG+GcVmAUBoxFWoyy1hS1qg0RuX0M9tmqsIzFX8= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= @@ -1380,9 +1391,14 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/twpayne/go-geom v1.5.2 h1:LyRfBX2W0LM7XN/bGqX0XxrJ7SZc3XwmxU4aj4kSoxw= +github.com/twpayne/go-geom v1.5.2/go.mod h1:3z6O2sAnGtGCXx4Q+5nPOLCA5e8WI2t3cthdb1P2HH8= +github.com/twpayne/go-geos v0.13.2 h1:kyQenUwpU+mGrNX403KNlRDuIIIL22OHVVj1SGfXUm0= +github.com/twpayne/go-geos v0.13.2/go.mod h1:r5O89NwzDqYqiDF5HnkYjdgJtODwzpjeNlj/gL9ztXk= github.com/uber-go/tally/v4 v4.1.1/go.mod h1:aXeSTDMl4tNosyf6rdU8jlgScHyjEGGtfJ/uwCIf/vM= github.com/uber-go/tally/v4 v4.1.7 h1:YiKvvMKCCXlCKXI0i1hVk+xda8YxdIpjeFXohpvn8Zo= github.com/uber-go/tally/v4 v4.1.7/go.mod h1:pPR56rjthjtLB8xQlEx2I1VwAwRGCh/i4xMUcmG+6z4= @@ -1390,8 +1406,12 @@ github.com/uber-go/tally/v4 v4.1.10 h1:2GSX7Tmq26wjAvOtQEc5EvRROIkX2OX4vpROt6mlR github.com/uber-go/tally/v4 v4.1.10/go.mod h1:pPR56rjthjtLB8xQlEx2I1VwAwRGCh/i4xMUcmG+6z4= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1405,6 +1425,7 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -1454,6 +1475,7 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= diff --git a/stacks/flow-api.Dockerfile b/stacks/flow-api.Dockerfile index 31c877c768..22af707554 100644 --- a/stacks/flow-api.Dockerfile +++ b/stacks/flow-api.Dockerfile @@ -1,7 +1,11 @@ # syntax=docker/dockerfile:1.2 # Start from the latest Golang base image -FROM golang:1.21-alpine AS builder +FROM golang:1.21.3-bookworm AS builder + +RUN apt-get update && apt-get install -y gcc + +RUN apt-get update && apt-get install -y libgeos-dev WORKDIR /root/flow # first copy only go.mod and go.sum to cache dependencies @@ -18,10 +22,15 @@ COPY flow . # build the binary from cmd folder WORKDIR /root/flow/cmd RUN --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 go build -ldflags="-s -w" -o /root/peer-flow . + CGO_ENABLED=1 go build -ldflags="-s -w" -o /root/peer-flow . + +FROM golang:1.21.3-bookworm +RUN apt-get update && apt-get install -y ca-certificates + +RUN apt-get update && apt-get install -y gcc -FROM ubuntu:20.04 -RUN apt-get update && apt-get install -y ca-certificates curl +# install lib geos dev +RUN apt-get update && apt-get install -y libgeos-dev WORKDIR /root COPY --from=builder /root/peer-flow . EXPOSE 8112 diff --git a/stacks/flow-snapshot-worker.Dockerfile b/stacks/flow-snapshot-worker.Dockerfile index cf3a788818..0383a03745 100644 --- a/stacks/flow-snapshot-worker.Dockerfile +++ b/stacks/flow-snapshot-worker.Dockerfile @@ -1,7 +1,12 @@ # syntax=docker/dockerfile:1.2 # Start from the latest Golang base image -FROM golang:1.21-alpine AS builder +FROM golang:1.21.3-bookworm AS builder + +RUN apt-get update && apt-get install -y gcc + +RUN apt-get update && apt-get install -y libgeos-dev + WORKDIR /root/ @@ -18,10 +23,14 @@ COPY flow . # build the binary from cmd folder WORKDIR /root/cmd RUN --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 go build -ldflags="-s -w" -o /root/peer-flow . + CGO_ENABLED=1 go build -ldflags="-s -w" -o /root/peer-flow . -FROM ubuntu:20.04 +FROM golang:1.21.3-bookworm RUN apt-get update && apt-get install -y ca-certificates + +RUN apt-get update && apt-get install -y gcc +# install lib geos dev +RUN apt-get update && apt-get install -y libgeos-dev WORKDIR /root COPY --from=builder /root/peer-flow . EXPOSE 8112 diff --git a/stacks/flow-worker.Dockerfile b/stacks/flow-worker.Dockerfile index cd6725b9fb..526fb59d45 100644 --- a/stacks/flow-worker.Dockerfile +++ b/stacks/flow-worker.Dockerfile @@ -1,7 +1,11 @@ # syntax=docker/dockerfile:1.2 # Start from the latest Golang base image -FROM golang:1.21-alpine AS builder +FROM golang:1.21.3-bookworm AS builder + +RUN apt-get update && apt-get install -y gcc + +RUN apt-get update && apt-get install -y libgeos-dev WORKDIR /root/ @@ -18,10 +22,15 @@ COPY flow . # build the binary from cmd folder WORKDIR /root/cmd RUN --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 go build -ldflags="-s -w" -o /root/peer-flow . + CGO_ENABLED=1 go build -ldflags="-s -w"\ + -o /root/peer-flow . -FROM ubuntu:20.04 +FROM golang:1.21.3-bookworm RUN apt-get update && apt-get install -y ca-certificates + +RUN apt-get update && apt-get install -y gcc +# install lib geos dev +RUN apt-get update && apt-get install -y libgeos-dev WORKDIR /root COPY --from=builder /root/peer-flow . EXPOSE 8112