diff --git a/go.mod b/go.mod index f4ff9cf42..83c2f5e00 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/ClickHouse/clickhouse-go v1.4.5 github.com/IBM/sarama v1.43.2 github.com/MakeNowJust/heredoc v1.0.0 - github.com/aliyun/aliyun-odps-go-sdk v0.3.14 + github.com/aliyun/aliyun-odps-go-sdk v0.4.0 github.com/aws/aws-sdk-go v1.44.151 github.com/blastrain/vitess-sqlparser v0.0.0-20201030050434-a139afbb1aba github.com/cenkalti/backoff/v4 v4.2.1 diff --git a/go.sum b/go.sum index affa47aa9..e9e02f50b 100644 --- a/go.sum +++ b/go.sum @@ -193,8 +193,8 @@ github.com/alibabacloud-go/debug v1.0.1 h1:MsW9SmUtbb1Fnt3ieC6NNZi6aEwrXfDksD4QA github.com/alibabacloud-go/debug v1.0.1/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc= github.com/alibabacloud-go/tea v1.2.2 h1:aTsR6Rl3ANWPfqeQugPglfurloyBJY85eFy7Gc1+8oU= github.com/alibabacloud-go/tea v1.2.2/go.mod h1:CF3vOzEMAG+bR4WOql8gc2G9H3EkH3ZLAQdpmpXMgwk= -github.com/aliyun/aliyun-odps-go-sdk v0.3.14 h1:H+p9+2n4kSBOiJvndKC7M+0UxK0mW4azQHQzHy2hI2A= -github.com/aliyun/aliyun-odps-go-sdk v0.3.14/go.mod h1:t/tgF/iN5aAs/gLL7sEI8/qdax4NuFCKEjO3OJbHZqI= +github.com/aliyun/aliyun-odps-go-sdk v0.4.0 h1:QzEjRsMnB+FjTbO33PypiCbFJ4Wba55h3swSoZZVqgM= +github.com/aliyun/aliyun-odps-go-sdk v0.4.0/go.mod h1:h3n3Jy9qCcq9GhKakuF7Y47W1EP71hfTDx8MCEeQYbA= github.com/aliyun/credentials-go v1.3.10 h1:45Xxrae/evfzQL9V10zL3xX31eqgLWEaIdCoPipOEQA= github.com/aliyun/credentials-go v1.3.10/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= @@ -1524,7 +1524,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= diff --git a/plugins/extractors/maxcompute/maxcompute.go b/plugins/extractors/maxcompute/maxcompute.go index f7acc4151..22fcc54f7 100644 --- a/plugins/extractors/maxcompute/maxcompute.go +++ b/plugins/extractors/maxcompute/maxcompute.go @@ -231,7 +231,7 @@ func (e *Extractor) buildAsset(ctx context.Context, schema *odps.Schema, Name: col.Name, DataType: dataTypeToString(col.Type), Description: col.Comment, - IsNullable: col.IsNullable, + IsNullable: !col.NotNull, Attributes: utils.TryParseMapToProto(buildColumnAttributesData(&tableSchema.Columns[i])), Columns: buildColumns(col.Type), } @@ -306,7 +306,7 @@ func (e *Extractor) buildTableAttributesData(schemaName, tableType string, table attributesData["type"] = tableType rb := common.ResourceBuilder{ProjectName: e.config.ProjectName} - attributesData["resource_url"] = rb.Table(tableInfo.TableName) + attributesData["resource_url"] = rb.Table(schemaName, tableInfo.TableName) if tableInfo.ViewText != "" { attributesData["sql"] = tableInfo.ViewText diff --git a/plugins/extractors/maxcompute/maxcompute_test.go b/plugins/extractors/maxcompute/maxcompute_test.go index de7104f1f..7c1d03b10 100644 --- a/plugins/extractors/maxcompute/maxcompute_test.go +++ b/plugins/extractors/maxcompute/maxcompute_test.go @@ -84,9 +84,9 @@ func TestExtract(t *testing.T) { } c1 := tableschema.Column{ - Name: "id", - Type: datatype.BigIntType, - IsNullable: true, + Name: "id", + Type: datatype.BigIntType, + NotNull: false, } c2 := tableschema.Column{ @@ -103,7 +103,7 @@ func TestExtract(t *testing.T) { }, }, }, - IsNullable: true, + NotNull: false, } // Schema for dummy_table @@ -121,17 +121,17 @@ func TestExtract(t *testing.T) { dummyTableSchema.Comment = "dummy table description" c3 := tableschema.Column{ - Name: "user_id", - Type: datatype.BigIntType, - Comment: "Unique identifier for users", - IsNullable: false, + Name: "user_id", + Type: datatype.BigIntType, + Comment: "Unique identifier for users", + NotNull: true, } c4 := tableschema.Column{ - Name: "email", - Type: datatype.StringType, - IsNullable: false, - Comment: "User email address", + Name: "email", + Type: datatype.StringType, + NotNull: true, + Comment: "User email address", } // Schema for new_table diff --git a/plugins/extractors/maxcompute/testdata/expected-assets-with-table-exclusion.json b/plugins/extractors/maxcompute/testdata/expected-assets-with-table-exclusion.json index 3d6e26c87..327838d67 100644 --- a/plugins/extractors/maxcompute/testdata/expected-assets-with-table-exclusion.json +++ b/plugins/extractors/maxcompute/testdata/expected-assets-with-table-exclusion.json @@ -35,7 +35,7 @@ "preview_rows": null, "attributes": { "project_name": "test-project-id", - "resource_url": "/projects/test-project-id/tables/new_table", + "resource_url": "/projects/test-project-id/schemas/my_schema/tables/new_table", "schema": "my_schema", "sql": "SELECT user_id, email FROM test-project-id.my_schema.new_table", "type": "MANAGED_TABLE" diff --git a/plugins/extractors/maxcompute/testdata/expected-assets-with-view-lineage.json b/plugins/extractors/maxcompute/testdata/expected-assets-with-view-lineage.json index 25193856b..ef896f187 100644 --- a/plugins/extractors/maxcompute/testdata/expected-assets-with-view-lineage.json +++ b/plugins/extractors/maxcompute/testdata/expected-assets-with-view-lineage.json @@ -56,7 +56,7 @@ "preview_rows": null, "attributes": { "project_name": "test-project-id", - "resource_url": "/projects/test-project-id/tables/dummy_table", + "resource_url": "/projects/test-project-id/schemas/my_schema/tables/dummy_table", "schema": "my_schema", "sql": "SELECT id, name, user_info\nFROM test-project-id.default.my_dummy_table", "type": "VIRTUAL_VIEW" @@ -129,7 +129,7 @@ ], "attributes": { "project_name": "test-project-id", - "resource_url": "/projects/test-project-id/tables/new_table", + "resource_url": "/projects/test-project-id/schemas/my_schema/tables/new_table", "schema": "my_schema", "sql": "SELECT user_id, email FROM test-project-id.my_schema.new_table", "type": "MANAGED_TABLE" diff --git a/plugins/extractors/maxcompute/testdata/expected-assets.json b/plugins/extractors/maxcompute/testdata/expected-assets.json index d1a75dc21..4666b5413 100644 --- a/plugins/extractors/maxcompute/testdata/expected-assets.json +++ b/plugins/extractors/maxcompute/testdata/expected-assets.json @@ -56,7 +56,7 @@ "preview_rows": null, "attributes": { "project_name": "test-project-id", - "resource_url": "/projects/test-project-id/tables/dummy_table", + "resource_url": "/projects/test-project-id/schemas/my_schema/tables/dummy_table", "schema": "my_schema", "sql": "SELECT id, name, user_info\nFROM test-project-id.default.my_dummy_table", "type": "VIRTUAL_VIEW" @@ -119,7 +119,7 @@ ], "attributes": { "project_name": "test-project-id", - "resource_url": "/projects/test-project-id/tables/new_table", + "resource_url": "/projects/test-project-id/schemas/my_schema/tables/new_table", "schema": "my_schema", "sql": "SELECT user_id, email FROM test-project-id.my_schema.new_table", "type": "MANAGED_TABLE"