feat: upgrade odps version for maxcompute plugin (#76)

feat: upgrade odps version

Co-authored-by: Muhammad Luthfi Fahlevi <[email protected]>

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/ClickHouse/clickhouse-go v1.4.5
 	github.com/IBM/sarama v1.43.2
 	github.com/MakeNowJust/heredoc v1.0.0
-	github.com/aliyun/aliyun-odps-go-sdk v0.3.14
+	github.com/aliyun/aliyun-odps-go-sdk v0.4.0
 	github.com/aws/aws-sdk-go v1.44.151
 	github.com/blastrain/vitess-sqlparser v0.0.0-20201030050434-a139afbb1aba
 	github.com/cenkalti/backoff/v4 v4.2.1

go.sum

@@ -193,8 +193,8 @@ github.com/alibabacloud-go/debug v1.0.1 h1:MsW9SmUtbb1Fnt3ieC6NNZi6aEwrXfDksD4QA
 github.com/alibabacloud-go/debug v1.0.1/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc=
 github.com/alibabacloud-go/tea v1.2.2 h1:aTsR6Rl3ANWPfqeQugPglfurloyBJY85eFy7Gc1+8oU=
 github.com/alibabacloud-go/tea v1.2.2/go.mod h1:CF3vOzEMAG+bR4WOql8gc2G9H3EkH3ZLAQdpmpXMgwk=
-github.com/aliyun/aliyun-odps-go-sdk v0.3.14 h1:H+p9+2n4kSBOiJvndKC7M+0UxK0mW4azQHQzHy2hI2A=
-github.com/aliyun/aliyun-odps-go-sdk v0.3.14/go.mod h1:t/tgF/iN5aAs/gLL7sEI8/qdax4NuFCKEjO3OJbHZqI=
+github.com/aliyun/aliyun-odps-go-sdk v0.4.0 h1:QzEjRsMnB+FjTbO33PypiCbFJ4Wba55h3swSoZZVqgM=
+github.com/aliyun/aliyun-odps-go-sdk v0.4.0/go.mod h1:h3n3Jy9qCcq9GhKakuF7Y47W1EP71hfTDx8MCEeQYbA=
 github.com/aliyun/credentials-go v1.3.10 h1:45Xxrae/evfzQL9V10zL3xX31eqgLWEaIdCoPipOEQA=
 github.com/aliyun/credentials-go v1.3.10/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
@@ -1524,7 +1524,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=

plugins/extractors/maxcompute/maxcompute.go

@@ -231,7 +231,7 @@ func (e *Extractor) buildAsset(ctx context.Context, schema *odps.Schema,
 			Name:        col.Name,
 			DataType:    dataTypeToString(col.Type),
 			Description: col.Comment,
-			IsNullable:  col.IsNullable,
+			IsNullable:  !col.NotNull,
 			Attributes:  utils.TryParseMapToProto(buildColumnAttributesData(&tableSchema.Columns[i])),
 			Columns:     buildColumns(col.Type),
 		}
@@ -306,7 +306,7 @@ func (e *Extractor) buildTableAttributesData(schemaName, tableType string, table
 	attributesData["type"] = tableType
 
 	rb := common.ResourceBuilder{ProjectName: e.config.ProjectName}
-	attributesData["resource_url"] = rb.Table(tableInfo.TableName)
+	attributesData["resource_url"] = rb.Table(schemaName, tableInfo.TableName)
 
 	if tableInfo.ViewText != "" {
 		attributesData["sql"] = tableInfo.ViewText

plugins/extractors/maxcompute/maxcompute_test.go

@@ -84,9 +84,9 @@ func TestExtract(t *testing.T) {
 	}
 
 	c1 := tableschema.Column{
-		Name:       "id",
-		Type:       datatype.BigIntType,
-		IsNullable: true,
+		Name:    "id",
+		Type:    datatype.BigIntType,
+		NotNull: false,
 	}
 
 	c2 := tableschema.Column{
@@ -103,7 +103,7 @@ func TestExtract(t *testing.T) {
 				},
 			},
 		},
-		IsNullable: true,
+		NotNull: false,
 	}
 
 	// Schema for dummy_table
@@ -121,17 +121,17 @@ func TestExtract(t *testing.T) {
 	dummyTableSchema.Comment = "dummy table description"
 
 	c3 := tableschema.Column{
-		Name:       "user_id",
-		Type:       datatype.BigIntType,
-		Comment:    "Unique identifier for users",
-		IsNullable: false,
+		Name:    "user_id",
+		Type:    datatype.BigIntType,
+		Comment: "Unique identifier for users",
+		NotNull: true,
 	}
 
 	c4 := tableschema.Column{
-		Name:       "email",
-		Type:       datatype.StringType,
-		IsNullable: false,
-		Comment:    "User email address",
+		Name:    "email",
+		Type:    datatype.StringType,
+		NotNull: true,
+		Comment: "User email address",
 	}
 
 	// Schema for new_table

plugins/extractors/maxcompute/testdata/expected-assets-with-table-exclusion.json

@@ -35,7 +35,7 @@
             "preview_rows": null,
             "attributes": {
                 "project_name": "test-project-id",
-                "resource_url": "/projects/test-project-id/tables/new_table",
+                "resource_url": "/projects/test-project-id/schemas/my_schema/tables/new_table",
                 "schema": "my_schema",
                 "sql": "SELECT user_id, email FROM test-project-id.my_schema.new_table",
                 "type": "MANAGED_TABLE"

plugins/extractors/maxcompute/testdata/expected-assets-with-view-lineage.json

@@ -56,7 +56,7 @@
             "preview_rows": null,
             "attributes": {
                 "project_name": "test-project-id",
-                "resource_url": "/projects/test-project-id/tables/dummy_table",
+                "resource_url": "/projects/test-project-id/schemas/my_schema/tables/dummy_table",
                 "schema": "my_schema",
                 "sql": "SELECT id, name, user_info\nFROM test-project-id.default.my_dummy_table",
                 "type": "VIRTUAL_VIEW"
@@ -129,7 +129,7 @@
             ],
             "attributes": {
                 "project_name": "test-project-id",
-                "resource_url": "/projects/test-project-id/tables/new_table",
+                "resource_url": "/projects/test-project-id/schemas/my_schema/tables/new_table",
                 "schema": "my_schema",
                 "sql": "SELECT user_id, email FROM test-project-id.my_schema.new_table",
                 "type": "MANAGED_TABLE"

plugins/extractors/maxcompute/testdata/expected-assets.json

@@ -56,7 +56,7 @@
             "preview_rows": null,
             "attributes": {
                 "project_name": "test-project-id",
-                "resource_url": "/projects/test-project-id/tables/dummy_table",
+                "resource_url": "/projects/test-project-id/schemas/my_schema/tables/dummy_table",
                 "schema": "my_schema",
                 "sql": "SELECT id, name, user_info\nFROM test-project-id.default.my_dummy_table",
                 "type": "VIRTUAL_VIEW"
@@ -119,7 +119,7 @@
             ],
             "attributes": {
                 "project_name": "test-project-id",
-                "resource_url": "/projects/test-project-id/tables/new_table",
+                "resource_url": "/projects/test-project-id/schemas/my_schema/tables/new_table",
                 "schema": "my_schema",
                 "sql": "SELECT user_id, email FROM test-project-id.my_schema.new_table",
                 "type": "MANAGED_TABLE"