From 408de84024d2f2cbefbd24787366ed34ce913dad Mon Sep 17 00:00:00 2001 From: Nikita Jain Date: Tue, 3 Dec 2024 11:03:03 +0530 Subject: [PATCH 01/15] upgraded to latest go version (#3) --- go.mod | 2 +- go.sum | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f96a094..fcd2080 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/cloudspannerecosystem/dynamodb-adapter -go 1.17 +go 1.22 require ( cloud.google.com/go/pubsub v1.17.0 diff --git a/go.sum b/go.sum index d47c92f..f40d8d6 100644 --- a/go.sum +++ b/go.sum @@ -358,7 +358,6 @@ github.com/swaggo/swag v1.7.1 h1:gY9ZakXlNWg/i/v5bQBic7VMZ4teq4m89lpiao74p/s= github.com/swaggo/swag v1.7.1/go.mod h1:gAiHxNTb9cIpNmA/VEGUP+CyZMCP/EW7mdtc8Bny+p8= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.1.13/go.mod h1:jxau1n+/wyTGLQoCkjok9r5zFa/FxT6eI5HiHKQszjc= -github.com/ugorji/go v1.2.6 h1:tGiWC9HENWE2tqYycIqFTNorMmFRVhNwCpDOpWqnk8E= github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= From bb2ff12f88148cd7722aaac328a4dd371f81e14d Mon Sep 17 00:00:00 2001 From: taherkl Date: Wed, 4 Dec 2024 15:59:14 +0530 Subject: [PATCH 02/15] Support StringSET --- api/v1/condition.go | 22 +++- api/v1/condition_test.go | 2 + api/v1/db.go | 2 +- storage/spanner.go | 255 +++++++++++++++++++++------------------ storage/spanner_test.go | 32 +++-- 5 files changed, 181 insertions(+), 132 deletions(-) diff --git a/api/v1/condition.go b/api/v1/condition.go index 7bcd7d1..4d40416 100644 --- a/api/v1/condition.go +++ b/api/v1/condition.go @@ -454,7 +454,7 @@ func ChangeResponseColumn(obj map[string]interface{}) map[string]interface{} { // ChangeColumnToSpanner converts original column name to spanner supported column names func ChangeColumnToSpanner(obj map[string]interface{}) map[string]interface{} { rs := make(map[string]interface{}) - + for k, v := range obj { if k1, ok := models.ColumnToOriginalCol[k]; ok { @@ -519,9 +519,13 @@ func convertFrom(a *dynamodb.AttributeValue, tableName string) interface{} { return a.B } if a.SS != nil { - l := make([]interface{}, len(a.SS)) - for index, v := range a.SS { - l[index] = *v + l := []string{} + stringMap := make(map[string]struct{}) + for _, v := range a.SS { + if _, exists := stringMap[*v]; !exists { + stringMap[*v] = struct{}{} + l = append(l, *v) + } } return l } @@ -631,7 +635,7 @@ func ChangeQueryResponseColumn(tableName string, obj map[string]interface{}) map return obj } -//ChangeMaptoDynamoMap converts simple map into dynamo map +// ChangeMaptoDynamoMap converts simple map into dynamo map func ChangeMaptoDynamoMap(in interface{}) (map[string]interface{}, error) { if in == nil { return nil, nil @@ -699,6 +703,14 @@ func convertSlice(output map[string]interface{}, v reflect.Value) error { return nil } output["B"] = append([]byte{}, b...) + case reflect.String: + listVal := []string{} + count := 0 + for i := 0; i < v.Len(); i++ { + listVal = append(listVal, v.Index(i).String()) + count++ + } + output["SS"] = listVal default: listVal := make([]map[string]interface{}, 0, v.Len()) diff --git a/api/v1/condition_test.go b/api/v1/condition_test.go index 6d12f64..5d53a31 100644 --- a/api/v1/condition_test.go +++ b/api/v1/condition_test.go @@ -327,11 +327,13 @@ func TestConvertDynamoToMap(t *testing.T) { "address": {S: aws.String("Ney York")}, "first_name": {S: aws.String("Catalina")}, "last_name": {S: aws.String("Smith")}, + "titles": {SS: aws.StringSlice([]string{"Mr", "Dr"})}, }, map[string]interface{}{ "address": "Ney York", "first_name": "Catalina", "last_name": "Smith", + "titles": []string{"Mr", "Dr"}, }, }, { diff --git a/api/v1/db.go b/api/v1/db.go index e499c65..dce1f0a 100644 --- a/api/v1/db.go +++ b/api/v1/db.go @@ -59,7 +59,7 @@ func RouteRequest(c *gin.Context) { case "UpdateItem": Update(c) default: - c.JSON(errors.New("ValidationException", "Invalid X-Amz-Target header value of" + amzTarget). + c.JSON(errors.New("ValidationException", "Invalid X-Amz-Target header value of"+amzTarget). HTTPResponse("X-Amz-Target Header not supported")) } } diff --git a/storage/spanner.go b/storage/spanner.go index 544f857..048e781 100755 --- a/storage/spanner.go +++ b/storage/spanner.go @@ -18,6 +18,7 @@ import ( "context" "encoding/base64" "encoding/json" + "fmt" "math" "reflect" "regexp" @@ -612,7 +613,6 @@ func (s Storage) performPutOperation(ctx context.Context, t *spanner.ReadWriteTr m[k] = ba } } - mutation := spanner.InsertOrUpdateMap(table, m) mutations := []*spanner.Mutation{mutation} err := t.BufferWrite(mutations) @@ -733,7 +733,7 @@ func evaluateStatementFromRowMap(conditionalExpression, colName string, rowMap m return true } _, ok := rowMap[colName] - return !ok + return !ok } if strings.HasPrefix(conditionalExpression, "attribute_exists") || strings.HasPrefix(conditionalExpression, "if_exists") { if len(rowMap) == 0 { @@ -745,7 +745,6 @@ func evaluateStatementFromRowMap(conditionalExpression, colName string, rowMap m return rowMap[conditionalExpression] } -//parseRow - Converts Spanner row and datatypes to a map removing null columns from the result. func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{}, error) { singleRow := make(map[string]interface{}) if r == nil { @@ -759,133 +758,157 @@ func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{}, } v, ok := colDDL[k] if !ok { + fmt.Println(k) return nil, errors.New("ResourceNotFoundException", k) } + + var err error switch v { case "STRING(MAX)": - var s spanner.NullString - err := r.Column(i, &s) - if err != nil { - if strings.Contains(err.Error(), "ambiguous column name") { - continue - } - return nil, errors.New("ValidationException", err, k) - } - if !s.IsNull() { - singleRow[k] = s.StringVal - } + err = parseStringColumn(r, i, k, singleRow) case "BYTES(MAX)": - var s []byte - err := r.Column(i, &s) - if err != nil { - if strings.Contains(err.Error(), "ambiguous column name") { - continue - } - return nil, errors.New("ValidationException", err, k) - } - if len(s) > 0 { - var m interface{} - err := json.Unmarshal(s, &m) - if err != nil { - logger.LogError(err, string(s)) - singleRow[k] = string(s) - continue - } - val1, ok := m.(string) - if ok { - if base64Regexp.MatchString(val1) { - ba, err := base64.StdEncoding.DecodeString(val1) - if err == nil { - var sample interface{} - err = json.Unmarshal(ba, &sample) - if err == nil { - singleRow[k] = sample - continue - } else { - singleRow[k] = string(s) - continue - } - } - } - } - - if mp, ok := m.(map[string]interface{}); ok { - for k, v := range mp { - if val, ok := v.(string); ok { - if base64Regexp.MatchString(val) { - ba, err := base64.StdEncoding.DecodeString(val) - if err == nil { - var sample interface{} - err = json.Unmarshal(ba, &sample) - if err == nil { - mp[k] = sample - m = mp - } - } - } - } - } - } - singleRow[k] = m - } + err = parseBytesColumn(r, i, k, singleRow) case "INT64": - var s spanner.NullInt64 - err := r.Column(i, &s) - if err != nil { - if strings.Contains(err.Error(), "ambiguous column name") { - continue - } - return nil, errors.New("ValidationException", err, k) - } - if !s.IsNull() { - singleRow[k] = s.Int64 - } + err = parseInt64Column(r, i, k, singleRow) case "FLOAT64": - var s spanner.NullFloat64 - err := r.Column(i, &s) - if err != nil { - if strings.Contains(err.Error(), "ambiguous column name") { - continue - } - return nil, errors.New("ValidationException", err, k) - - } - if !s.IsNull() { - singleRow[k] = s.Float64 - } + err = parseFloat64Column(r, i, k, singleRow) case "NUMERIC": - var s spanner.NullNumeric - err := r.Column(i, &s) - if err != nil { - if strings.Contains(err.Error(), "ambiguous column name") { - continue - } - return nil, errors.New("ValidationException", err, k) - } - if !s.IsNull() { - if s.Numeric.IsInt() { - tmp, _ := s.Numeric.Float64() - singleRow[k] = int64(tmp) - } else { - singleRow[k], _ = s.Numeric.Float64() - } - } + err = parseNumericColumn(r, i, k, singleRow) case "BOOL": - var s spanner.NullBool - err := r.Column(i, &s) - if err != nil { - if strings.Contains(err.Error(), "ambiguous column name") { - continue - } - return nil, errors.New("ValidationException", err, k) + err = parseBoolColumn(r, i, k, singleRow) + case "ARRAY": + err = parseStringArrayColumn(r, i, k, singleRow) + } + + if err != nil { + return nil, errors.New("ValidationException", err, k) + } + } + return singleRow, nil +} + +func parseStringColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error { + var s spanner.NullString + err := r.Column(idx, &s) + if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { + return err + } + if !s.IsNull() { + row[col] = s.StringVal + } + return nil +} + +func parseBytesColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error { + var s []byte + err := r.Column(idx, &s) + if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { + return err + } + if len(s) > 0 { + var m interface{} + if err := json.Unmarshal(s, &m); err != nil { + logger.LogError(err, string(s)) + row[col] = string(s) + return nil + } + m = processDecodedData(m) + row[col] = m + } + return nil +} + +func parseInt64Column(r *spanner.Row, idx int, col string, row map[string]interface{}) error { + var s spanner.NullInt64 + err := r.Column(idx, &s) + if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { + return err + } + if !s.IsNull() { + row[col] = s.Int64 + } + return nil +} + +func parseFloat64Column(r *spanner.Row, idx int, col string, row map[string]interface{}) error { + var s spanner.NullFloat64 + err := r.Column(idx, &s) + if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { + return err + } + if !s.IsNull() { + row[col] = s.Float64 + } + return nil +} +func parseNumericColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error { + var s spanner.NullNumeric + err := r.Column(idx, &s) + if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { + return err + } + if !s.IsNull() { + val, _ := s.Numeric.Float64() + if s.Numeric.IsInt() { + row[col] = int64(val) + } else { + row[col] = val + } + } + return nil +} + +func parseBoolColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error { + var s spanner.NullBool + err := r.Column(idx, &s) + if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { + return err + } + if !s.IsNull() { + row[col] = s.Bool + } + return nil +} + +func parseStringArrayColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error { + var s []spanner.NullString + err := r.Column(idx, &s) + if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { + return err + } + var temp []string + for _, val := range s { + temp = append(temp, val.StringVal) + } + if len(s) > 0 { + row[col] = temp + } + return nil +} + +func processDecodedData(m interface{}) interface{} { + if val, ok := m.(string); ok && base64Regexp.MatchString(val) { + if ba, err := base64.StdEncoding.DecodeString(val); err == nil { + var sample interface{} + if err := json.Unmarshal(ba, &sample); err == nil { + return sample } - if !s.IsNull() { - singleRow[k] = s.Bool + } + } + if mp, ok := m.(map[string]interface{}); ok { + for k, v := range mp { + if val, ok := v.(string); ok && base64Regexp.MatchString(val) { + if ba, err := base64.StdEncoding.DecodeString(val); err == nil { + var sample interface{} + if err := json.Unmarshal(ba, &sample); err == nil { + mp[k] = sample + } + } } } } - return singleRow, nil + return m } func checkInifinty(value float64, logData interface{}) error { diff --git a/storage/spanner_test.go b/storage/spanner_test.go index d2a40be..95ee8b7 100644 --- a/storage/spanner_test.go +++ b/storage/spanner_test.go @@ -31,7 +31,13 @@ func Test_parseRow(t *testing.T) { removeNullRow, _ := spanner.NewRow([]string{"strCol", "nullCol"}, []interface{}{"my-text", spanner.NullString{}}) skipCommitTimestampRow, _ := spanner.NewRow([]string{"strCol", "commit_timestamp"}, []interface{}{"my-text", "2021-01-01"}) multipleValuesRow, _ := spanner.NewRow([]string{"strCol", "intCol", "nullCol", "boolCol"}, []interface{}{"my-text", int64(32), spanner.NullString{}, true}) - + simpleArrayRow, _ := spanner.NewRow([]string{"arrayCol"}, []interface{}{ + []spanner.NullString{ + {StringVal: "element1", Valid: true}, + {StringVal: "element2", Valid: true}, + {StringVal: "element3", Valid: true}, + }, + }) type args struct { r *spanner.Row @@ -45,58 +51,64 @@ func Test_parseRow(t *testing.T) { }{ { "ParseStringValue", - args{simpleStringRow, map[string]string{"strCol": "STRING(MAX)"}}, + args{simpleStringRow, map[string]string{"strCol": "STRING(MAX)"}}, map[string]interface{}{"strCol": "my-text"}, false, }, { "ParseIntValue", - args{simpleIntRow, map[string]string{"intCol": "INT64"}}, + args{simpleIntRow, map[string]string{"intCol": "INT64"}}, map[string]interface{}{"intCol": int64(314)}, false, }, { "ParseFloatValue", - args{simpleFloatRow, map[string]string{"floatCol": "FLOAT64"}}, + args{simpleFloatRow, map[string]string{"floatCol": "FLOAT64"}}, map[string]interface{}{"floatCol": 3.14}, false, }, { "ParseNumericIntValue", - args{simpleNumericIntRow, map[string]string{"numericCol": "NUMERIC"}}, + args{simpleNumericIntRow, map[string]string{"numericCol": "NUMERIC"}}, map[string]interface{}{"numericCol": int64(314)}, false, }, { "ParseNumericFloatValue", - args{simpleNumericFloatRow, map[string]string{"numericCol": "NUMERIC"}}, + args{simpleNumericFloatRow, map[string]string{"numericCol": "NUMERIC"}}, map[string]interface{}{"numericCol": 3.25}, false, }, { "ParseBoolValue", - args{simpleBoolRow, map[string]string{"boolCol": "BOOL"}}, + args{simpleBoolRow, map[string]string{"boolCol": "BOOL"}}, map[string]interface{}{"boolCol": true}, false, }, { "RemoveNulls", - args{removeNullRow, map[string]string{"strCol": "STRING(MAX)", "nullCol": "STRING(MAX)"}}, + args{removeNullRow, map[string]string{"strCol": "STRING(MAX)", "nullCol": "STRING(MAX)"}}, map[string]interface{}{"strCol": "my-text"}, false, }, { "SkipCommitTimestamp", - args{skipCommitTimestampRow, map[string]string{"strCol": "STRING(MAX)", "commit_timestamp": "TIMESTAMP"}}, + args{skipCommitTimestampRow, map[string]string{"strCol": "STRING(MAX)", "commit_timestamp": "TIMESTAMP"}}, map[string]interface{}{"strCol": "my-text"}, false, }, { "MultiValueRow", - args{multipleValuesRow, map[string]string{"strCol": "STRING(MAX)", "intCol": "INT64", "nullCol": "STRING(MAX)", "boolCol": "BOOL"}}, + args{multipleValuesRow, map[string]string{"strCol": "STRING(MAX)", "intCol": "INT64", "nullCol": "STRING(MAX)", "boolCol": "BOOL"}}, map[string]interface{}{"strCol": "my-text", "intCol": int64(32), "boolCol": true}, false, }, + { + "ParseStringArray", + args{simpleArrayRow, map[string]string{"arrayCol": "ARRAY"}}, + map[string]interface{}{"arrayCol": []string{"element1", "element2", "element3"}}, + false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From fe02ba58b47457ed61698d3c8214b6f3d10291cc Mon Sep 17 00:00:00 2001 From: taherkl Date: Wed, 4 Dec 2024 16:02:16 +0530 Subject: [PATCH 03/15] Support StringSET --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3b9d84d..63796a2 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ DynamoDB Adapter currently supports the following DynamoDB data types | `BOOL` (boolean) | `BOOL` | `B` (binary type) | `BYTES(MAX)` | `S` (string and data values) | `STRING(MAX)` +| `SS` (string set) | `ARRAY` ## Configuration From 0c7fcaa3e7a56d8c7eef6f3705de09a1760d43e1 Mon Sep 17 00:00:00 2001 From: taherkl Date: Wed, 4 Dec 2024 16:10:01 +0530 Subject: [PATCH 04/15] Add negative test cases --- storage/spanner.go | 2 ++ storage/spanner_test.go | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/storage/spanner.go b/storage/spanner.go index 048e781..0342377 100755 --- a/storage/spanner.go +++ b/storage/spanner.go @@ -778,6 +778,8 @@ func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{}, err = parseBoolColumn(r, i, k, singleRow) case "ARRAY": err = parseStringArrayColumn(r, i, k, singleRow) + default: + return nil, errors.New("TypeNotFound", err, k) } if err != nil { diff --git a/storage/spanner_test.go b/storage/spanner_test.go index 95ee8b7..9663162 100644 --- a/storage/spanner_test.go +++ b/storage/spanner_test.go @@ -38,6 +38,8 @@ func Test_parseRow(t *testing.T) { {StringVal: "element3", Valid: true}, }, }) + invalidTypeRow, _ := spanner.NewRow([]string{"strCol"}, []interface{}{1234}) // Invalid data type + missingColumnRow, _ := spanner.NewRow([]string{"missingCol"}, []interface{}{"value"}) type args struct { r *spanner.Row @@ -109,6 +111,24 @@ func Test_parseRow(t *testing.T) { map[string]interface{}{"arrayCol": []string{"element1", "element2", "element3"}}, false, }, + { + "MissingColumnTypeInDDL", + args{simpleStringRow, map[string]string{"strCol": ""}}, // Missing type in DDL + nil, + true, + }, + { + "InvalidTypeConversion", + args{invalidTypeRow, map[string]string{"strCol": "STRING(MAX)"}}, // Incorrectly trying to parse an int as a string + nil, + true, + }, + { + "ColumnNotInDDL", + args{missingColumnRow, map[string]string{"strCol": "STRING(MAX)"}}, // Column not defined in DDL + nil, + true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From ea241a7c088864fb5db9f1e7d55ecf08e20fb62c Mon Sep 17 00:00:00 2001 From: taherkl Date: Wed, 4 Dec 2024 16:56:41 +0530 Subject: [PATCH 05/15] fix UT --- api/v1/condition_test.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/api/v1/condition_test.go b/api/v1/condition_test.go index 5d53a31..4b30739 100644 --- a/api/v1/condition_test.go +++ b/api/v1/condition_test.go @@ -404,11 +404,7 @@ func TestChangeMaptoDynamoMap(t *testing.T) { "age": map[string]interface{}{"N": "20"}, "value": map[string]interface{}{"N": "10"}, "array": map[string]interface{}{ - "L": []map[string]interface{}{ - {"S": "first"}, - {"S": "second"}, - {"S": "third"}, - }, + "SS": []string{"first", "second", "third"}, }, }, }, From ceb3b1b279fa6db420d22be3b5ba0f572b790795 Mon Sep 17 00:00:00 2001 From: taherkl Date: Wed, 18 Dec 2024 17:48:34 +0530 Subject: [PATCH 06/15] Handling of Binary and Number SET --- api/v1/condition.go | 65 ++++++++++++++++++++++++++++++++++++++------- storage/spanner.go | 36 +++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/api/v1/condition.go b/api/v1/condition.go index 4d40416..e6457b8 100644 --- a/api/v1/condition.go +++ b/api/v1/condition.go @@ -519,23 +519,50 @@ func convertFrom(a *dynamodb.AttributeValue, tableName string) interface{} { return a.B } if a.SS != nil { - l := []string{} - stringMap := make(map[string]struct{}) + uniqueStrings := make(map[string]struct{}) for _, v := range a.SS { - if _, exists := stringMap[*v]; !exists { - stringMap[*v] = struct{}{} - l = append(l, *v) - } + uniqueStrings[*v] = struct{}{} + } + + // Convert map keys to a slice + l := make([]string, 0, len(uniqueStrings)) + for str := range uniqueStrings { + l = append(l, str) } + return l } if a.NS != nil { - l := make([]interface{}, len(a.NS)) - for index, v := range a.NS { - l[index], _ = strconv.ParseFloat(*v, 64) + l := []float64{} + numberMap := make(map[string]struct{}) + for _, v := range a.NS { + // Deduplicate using the string value of the number + if _, exists := numberMap[*v]; !exists { + numberMap[*v] = struct{}{} + // Parse the number and add to the result slice + n, err := strconv.ParseFloat(*v, 64) + if err != nil { + panic(fmt.Sprintf("Invalid number in NS: %s", *v)) + } + l = append(l, n) + } } return l } + if a.BS != nil { + // Handle Binary Set + binarySet := [][]byte{} + binaryMap := make(map[string]struct{}) + for _, v := range a.BS { + // Convert binary slice to string for deduplication + key := string(v) + if _, exists := binaryMap[key]; !exists { + binaryMap[key] = struct{}{} + binarySet = append(binarySet, v) + } + } + return binarySet + } panic(fmt.Sprintf("%#v is not a supported dynamodb.AttributeValue", a)) } @@ -711,6 +738,26 @@ func convertSlice(output map[string]interface{}, v reflect.Value) error { count++ } output["SS"] = listVal + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Float32, reflect.Float64: + listVal := []string{} + for i := 0; i < v.Len(); i++ { + listVal = append(listVal, fmt.Sprintf("%v", v.Index(i).Interface())) + } + output["NS"] = listVal + + case reflect.Slice: + if v.Type().Elem().Kind() == reflect.Uint8 { + binarySet := [][]byte{} + for i := 0; i < v.Len(); i++ { + elem := v.Index(i) + if elem.Kind() == reflect.Slice && elem.IsValid() && !elem.IsNil() { + binarySet = append(binarySet, elem.Bytes()) + } + } + output["BS"] = binarySet + } + default: listVal := make([]map[string]interface{}, 0, v.Len()) diff --git a/storage/spanner.go b/storage/spanner.go index 0342377..625aa4b 100755 --- a/storage/spanner.go +++ b/storage/spanner.go @@ -18,7 +18,6 @@ import ( "context" "encoding/base64" "encoding/json" - "fmt" "math" "reflect" "regexp" @@ -758,7 +757,6 @@ func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{}, } v, ok := colDDL[k] if !ok { - fmt.Println(k) return nil, errors.New("ResourceNotFoundException", k) } @@ -778,6 +776,10 @@ func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{}, err = parseBoolColumn(r, i, k, singleRow) case "ARRAY": err = parseStringArrayColumn(r, i, k, singleRow) + case "ARRAY": + err = parseByteArrayColumn(r, i, k, singleRow) + case "ARRAY": + err = parseNumberArrayColumn(r, i, k, singleRow) default: return nil, errors.New("TypeNotFound", err, k) } @@ -889,6 +891,36 @@ func parseStringArrayColumn(r *spanner.Row, idx int, col string, row map[string] return nil } +func parseByteArrayColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error { + var b [][]byte + err := r.Column(idx, &b) + if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { + return err + } + if len(b) > 0 { + row[col] = b + } + return nil +} + +func parseNumberArrayColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error { + var nums []spanner.NullFloat64 + err := r.Column(idx, &nums) + if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { + return err + } + var temp []float64 + for _, val := range nums { + if val.Valid { + temp = append(temp, val.Float64) + } + } + if len(nums) > 0 { + row[col] = temp + } + return nil +} + func processDecodedData(m interface{}) interface{} { if val, ok := m.(string); ok && base64Regexp.MatchString(val) { if ba, err := base64.StdEncoding.DecodeString(val); err == nil { From bd512994b4859b0f76c4991bf731769d6f27b8bc Mon Sep 17 00:00:00 2001 From: taherkl Date: Wed, 18 Dec 2024 17:49:03 +0530 Subject: [PATCH 07/15] Handling of Binary and Number SET --- api/v1/condition.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/api/v1/condition.go b/api/v1/condition.go index e6457b8..29c1a1f 100644 --- a/api/v1/condition.go +++ b/api/v1/condition.go @@ -536,10 +536,8 @@ func convertFrom(a *dynamodb.AttributeValue, tableName string) interface{} { l := []float64{} numberMap := make(map[string]struct{}) for _, v := range a.NS { - // Deduplicate using the string value of the number if _, exists := numberMap[*v]; !exists { numberMap[*v] = struct{}{} - // Parse the number and add to the result slice n, err := strconv.ParseFloat(*v, 64) if err != nil { panic(fmt.Sprintf("Invalid number in NS: %s", *v)) @@ -554,7 +552,6 @@ func convertFrom(a *dynamodb.AttributeValue, tableName string) interface{} { binarySet := [][]byte{} binaryMap := make(map[string]struct{}) for _, v := range a.BS { - // Convert binary slice to string for deduplication key := string(v) if _, exists := binaryMap[key]; !exists { binaryMap[key] = struct{}{} From 0438af31b8f7da6a2f43349b4f613e88153276cc Mon Sep 17 00:00:00 2001 From: taherkl Date: Sun, 2 Feb 2025 14:11:18 +0530 Subject: [PATCH 08/15] support operations for set type --- api/v1/condition.go | 128 ++++++++++++++++++++++++++++++++++--------- config-files/init.go | 24 ++++---- storage/spanner.go | 11 +--- utils/utils.go | 4 +- 4 files changed, 120 insertions(+), 47 deletions(-) diff --git a/api/v1/condition.go b/api/v1/condition.go index 56dcb04..106dbd9 100644 --- a/api/v1/condition.go +++ b/api/v1/condition.go @@ -86,72 +86,114 @@ func deleteEmpty(s []string) []string { return r } -func parseActionValue(actionValue string, updateAtrr models.UpdateAttr, assignment bool) (map[string]interface{}, *models.UpdateExpressionCondition) { +func parseActionValue(actionValue string, updateAtrr models.UpdateAttr, assignment bool, oldRes map[string]interface{}) (map[string]interface{}, *models.UpdateExpressionCondition) { expr := parseUpdateExpresstion(actionValue) if expr != nil { actionValue = expr.ActionVal expr.AddValues = make(map[string]float64) } + resp := make(map[string]interface{}) pairs := strings.Split(actionValue, ",") var v []string + for _, p := range pairs { var addValue float64 status := false + + // Handle addition (e.g., "count + 1") if strings.Contains(p, "+") { tokens := strings.Split(p, "+") tokens[1] = strings.TrimSpace(tokens[1]) p = tokens[0] - v1, ok := updateAtrr.ExpressionAttributeMap[(tokens[1])] + v1, ok := updateAtrr.ExpressionAttributeMap[tokens[1]] if ok { - v2, ok := v1.(float64) - if ok { + switch v2 := v1.(type) { + case float64: addValue = v2 status = true - } else { - v2, ok := v1.(int64) - if ok { - addValue = float64(v2) - status = true - } + case int64: + addValue = float64(v2) + status = true } } } + + // Handle subtraction (e.g., "count - 2") if strings.Contains(p, "-") { tokens := strings.Split(p, "-") tokens[1] = strings.TrimSpace(tokens[1]) - v1, ok := updateAtrr.ExpressionAttributeMap[(tokens[1])] + v1, ok := updateAtrr.ExpressionAttributeMap[tokens[1]] if ok { - v2, ok := v1.(float64) - if ok { + switch v2 := v1.(type) { + case float64: addValue = -v2 status = true - - } else { - v2, ok := v1.(int64) - if ok { - addValue = float64(-v2) - status = true - } + case int64: + addValue = float64(-v2) + status = true } } } + + // Parse key-value pairs if assignment { v = strings.Split(p, " ") v = deleteEmpty(v) } else { v = strings.Split(p, "=") } + + if len(v) < 2 { + continue + } + v[0] = strings.Replace(v[0], " ", "", -1) v[1] = strings.Replace(v[1], " ", "", -1) + // Handle numeric additions if status { expr.AddValues[v[0]] = addValue } + + // **Detect Set Data Type and Append Values** if updateAtrr.ExpressionAttributeNames[v[0]] != "" { tmp, ok := updateAtrr.ExpressionAttributeMap[v[1]] if ok { - resp[updateAtrr.ExpressionAttributeNames[v[0]]] = tmp + switch existingValue := tmp.(type) { + case []string: // String Set + if strSlice, ok := oldRes[v[0]].([]string); ok { + if strings.Contains(updateAtrr.UpdateExpression, "ADD") { + resp[v[0]] = append(strSlice, existingValue...) + } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { + resp[v[0]] = removeFromSlice(strSlice, existingValue) + } + + } else { + resp[v[0]] = tmp + } + case []float64: // Number Set + if floatSlice, ok := oldRes[v[0]].([]float64); ok { + if strings.Contains(updateAtrr.UpdateExpression, "ADD") { + resp[v[0]] = append(floatSlice, existingValue...) + } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { + resp[v[0]] = removeFromSlice(floatSlice, existingValue) + } + } else { + resp[v[0]] = tmp + } + + case [][]byte: // Binary Set + if byteSlice, ok := oldRes[v[0]].([][]byte); ok { + if strings.Contains(updateAtrr.UpdateExpression, "ADD") { + resp[v[0]] = append(byteSlice, existingValue...) + } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { + resp[v[0]] = removeFromByteSlice(byteSlice, existingValue) + } + } else { + resp[v[0]] = tmp + } + } } } else { if strings.Contains(v[1], "%") { @@ -171,13 +213,49 @@ func parseActionValue(actionValue string, updateAtrr models.UpdateAttr, assignme } } } + // Merge primaryKeyMap and updateAttributes for k, v := range updateAtrr.PrimaryKeyMap { resp[k] = v } + return resp, expr } +func removeFromSlice[T comparable](slice []T, toRemove []T) []T { + result := []T{} + removeMap := make(map[T]struct{}, len(toRemove)) + + for _, val := range toRemove { + removeMap[val] = struct{}{} + } + + for _, val := range slice { + if _, found := removeMap[val]; !found { + result = append(result, val) + } + } + return result +} + +func removeFromByteSlice(slice [][]byte, toRemove [][]byte) [][]byte { + result := [][]byte{} + + for _, item := range slice { + found := false + for _, rem := range toRemove { + if bytes.Equal(item, rem) { // Use bytes.Equal to compare byte slices + found = true + break + } + } + if !found { + result = append(result, item) + } + } + return result +} + func parseUpdateExpresstion(actionValue string) *models.UpdateExpressionCondition { if actionValue == "" { return nil @@ -228,17 +306,17 @@ func performOperation(ctx context.Context, action string, actionValue string, up switch { case action == "DELETE": // perform delete - m, expr := parseActionValue(actionValue, updateAtrr, true) + m, expr := parseActionValue(actionValue, updateAtrr, true, oldRes) res, err := services.Del(ctx, updateAtrr.TableName, updateAtrr.PrimaryKeyMap, updateAtrr.ConditionExpression, m, expr) return res, m, err case action == "SET": // Update data in table - m, expr := parseActionValue(actionValue, updateAtrr, false) + m, expr := parseActionValue(actionValue, updateAtrr, false, oldRes) res, err := services.Put(ctx, updateAtrr.TableName, m, expr, updateAtrr.ConditionExpression, updateAtrr.ExpressionAttributeMap, oldRes) return res, m, err case action == "ADD": // Add data in table - m, expr := parseActionValue(actionValue, updateAtrr, true) + m, expr := parseActionValue(actionValue, updateAtrr, true, oldRes) res, err := services.Add(ctx, updateAtrr.TableName, updateAtrr.PrimaryKeyMap, updateAtrr.ConditionExpression, m, updateAtrr.ExpressionAttributeMap, expr, oldRes) return res, m, err @@ -744,7 +822,7 @@ func convertSlice(output map[string]interface{}, v reflect.Value) error { output["NS"] = listVal case reflect.Slice: - if v.Type().Elem().Kind() == reflect.Uint8 { + if v.Type().Elem().Kind() == reflect.Slice { binarySet := [][]byte{} for i := 0; i < v.Len(); i++ { elem := v.Index(i) diff --git a/config-files/init.go b/config-files/init.go index bdcb118..278a414 100644 --- a/config-files/init.go +++ b/config-files/init.go @@ -51,7 +51,7 @@ var ( spannerIndexName STRING(MAX), actualTable STRING(MAX), spannerDataType STRING(MAX) - ) PRIMARY KEY (tableName, column)` + ) PRIMARY KEY (tableName, column);` ) // Entry point for the application @@ -83,7 +83,7 @@ func main() { runDryRun(config.Spanner.DynamoQueryLimit) } else { fmt.Println("-- Executing Setup on Spanner --") - executeSetup(ctx, adminClient, databaseName) + executeSetup(ctx, adminClient, databaseName, config.Spanner.DynamoQueryLimit) } } @@ -118,17 +118,17 @@ func runDryRun(limit int32) { fmt.Printf("Processing table: %s\n", tableName) // Generate and print table-specific DDL - ddl := generateTableDDL(tableName, client) + ddl := generateTableDDL(tableName, client, limit) fmt.Printf("-- DDL for table: %s --\n%s\n", tableName, ddl) // Generate and print insert queries - generateInsertQueries(tableName, client) + generateInsertQueries(tableName, client, limit) } } // Generate DDL statement for a specific DynamoDB table -func generateTableDDL(tableName string, client *dynamodb.Client) string { - attributes, partitionKey, sortKey, err := fetchTableAttributes(client, tableName, models.GlobalConfig.Spanner.DynamoQueryLimit) +func generateTableDDL(tableName string, client *dynamodb.Client, limit int32) string { + attributes, partitionKey, sortKey, err := fetchTableAttributes(client, tableName, limit) if err != nil { log.Printf("Failed to fetch attributes for table %s: %v", tableName, err) return "" @@ -146,14 +146,14 @@ func generateTableDDL(tableName string, client *dynamodb.Client) string { }()) return fmt.Sprintf( - "CREATE TABLE %s (\n\t%s\n) %s", + "CREATE TABLE %s (\n\t%s\n) %s;", tableName, strings.Join(columns, ",\n\t"), primaryKey, ) } // Generate insert queries for a given DynamoDB table -func generateInsertQueries(tableName string, client *dynamodb.Client) { - attributes, partitionKey, sortKey, err := fetchTableAttributes(client, tableName, models.GlobalConfig.Spanner.DynamoQueryLimit) +func generateInsertQueries(tableName string, client *dynamodb.Client, limit int32) { + attributes, partitionKey, sortKey, err := fetchTableAttributes(client, tableName, limit) if err != nil { log.Printf("Failed to fetch attributes for table %s: %v", tableName, err) return @@ -163,7 +163,7 @@ func generateInsertQueries(tableName string, client *dynamodb.Client) { spannerDataType := utils.ConvertDynamoTypeToSpannerType(dataType) query := fmt.Sprintf( `INSERT INTO dynamodb_adapter_table_ddl - (column, tableName, dataType, originalColumn, partitionKey, sortKey, spannerIndexName, actualTable, spannerDataType) + (column, tableName, dynamoDataType, originalColumn, partitionKey, sortKey, spannerIndexName, actualTable, spannerDataType) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s');`, column, tableName, dataType, column, partitionKey, sortKey, column, tableName, spannerDataType, ) @@ -172,7 +172,7 @@ func generateInsertQueries(tableName string, client *dynamodb.Client) { } // Execute the setup process: create database, tables, and migrate data -func executeSetup(ctx context.Context, adminClient *Admindatabase.DatabaseAdminClient, databaseName string) { +func executeSetup(ctx context.Context, adminClient *Admindatabase.DatabaseAdminClient, databaseName string, limit int32) { // Create the Spanner database if it doesn't exist if err := createDatabase(ctx, adminClient, databaseName); err != nil { @@ -193,7 +193,7 @@ func executeSetup(ctx context.Context, adminClient *Admindatabase.DatabaseAdminC for _, tableName := range tables { // Generate and apply table-specific DDL - ddl := generateTableDDL(tableName, client) + ddl := generateTableDDL(tableName, client, limit) if err := createTable(ctx, adminClient, databaseName, ddl); err != nil { log.Printf("Failed to create table %s: %v", tableName, err) continue diff --git a/storage/spanner.go b/storage/spanner.go index 049079d..cbff6c7 100755 --- a/storage/spanner.go +++ b/storage/spanner.go @@ -810,10 +810,10 @@ func parseBytesColumn(r *spanner.Row, idx int, col string, row map[string]interf if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { return err } + if len(s) > 0 { var m interface{} if err := json.Unmarshal(s, &m); err != nil { - logger.LogError(err, string(s)) row[col] = string(s) return nil } @@ -824,18 +824,13 @@ func parseBytesColumn(r *spanner.Row, idx int, col string, row map[string]interf } func parseNumericColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error { - var s spanner.NullNumeric + var s spanner.NullFloat64 err := r.Column(idx, &s) if err != nil && !strings.Contains(err.Error(), "ambiguous column name") { return err } if !s.IsNull() { - val, _ := s.Numeric.Float64() - if s.Numeric.IsInt() { - row[col] = int64(val) - } else { - row[col] = val - } + row[col] = s.Float64 } return nil } diff --git a/utils/utils.go b/utils/utils.go index 47829b7..d882c4c 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -187,11 +187,11 @@ func ConvertDynamoTypeToSpannerType(dynamoType string) string { case "NULL": return "NULL" case "SS": - return "ARRAY" + return "ARRAY" case "NS": return "ARRAY" case "BS": - return "ARRAY" + return "ARRAY" case "M": return "JSON" case "L": From 5205d5e54d3bf9f121693b3e2842bdf510629cc0 Mon Sep 17 00:00:00 2001 From: taherkl Date: Mon, 3 Feb 2025 10:04:32 +0530 Subject: [PATCH 09/15] test cases for parseActionValue --- api/v1/condition.go | 62 ++++++------ api/v1/condition_test.go | 201 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 231 insertions(+), 32 deletions(-) diff --git a/api/v1/condition.go b/api/v1/condition.go index 106dbd9..9d3c72c 100644 --- a/api/v1/condition.go +++ b/api/v1/condition.go @@ -156,62 +156,60 @@ func parseActionValue(actionValue string, updateAtrr models.UpdateAttr, assignme expr.AddValues[v[0]] = addValue } - // **Detect Set Data Type and Append Values** + key := v[0] if updateAtrr.ExpressionAttributeNames[v[0]] != "" { + key = updateAtrr.ExpressionAttributeNames[v[0]] + } + + if strings.Contains(v[1], "%") { + for j := 0; j < len(expr.Field); j++ { + if strings.Contains(v[1], "%"+expr.Value[j]+"%") { + tmp, ok := updateAtrr.ExpressionAttributeMap[expr.Value[j]] + if ok { + resp[key] = tmp + } + } + } + } else { tmp, ok := updateAtrr.ExpressionAttributeMap[v[1]] if ok { - switch existingValue := tmp.(type) { + switch newValue := tmp.(type) { case []string: // String Set - if strSlice, ok := oldRes[v[0]].([]string); ok { + if strSlice, ok := oldRes[key].([]string); ok { if strings.Contains(updateAtrr.UpdateExpression, "ADD") { - resp[v[0]] = append(strSlice, existingValue...) + resp[key] = append(strSlice, newValue...) } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { - resp[v[0]] = removeFromSlice(strSlice, existingValue) + resp[key] = removeFromSlice(strSlice, newValue) } - } else { - resp[v[0]] = tmp + resp[key] = tmp } case []float64: // Number Set - if floatSlice, ok := oldRes[v[0]].([]float64); ok { + if floatSlice, ok := oldRes[key].([]float64); ok { if strings.Contains(updateAtrr.UpdateExpression, "ADD") { - resp[v[0]] = append(floatSlice, existingValue...) + resp[key] = append(floatSlice, newValue...) } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { - resp[v[0]] = removeFromSlice(floatSlice, existingValue) + resp[key] = removeFromSlice(floatSlice, newValue) } } else { - resp[v[0]] = tmp + resp[key] = tmp } - case [][]byte: // Binary Set - if byteSlice, ok := oldRes[v[0]].([][]byte); ok { + if byteSlice, ok := oldRes[key].([][]byte); ok { if strings.Contains(updateAtrr.UpdateExpression, "ADD") { - resp[v[0]] = append(byteSlice, existingValue...) + resp[key] = append(byteSlice, newValue...) } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { - resp[v[0]] = removeFromByteSlice(byteSlice, existingValue) + resp[key] = removeFromByteSlice(byteSlice, newValue) } } else { - resp[v[0]] = tmp + resp[key] = tmp } - } - } - } else { - if strings.Contains(v[1], "%") { - for j := 0; j < len(expr.Field); j++ { - if strings.Contains(v[1], "%"+expr.Value[j]+"%") { - tmp, ok := updateAtrr.ExpressionAttributeMap[expr.Value[j]] - if ok { - resp[v[0]] = tmp - } - } - } - } else { - tmp, ok := updateAtrr.ExpressionAttributeMap[v[1]] - if ok { - resp[v[0]] = tmp + default: + resp[key] = tmp } } } + } // Merge primaryKeyMap and updateAttributes diff --git a/api/v1/condition_test.go b/api/v1/condition_test.go index 4b30739..a84edd6 100644 --- a/api/v1/condition_test.go +++ b/api/v1/condition_test.go @@ -15,6 +15,7 @@ package v1 import ( + "reflect" "testing" "github.com/aws/aws-sdk-go/aws" @@ -415,3 +416,203 @@ func TestChangeMaptoDynamoMap(t *testing.T) { assert.Equal(t, got, tc.want) } } + +func TestParseActionValue(t *testing.T) { + tests := []struct { + name string + updateAttr models.UpdateAttr + oldRes map[string]interface{} + expectedResult map[string]interface{} + actionValue string + }{ + { + name: "Simple key-value assignment", + updateAttr: models.UpdateAttr{ + UpdateExpression: "SET count = :countVal", + ExpressionAttributeMap: map[string]interface{}{ + ":countVal": 10, + }, + ExpressionAttributeNames: map[string]string{}, + PrimaryKeyMap: map[string]interface{}{ + "id": "1", + }, + }, + oldRes: map[string]interface{}{}, + expectedResult: map[string]interface{}{ + "id": "1", + "count": 10, + }, + actionValue: "count :countVal", + }, + { + name: "Addition operation", + updateAttr: models.UpdateAttr{ + UpdateExpression: "SET count = count + :incr", + ExpressionAttributeMap: map[string]interface{}{ + ":incr": 1, + }, + ExpressionAttributeNames: map[string]string{}, + PrimaryKeyMap: map[string]interface{}{ + "id": "1", + }, + }, + expectedResult: map[string]interface{}{ + "id": "1", + }, + actionValue: "count = count + :incr", + }, + { + name: "Subtraction operation", + updateAttr: models.UpdateAttr{ + UpdateExpression: "SET count = count - :decr", + ExpressionAttributeMap: map[string]interface{}{ + ":decr": 2, + }, + ExpressionAttributeNames: map[string]string{}, + PrimaryKeyMap: map[string]interface{}{ + "id": "1", + }, + }, + oldRes: map[string]interface{}{}, + expectedResult: map[string]interface{}{ + "id": "1", + }, + actionValue: "count = count - :decr", + }, + { + name: "String set append with ADD", + updateAttr: models.UpdateAttr{ + UpdateExpression: "ADD tags :newTags", + ExpressionAttributeMap: map[string]interface{}{ + ":newTags": []string{"newTag"}, + }, + ExpressionAttributeNames: map[string]string{}, + PrimaryKeyMap: map[string]interface{}{ + "id": "1", + }, + }, + oldRes: map[string]interface{}{ + "tags": []string{"oldTag"}, + }, + expectedResult: map[string]interface{}{ + "id": "1", + "tags": []string{"oldTag", "newTag"}, + }, + actionValue: "tags :newTags", + }, + { + name: "String set removal with DELETE", + updateAttr: models.UpdateAttr{ + UpdateExpression: "DELETE tags :removeTags", + ExpressionAttributeMap: map[string]interface{}{ + ":removeTags": []string{"oldTag"}, + }, + ExpressionAttributeNames: map[string]string{}, + PrimaryKeyMap: map[string]interface{}{ + "id": "1", + }, + }, + oldRes: map[string]interface{}{ + "tags": []string{"oldTag", "newTag"}, + }, + expectedResult: map[string]interface{}{ + "id": "1", + "tags": []string{"newTag"}, + }, + actionValue: "tags :removeTags", + }, + { + name: "Number set append with ADD", + updateAttr: models.UpdateAttr{ + UpdateExpression: "ADD tags :newTags", + ExpressionAttributeMap: map[string]interface{}{ + ":newTags": []float64{10}, + }, + ExpressionAttributeNames: map[string]string{}, + PrimaryKeyMap: map[string]interface{}{ + "id": "1", + }, + }, + oldRes: map[string]interface{}{ + "tags": []float64{20}, + }, + expectedResult: map[string]interface{}{ + "id": "1", + "tags": []float64{20, 10}, + }, + actionValue: "tags :newTags", + }, + { + name: "Number set removal with DELETE", + updateAttr: models.UpdateAttr{ + UpdateExpression: "DELETE tags :removeTags", + ExpressionAttributeMap: map[string]interface{}{ + ":removeTags": []float64{10}, + }, + ExpressionAttributeNames: map[string]string{}, + PrimaryKeyMap: map[string]interface{}{ + "id": "1", + }, + }, + oldRes: map[string]interface{}{ + "tags": []float64{20, 10}, + }, + expectedResult: map[string]interface{}{ + "id": "1", + "tags": []float64{20}, + }, + actionValue: "tags :removeTags", + }, + { + name: "Binary set append with ADD", + updateAttr: models.UpdateAttr{ + UpdateExpression: "ADD binaryData :newBinary", + ExpressionAttributeMap: map[string]interface{}{ + ":newBinary": [][]byte{[]byte("newData")}, + }, + ExpressionAttributeNames: map[string]string{}, + PrimaryKeyMap: map[string]interface{}{ + "id": "1", + }, + }, + oldRes: map[string]interface{}{ + "binaryData": [][]byte{[]byte("oldData")}, + }, + expectedResult: map[string]interface{}{ + "id": "1", + "binaryData": [][]byte{[]byte("oldData"), []byte("newData")}, + }, + actionValue: "binaryData :newBinary", + }, + { + name: "Binary set removal with DELETE", + updateAttr: models.UpdateAttr{ + UpdateExpression: "DELETE binaryData :removeBinary", + ExpressionAttributeMap: map[string]interface{}{ + ":removeBinary": [][]byte{[]byte("oldData")}, + }, + ExpressionAttributeNames: map[string]string{}, + PrimaryKeyMap: map[string]interface{}{ + "id": "1", + }, + }, + oldRes: map[string]interface{}{ + "binaryData": [][]byte{[]byte("oldData"), []byte("newData")}, + }, + expectedResult: map[string]interface{}{ + "id": "1", + "binaryData": [][]byte{[]byte("newData")}, + }, + actionValue: "binaryData :removeBinary", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, _ := parseActionValue(tt.actionValue, tt.updateAttr, true, tt.oldRes) + if !reflect.DeepEqual(result, tt.expectedResult) { + t.Errorf("Test %s failed: expected %v, got %v", tt.name, tt.expectedResult, result) + } + }) + } +} From ede259a03ac2bf60d78ce66c6491dd6affb8530c Mon Sep 17 00:00:00 2001 From: taherkl Date: Mon, 3 Feb 2025 10:20:47 +0530 Subject: [PATCH 10/15] UT for parseRow --- storage/spanner_test.go | 189 +++++++++++++++++++++++++++++++++++----- 1 file changed, 165 insertions(+), 24 deletions(-) diff --git a/storage/spanner_test.go b/storage/spanner_test.go index 7dfa1ba..86901d4 100644 --- a/storage/spanner_test.go +++ b/storage/spanner_test.go @@ -128,30 +128,171 @@ func Test_parseRow(t *testing.T) { colDDL: map[string]string{"boolCol": "BOOL", "intCol": "N", "strCol": "S"}, want: map[string]interface{}{"boolCol": true, "intCol": 32.0, "strCol": "my-text"}, }, - // { - // "ParseStringArray", - // args{simpleArrayRow, map[string]string{"arrayCol": "ARRAY"}}, - // map[string]interface{}{"arrayCol": []string{"element1", "element2", "element3"}}, - // false, - // }, - // { - // "MissingColumnTypeInDDL", - // args{simpleStringRow, map[string]string{"strCol": ""}}, // Missing type in DDL - // nil, - // true, - // }, - // { - // "InvalidTypeConversion", - // args{invalidTypeRow, map[string]string{"strCol": "STRING(MAX)"}}, // Incorrectly trying to parse an int as a string - // nil, - // true, - // }, - // { - // "ColumnNotInDDL", - // args{missingColumnRow, map[string]string{"strCol": "STRING(MAX)"}}, // Column not defined in DDL - // nil, - // true, - // }, + { + name: "ParseStringArray", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"arrayCol"}, []interface{}{ + []spanner.NullString{ + {StringVal: "element1", Valid: true}, + {StringVal: "element2", Valid: true}, + {StringVal: "element3", Valid: true}, + }, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"arrayCol": "SS"}, + want: map[string]interface{}{"arrayCol": []string{"element1", "element2", "element3"}}, + wantError: false, + }, + { + name: "MissingColumnTypeInDDL", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"strCol"}, []interface{}{ + spanner.NullString{StringVal: "test", Valid: true}, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"strCol": ""}, // Missing type in DDL + want: nil, + wantError: true, + }, + { + name: "InvalidTypeConversion", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"strCol"}, []interface{}{ + spanner.NullFloat64{Float64: 123.45, Valid: true}, // Trying to parse float as a string + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"strCol": "S"}, + want: nil, + wantError: true, + }, + { + name: "ColumnNotInDDL", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"someOtherCol"}, []interface{}{ + spanner.NullString{StringVal: "missing-column", Valid: true}, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"strCol": "S"}, // Column "someOtherCol" not in DDL + want: nil, + wantError: true, + }, + { + name: "ParseNumberArray", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"numberArrayCol"}, []interface{}{ + []spanner.NullFloat64{ + {Float64: 1.1, Valid: true}, + {Float64: 2.2, Valid: true}, + {Float64: 3.3, Valid: true}, + }, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"numberArrayCol": "NS"}, + want: map[string]interface{}{"numberArrayCol": []float64{1.1, 2.2, 3.3}}, + wantError: false, + }, + { + name: "ParseBinaryArray", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"binaryArrayCol"}, []interface{}{ + [][]byte{ + []byte("binaryData1"), + []byte("binaryData2"), + }, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"binaryArrayCol": "BS"}, + want: map[string]interface{}{"binaryArrayCol": [][]byte{[]byte("binaryData1"), []byte("binaryData2")}}, + wantError: false, + }, + { + name: "EmptyNumberArray", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"numberArrayCol"}, []interface{}{ + []spanner.NullFloat64{}, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"numberArrayCol": "NS"}, + want: map[string]interface{}{}, + wantError: false, + }, + { + name: "EmptyBinaryArray", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"binaryArrayCol"}, []interface{}{ + [][]byte{}, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"binaryArrayCol": "BS"}, + want: map[string]interface{}{}, + wantError: false, + }, + { + name: "InvalidNumberArrayConversion", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"numberArrayCol"}, []interface{}{ + []spanner.NullString{ + {StringVal: "not-a-number", Valid: true}, // Invalid conversion + }, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"numberArrayCol": "NS"}, + want: nil, + wantError: true, + }, + { + name: "InvalidBinaryArrayConversion", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"binaryArrayCol"}, []interface{}{ + []spanner.NullString{ + {StringVal: "not-binary-data", Valid: true}, // Invalid conversion + }, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"binaryArrayCol": "BS"}, + want: nil, + wantError: true, + }, } for _, tt := range tests { From 7da81588135ca87f6447786944ac64fc1bee45a2 Mon Sep 17 00:00:00 2001 From: taherkl Date: Mon, 3 Feb 2025 14:13:25 +0530 Subject: [PATCH 11/15] PR review comment fixes --- api/v1/condition.go | 6 ++---- storage/spanner.go | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/api/v1/condition.go b/api/v1/condition.go index 9d3c72c..4b9b266 100644 --- a/api/v1/condition.go +++ b/api/v1/condition.go @@ -805,10 +805,8 @@ func convertSlice(output map[string]interface{}, v reflect.Value) error { output["B"] = append([]byte{}, b...) case reflect.String: listVal := []string{} - count := 0 for i := 0; i < v.Len(); i++ { listVal = append(listVal, v.Index(i).String()) - count++ } output["SS"] = listVal case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, @@ -829,12 +827,13 @@ func convertSlice(output map[string]interface{}, v reflect.Value) error { } } output["BS"] = binarySet + } else { + return fmt.Errorf("type of slice not supported: %s", v.Type().Elem().Kind().String()) } default: listVal := make([]map[string]interface{}, 0, v.Len()) - count := 0 for i := 0; i < v.Len(); i++ { elem := make(map[string]interface{}) err := convertMapToDynamoObject(elem, v.Index(i)) @@ -842,7 +841,6 @@ func convertSlice(output map[string]interface{}, v reflect.Value) error { return err } listVal = append(listVal, elem) - count++ } output["L"] = listVal } diff --git a/storage/spanner.go b/storage/spanner.go index cbff6c7..4f78aba 100755 --- a/storage/spanner.go +++ b/storage/spanner.go @@ -814,6 +814,7 @@ func parseBytesColumn(r *spanner.Row, idx int, col string, row map[string]interf if len(s) > 0 { var m interface{} if err := json.Unmarshal(s, &m); err != nil { + // Instead of an error while unmarshalling fall back to the raw string. row[col] = string(s) return nil } From d6585353733fe93387732eb478099460a07918b5 Mon Sep 17 00:00:00 2001 From: Taher Lakdawala <78196491+taherkl@users.noreply.github.com> Date: Thu, 6 Feb 2025 23:15:14 +0530 Subject: [PATCH 12/15] Revised Init process (#76) Revised init process --------- Co-authored-by: Nikita Jain Co-authored-by: taherkl --- .circleci/config.yml | 23 +- Dockerfile | 2 +- README.md | 212 ++++------- api/v1/db.go | 4 +- config-files/init.go | 524 +++++++++++++++++++++++++++ config-files/production/config.json | 5 - config-files/production/spanner.json | 5 - config-files/production/tables.json | 20 - config-files/staging/config.json | 5 - config-files/staging/spanner.json | 5 - config-files/staging/tables.json | 20 - config.yaml | 6 + config/config.go | 81 ++--- config/config_test.go | 2 +- go.mod | 28 +- go.sum | 30 ++ initializer/initializer.go | 5 +- integrationtest/README.md | 68 +--- integrationtest/api_test.go | 58 ++- integrationtest/setup.go | 226 +++--------- main.go | 7 +- models/model.go | 36 +- service/services/services.go | 2 +- service/spanner/spanner.go | 31 +- storage/spanner.go | 27 +- storage/spanner_test.go | 160 ++++---- storage/storage.go | 19 +- utils/utils.go | 28 ++ 28 files changed, 986 insertions(+), 653 deletions(-) create mode 100644 config-files/init.go delete mode 100644 config-files/production/config.json delete mode 100644 config-files/production/spanner.json delete mode 100644 config-files/production/tables.json delete mode 100644 config-files/staging/config.json delete mode 100644 config-files/staging/spanner.json delete mode 100644 config-files/staging/tables.json create mode 100644 config.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index d7187fc..4d77766 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,11 +5,20 @@ version: 2 jobs: build: docker: - - image: circleci/golang:1.13 - working_directory: /go/src/github.com/cloudspannerecosystem/dynamodb-adapter + - image: cimg/go:1.22 + working_directory: ~/project steps: - checkout - - run: go build + - restore_cache: + keys: + - go-mod-v1-{{ checksum "go.mod" }} + - go-mod-v1- + - run: go mod tidy + - save_cache: + paths: + - ~/.go/pkg/mod + key: go-mod-v1-{{ checksum "go.mod" }} + - run: go build ./... lint_golang: docker: @@ -28,16 +37,16 @@ jobs: unit_test: docker: - - image: circleci/golang:1.13 - working_directory: /go/src/github.com/cloudspannerecosystem/dynamodb-adapter + - image: cimg/go:1.22 + working_directory: ~/project steps: - checkout - run: go test -v -short ./... integration_test: docker: - - image: circleci/golang:1.13 - working_directory: /go/src/github.com/cloudspannerecosystem/dynamodb-adapter + - image: cimg/go:1.22 + working_directory: ~/project steps: - checkout - run: diff --git a/Dockerfile b/Dockerfile index 5890d6c..58865fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.12 +FROM golang:1.22 # Set the Current Working Directory inside the container diff --git a/README.md b/README.md index 601608f..525eb4c 100644 --- a/README.md +++ b/README.md @@ -59,16 +59,24 @@ DynamoDB Adapter currently supports the following DynamoDB data types ## Configuration -DynamoDB Adapter requires two tables to store metadata and configuration for -the project: `dynamodb_adapter_table_ddl` and -`dynamodb_adapter_config_manager`. There are also three configuration files -required by the adapter: `config.json`, `spanner.json`, `tables.json`. +### config.yaml -By default there are two folders **production** and **staging** in -[config-files](./config-files). This is configurable by using the enviroment -variable `ACTIVE_ENV` and can be set to other environment names, so long as -there is a matching directory in the `config-files` directory. If `ACTIVE_ENV` -is not set the default environtment is **staging**. +This file defines the necessary settings for the adapter. +A sample configuration might look like this: + +spanner: + project_id: "my-project-id" + instance_id: "my-instance-id" + database_name: "my-database-name" + query_limit: "query_limit" + dynamo_query_limit: "dynamo_query_limit" + +The fields are: +project_id: The Google Cloud project ID. +instance_id: The Spanner instance ID. +database_name: The database name in Spanner. +query_limit: Database query limit. +dynamo_query_limit: DynamoDb query limit. ### dynamodb_adapter_table_ddl @@ -79,121 +87,59 @@ present in DynamoDB. This mapping is required because DynamoDB supports the special characters in column names while Cloud Spanner only supports underscores(_). For more: [Spanner Naming Conventions](https://cloud.google.com/spanner/docs/data-definition-language#naming_conventions) -```sql -CREATE TABLE -dynamodb_adapter_table_ddl -( - column STRING(MAX), - tableName STRING(MAX), - dataType STRING(MAX), - originalColumn STRING(MAX), -) PRIMARY KEY (tableName, column) -``` +### Initialization Modes + +DynamoDB Adapter supports two modes of initialization: + +#### Dry Run Mode -![dynamodb_adapter_table_ddl sample data](images/config_spanner.png) - -### dynamodb_adapter_config_manager - -`dynamodb_adapter_config_manager` contains the Pub/Sub configuration used for -DynamoDB Stream compatability. It is used to do some additional operation -required on the change of data in tables. It can trigger New and Old data on -given Pub/Sub topic. - -```sql -CREATE TABLE -dynamodb_adapter_config_manager -( - tableName STRING(MAX), - config STRING(MAX), - cronTime STRING(MAX), - enabledStream STRING(MAX), - pubsubTopic STRING(MAX), - uniqueValue STRING(MAX), -) PRIMARY KEY (tableName) +This mode generates the Spanner queries required to: + +Create the dynamodb_adapter_table_ddl table in Spanner. +Insert metadata for all DynamoDB tables into dynamodb_adapter_table_ddl. +These queries are printed to the console without executing them on Spanner, +allowing you to review them before making changes. + +```sh +go run config-files/init.go --dry_run ``` -### config-files/{env}/config.json +#### Execution Mode + +This mode executes the Spanner queries generated +during the dry run on the Spanner instance. It will: -`config.json` contains the basic settings for DynamoDB Adapter; GCP Project, -Cloud Spanner Database and query record limit. +Create the dynamodb_adapter_table_ddl table in Spanner if it does not exist. +Insert metadata for all DynamoDB tables into the dynamodb_adapter_table_ddl table. -| Key | Description | -| ----------------- | ----------- | -| GoogleProjectID | Your Google Project ID | -| SpannerDb | Your Spanner Database Name | -| QueryLimit | Default limit for the number of records returned in query | +```sh -For example: +go run config-files/init.go -```json -{ - "GoogleProjectID" : "first-project", - "SpannerDb" : "test-db", - "QueryLimit" : 5000 -} ``` -### config-files/{env}/spanner.json +### Prerequisites for Initialization + +AWS CLI: +Configure AWS credentials: -`spanner.json` is a key/value mapping file for table names with a Cloud Spanner -instance ids. This enables the adapter to query data for a particular table on -different Cloud Spanner instances. +```sh -For example: +aws configure set aws_access_key_id YOUR_ACCESS_KEY +aws configure set aws_secret_access_key YOUR_SECRET_KEY +aws configure set default.region YOUR_REGION +aws configure set aws_session_token YOUR_SESSION_TOKEN -```json -{ - "dynamodb_adapter_table_ddl": "spanner-2 ", - "dynamodb_adapter_config_manager": "spanner-2", - "tableName1": "spanner-1", - "tableName2": "spanner-1" - ... - ... -} ``` -### config-files/{env}/tables.json - -`tables.json` contains the description of the tables as they appear in -DynamoDB. This includes all table's primary key, columns and index information. -This file supports the update and query operations by providing the primary -key, sort key and any other indexes present. - -| Key | Description | -| ----------------- | ----------- | -| tableName | Name of the table in DynamoDB | -| partitionKey | Primary key of the table in DynamoDB | -| sortKey | Sorting key of the table in DynamoDB | -| attributeTypes | Key/Value list of column names and type | -| indices | Collection of index objects that represent the indexes present in the DynamoDB table | - -For example: - -```json -{ - "tableName": { - "partitionKey": "primary key or Partition key", - "sortKey": "sorting key of dynamoDB adapter", - "attributeTypes": { - "column_a": "N", - "column_b": "S", - "column_of_bytes": "B", - "my_boolean_column": "BOOL" - }, - "indices": { - "indexName1": { - "sortKey": "sort key for indexName1", - "partitionKey": "partition key for indexName1" - }, - "another_index": { - "sortKey": "sort key for another_index", - "partitionKey": "partition key for another_index" - } - } - }, - ..... - ..... -} +Google Cloud CLI: +Authenticate and set up your environment: + +```sh + +gcloud auth application-default login +gcloud config set project [MY_PROJECT_NAME] + ``` ## Starting DynamoDB Adapter @@ -207,7 +153,9 @@ credentials to use the Cloud Spanner API. In particular, ensure that you run ```sh + gcloud auth application-default login + ``` to set up your local development environment with authentication credentials. @@ -215,53 +163,17 @@ to set up your local development environment with authentication credentials. Set the GCLOUD_PROJECT environment variable to your Google Cloud project ID: ```sh + gcloud config set project [MY_PROJECT NAME] -``` -```sh -export ACTIVE_ENV=PRODUCTION -go run main.go ``` -### Internal Startup Stages - -When DynamoDB Adapter starts up the following steps are performed: - -* Stage 1 - Configuration is loaded according the Environment Variable - *ACTIVE_ENV* -* Stage 2 - Connections to Cloud Spanner instances are initialized. - Connections to all the instances are started it doesn't need to start the - connection again and again for every request. -* Stage 3 - `dynamodb_adapter_table_ddl` is parsed and will stored in ram for - faster access of data. -* Stage 4 - `dynamodb_adapter_config_manager` is loaded into ram. The adapter - will check every 1 min if configuration has been changed, if data is changed - it will be updated in memory. -* Stage 5 - Start the API listener to accept DynamoDB operations. - -## Advanced - -### Embedding the Configuration - -The rice-box package can be used to increase preformance by converting the -configuration files into Golang source code and there by compiling them into -the binary. If they are not found in the binary rice-box will look to the -disk for the configuration files. - -#### Install rice package - -This package is required to load the config files. This is required in the -first step of the running DynamoDB Adapter. - -Follow the [link](https://github.com/GeertJohan/go.rice#installation). +```sh -#### run command for creating the file +go run main.go -This is required to increase the performance when any config file is changed -so that configuration files can be loaded directly from go file. +``` -```sh -rice embed-go ``` ## API Documentation diff --git a/api/v1/db.go b/api/v1/db.go index e499c65..c95aaf7 100644 --- a/api/v1/db.go +++ b/api/v1/db.go @@ -59,7 +59,7 @@ func RouteRequest(c *gin.Context) { case "UpdateItem": Update(c) default: - c.JSON(errors.New("ValidationException", "Invalid X-Amz-Target header value of" + amzTarget). + c.JSON(errors.New("ValidationException", "Invalid X-Amz-Target header value of "+amzTarget). HTTPResponse("X-Amz-Target Header not supported")) } } @@ -186,7 +186,7 @@ func queryResponse(query models.Query, c *gin.Context) { } if query.Limit == 0 { - query.Limit = config.ConfigurationMap.QueryLimit + query.Limit = models.GlobalConfig.Spanner.QueryLimit } query.ExpressionAttributeNames = ChangeColumnToSpannerExpressionName(query.TableName, query.ExpressionAttributeNames) query = ReplaceHashRangeExpr(query) diff --git a/config-files/init.go b/config-files/init.go new file mode 100644 index 0000000..99656a5 --- /dev/null +++ b/config-files/init.go @@ -0,0 +1,524 @@ +// Copyright (c) DataStax, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "regexp" + "strings" + + "cloud.google.com/go/spanner" + Admindatabase "cloud.google.com/go/spanner/admin/database/apiv1" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + dynamodbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/cloudspannerecosystem/dynamodb-adapter/models" + "github.com/cloudspannerecosystem/dynamodb-adapter/utils" + "google.golang.org/genproto/googleapis/spanner/admin/database/v1" + "gopkg.in/yaml.v3" +) + +// Define a global variable for reading files (mockable for tests) +var readFile = os.ReadFile + +// DDL statement to create the DynamoDB adapter table in Spanner +var ( + adapterTableDDL = ` + CREATE TABLE dynamodb_adapter_table_ddl ( + column STRING(MAX) NOT NULL, + tableName STRING(MAX) NOT NULL, + dynamoDataType STRING(MAX) NOT NULL, + originalColumn STRING(MAX) NOT NULL, + partitionKey STRING(MAX), + sortKey STRING(MAX), + spannerIndexName STRING(MAX), + actualTable STRING(MAX), + spannerDataType STRING(MAX) + ) PRIMARY KEY (tableName, column);` +) + +// Entry point for the application +func main() { + // Parse command-line arguments for dry-run mode + dryRun := flag.Bool("dry_run", false, "Run the program in dry-run mode to output DDL and queries without making changes") + flag.Parse() + + // Load configuration from a YAML file + config, err := loadConfig("config.yaml") + if err != nil { + log.Fatalf("Error loading configuration: %v", err) + } + + // Build the Spanner database name + databaseName := fmt.Sprintf( + "projects/%s/instances/%s/databases/%s", + config.Spanner.ProjectID, config.Spanner.InstanceID, config.Spanner.DatabaseName, + ) + ctx := context.Background() + adminClient, err := Admindatabase.NewDatabaseAdminClient(ctx) + if err != nil { + log.Fatalf("Failed to create Spanner Admin client: %v", err) + } + defer adminClient.Close() + // Decide execution mode based on the dry-run flag + if *dryRun { + fmt.Println("-- Dry Run Mode: Generating Spanner DDL and Insert Queries Only --") + runDryRun(config) + } else { + fmt.Println("-- Executing Setup on Spanner --") + executeSetup(ctx, adminClient, databaseName, config) + } +} + +// Load configuration from a YAML file +func loadConfig(filename string) (*models.Config, error) { + data, err := readFile(filename) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + var config models.Config + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal config: %w", err) + } + + return &config, nil +} + +// Run in dry-run mode to output DDL and insert queries without making changes +func runDryRun(config *models.Config) { + fmt.Println("-- Spanner DDL to create the adapter table --") + fmt.Println(adapterTableDDL + ";") + + client := createDynamoClient() + tables, err := listDynamoTables(client) + if err != nil { + log.Fatalf("Failed to list DynamoDB tables: %v", err) + } + + // Process each DynamoDB table + for _, tableName := range tables { + fmt.Printf("-- Processing table: %s\n", tableName) + + // Generate and print table-specific DDL + ddl := generateTableDDL(tableName, client, config.Spanner.DynamoQueryLimit) + fmt.Printf("-- DDL for table: %s --\n%s\n", tableName, ddl+";") + + // Generate and print insert queries + generateInsertQueries(tableName, client, config.Spanner.DynamoQueryLimit) + } +} + +// Generate DDL statement for a specific DynamoDB table +func generateTableDDL(tableName string, client *dynamodb.Client, limit int32) string { + attributes, partitionKey, sortKey, err := fetchTableAttributes(client, tableName, limit) + if err != nil { + log.Printf("Failed to fetch attributes for table %s: %v", tableName, err) + return "" + } + + var columns []string + for column, dataType := range attributes { + columns = append(columns, fmt.Sprintf("%s %s", column, utils.ConvertDynamoTypeToSpannerType(dataType))) + } + primaryKey := fmt.Sprintf("PRIMARY KEY (%s%s)", partitionKey, func() string { + if sortKey != "" { + return ", " + sortKey + } + return "" + }()) + + return fmt.Sprintf( + "CREATE TABLE %s (\n\t%s\n) %s;", + tableName, strings.Join(columns, ",\n\t"), primaryKey, + ) +} + +// Generate insert queries for a given DynamoDB table +func generateInsertQueries(tableName string, client *dynamodb.Client, limit int32) { + attributes, partitionKey, sortKey, err := fetchTableAttributes(client, tableName, limit) + if err != nil { + log.Printf("Failed to fetch attributes for table %s: %v", tableName, err) + return + } + + for column, dataType := range attributes { + spannerDataType := utils.ConvertDynamoTypeToSpannerType(dataType) + query := fmt.Sprintf( + `INSERT INTO dynamodb_adapter_table_ddl + (column, tableName, dynamoDataType, originalColumn, partitionKey, sortKey, spannerIndexName, actualTable, spannerDataType) + VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s');`, + column, tableName, dataType, column, partitionKey, sortKey, column, tableName, spannerDataType, + ) + fmt.Println(query) + } +} + +// Execute the setup process: create database, tables, and migrate data +func executeSetup(ctx context.Context, adminClient *Admindatabase.DatabaseAdminClient, databaseName string, config *models.Config) { + + // Create the Spanner database if it doesn't exist + if err := createDatabase(ctx, adminClient, databaseName); err != nil { + log.Fatalf("Failed to create database: %v", err) + } + + // Create the adapter table + if err := createTable(ctx, adminClient, databaseName, adapterTableDDL); err != nil { + log.Fatalf("Failed to create adapter table: %v", err) + } + + // Process each DynamoDB table + client := createDynamoClient() + tables, err := listDynamoTables(client) + if err != nil { + log.Fatalf("Failed to list DynamoDB tables: %v", err) + } + + for _, tableName := range tables { + // Generate and apply table-specific DDL + ddl := generateTableDDL(tableName, client, config.Spanner.DynamoQueryLimit) + if err := createTable(ctx, adminClient, databaseName, ddl); err != nil { + log.Printf("Failed to create table %s: %v", tableName, err) + continue + } + + // Migrate table metadata to Spanner + err := migrateDynamoTableToSpanner(ctx, databaseName, tableName, client, config) + if err != nil { + log.Printf("Error migrating table %s: %v", tableName, err) + } + } + + fmt.Println("Initial setup complete.") +} + +// migrateDynamoTableToSpanner migrates a DynamoDB table schema and metadata to Spanner. +func migrateDynamoTableToSpanner(ctx context.Context, db, tableName string, client *dynamodb.Client, config *models.Config) error { + models.SpannerTableMap[tableName] = config.Spanner.InstanceID + + // Fetch table attributes and keys from DynamoDB + attributes, partitionKey, sortKey, err := fetchTableAttributes(client, tableName, int32(config.Spanner.DynamoQueryLimit)) + if err != nil { + return fmt.Errorf("failed to fetch attributes for table %s: %v", tableName, err) + } + + // Fetch the current Spanner schema for the table + spannerSchema, err := fetchSpannerSchema(ctx, db, tableName) + if err != nil { + return fmt.Errorf("failed to fetch Spanner schema for table %s: %v", tableName, err) + } + + // Generate and apply DDL statements for missing columns + var ddlStatements []string + for column, dynamoType := range attributes { + if _, exists := spannerSchema[column]; !exists { + spannerType := utils.ConvertDynamoTypeToSpannerType(dynamoType) + ddlStatements = append(ddlStatements, fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", tableName, column, spannerType)) + } + } + if len(ddlStatements) > 0 { + if err := applySpannerDDL(ctx, db, ddlStatements); err != nil { + return fmt.Errorf("failed to apply DDL to table %s: %v", tableName, err) + } + log.Printf("Schema updated for table %s in Spanner.", tableName) + } + + // Check for columns that are in Spanner but not in DynamoDB (columns that should be dropped) + var dropColumnStatements []string + for column := range spannerSchema { + if _, exists := attributes[column]; !exists { + dropColumnStatements = append(dropColumnStatements, fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", tableName, column)) + } + } + + // Apply DDL to drop removed columns + if len(dropColumnStatements) > 0 { + if err := applySpannerDDL(ctx, db, dropColumnStatements); err != nil { + return fmt.Errorf("failed to apply DROP COLUMN DDL to table %s: %v", tableName, err) + } + log.Printf("Removed columns from table %s in Spanner.", tableName) + } + + // Prepare mutations to insert metadata into the adapter table + var mutations []*spanner.Mutation + for column, dataType := range attributes { + spannerDataType := utils.ConvertDynamoTypeToSpannerType(dataType) + mutations = append(mutations, spanner.InsertOrUpdate( + "dynamodb_adapter_table_ddl", + []string{"column", "tableName", "dynamoDataType", "originalColumn", "partitionKey", "sortKey", "spannerIndexName", "actualTable", "spannerDataType"}, + []interface{}{column, tableName, dataType, column, partitionKey, sortKey, column, tableName, spannerDataType}, + )) + } + + // Perform batch insert into Spanner + if err := spannerBatchInsert(ctx, db, mutations); err != nil { + return fmt.Errorf("failed to insert metadata for table %s into Spanner: %v", tableName, err) + } + + log.Printf("Successfully migrated metadata for table %s to Spanner.", tableName) + return nil +} + +// createDatabase creates a new Spanner database if it does not exist. +func createDatabase(ctx context.Context, adminClient *Admindatabase.DatabaseAdminClient, db string) error { + // Parse database ID + matches := regexp.MustCompile("^(.*)/databases/(.*)$").FindStringSubmatch(db) + if matches == nil || len(matches) != 3 { + return fmt.Errorf("invalid database ID: %s", db) + } + parent, dbName := matches[1], matches[2] + + // Initiate database creation + op, err := adminClient.CreateDatabase(ctx, &database.CreateDatabaseRequest{ + Parent: parent, + CreateStatement: "CREATE DATABASE `" + dbName + "`", + }) + if err != nil { + if strings.Contains(err.Error(), "AlreadyExists") { + log.Printf("Database `%s` already exists. Skipping creation.", dbName) + return nil + } + return fmt.Errorf("failed to initiate database creation: %v", err) + } + + // Wait for database creation to complete + if _, err = op.Wait(ctx); err != nil { + return fmt.Errorf("error while waiting for database creation: %v", err) + } + log.Printf("Database `%s` created successfully.", dbName) + return nil +} + +// createTable creates a table in Spanner if it does not already exist. +func createTable(ctx context.Context, adminClient *Admindatabase.DatabaseAdminClient, db, ddl string) error { + // Extract table name from DDL + re := regexp.MustCompile(`CREATE TABLE (\w+)`) + matches := re.FindStringSubmatch(ddl) + if len(matches) < 2 { + return fmt.Errorf("unable to extract table name from DDL: %s", ddl) + } + tableName := matches[1] + + // Create Spanner client + client, err := spanner.NewClient(ctx, db) + if err != nil { + return fmt.Errorf("failed to create Spanner client: %v", err) + } + defer client.Close() + + // Check if the table already exists + stmt := spanner.Statement{ + SQL: `SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = @tableName`, + Params: map[string]interface{}{ + "tableName": tableName, + }, + } + iter := client.Single().Query(ctx, stmt) + defer iter.Stop() + + var tableCount int64 + err = iter.Do(func(row *spanner.Row) error { + return row.Columns(&tableCount) + }) + if err != nil { + return fmt.Errorf("failed to query table existence: %w", err) + } + if tableCount > 0 { + log.Printf("Table `%s` already exists. Skipping creation.", tableName) + return nil + } + + // Create the table + op, err := adminClient.UpdateDatabaseDdl(ctx, &database.UpdateDatabaseDdlRequest{ + Database: db, + Statements: []string{ddl}, + }) + if err != nil { + return fmt.Errorf("failed to create table: %w", err) + } + return op.Wait(ctx) +} + +func listDynamoTables(client *dynamodb.Client) ([]string, error) { + output, err := client.ListTables(context.TODO(), &dynamodb.ListTablesInput{}) + if err != nil { + return nil, err + } + return output.TableNames, nil +} + +// fetchTableAttributes retrieves attributes and key schema (partition and sort keys) of a DynamoDB table. +// It describes the table to get its key schema and scans the table to infer attribute types. +func fetchTableAttributes(client *dynamodb.Client, tableName string, limit int32) (map[string]string, string, string, error) { + // Describe the DynamoDB table to get its key schema and attributes. + output, err := client.DescribeTable(context.TODO(), &dynamodb.DescribeTableInput{ + TableName: aws.String(tableName), + }) + if err != nil { + return nil, "", "", fmt.Errorf("failed to describe table %s: %w", tableName, err) + } + + var partitionKey, sortKey string + // Extract the partition key and sort key from the table's key schema. + for _, keyElement := range output.Table.KeySchema { + switch keyElement.KeyType { + case dynamodbtypes.KeyTypeHash: + partitionKey = aws.ToString(keyElement.AttributeName) // Partition key + case dynamodbtypes.KeyTypeRange: + sortKey = aws.ToString(keyElement.AttributeName) // Sort key + } + } + + // Map to store inferred attribute types. + attributes := make(map[string]string) + + // Scan the table to retrieve data and infer attribute types. + scanOutput, err := client.Scan(context.TODO(), &dynamodb.ScanInput{ + TableName: aws.String(tableName), + Limit: aws.Int32(limit), + }) + if err != nil { + return nil, "", "", fmt.Errorf("failed to scan table %s: %w", tableName, err) + } + + // Iterate through the items and infer the attribute types. + for _, item := range scanOutput.Items { + for attr, value := range item { + attributes[attr] = inferDynamoDBType(value) + } + } + + return attributes, partitionKey, sortKey, nil +} + +// inferDynamoDBType determines the type of a DynamoDB attribute based on its value. +func inferDynamoDBType(attr dynamodbtypes.AttributeValue) string { + // Check the attribute type and return the corresponding DynamoDB type. + switch attr.(type) { + case *dynamodbtypes.AttributeValueMemberS: + return "S" // String type + case *dynamodbtypes.AttributeValueMemberN: + return "N" // Number type + case *dynamodbtypes.AttributeValueMemberB: + return "B" // Binary type + case *dynamodbtypes.AttributeValueMemberBOOL: + return "BOOL" // Boolean type + case *dynamodbtypes.AttributeValueMemberSS: + return "SS" // String Set type + case *dynamodbtypes.AttributeValueMemberNS: + return "NS" // Number Set type + case *dynamodbtypes.AttributeValueMemberBS: + return "BS" // Binary Set type + case *dynamodbtypes.AttributeValueMemberNULL: + return "NULL" // Null type + case *dynamodbtypes.AttributeValueMemberM: + return "M" // Map type + case *dynamodbtypes.AttributeValueMemberL: + return "L" // List type + default: + log.Printf("Unknown DynamoDB attribute type: %T\n", attr) + return "Unknown" // Unknown type + } +} + +// spannerBatchInsert applies a batch of mutations to a Spanner database. +func spannerBatchInsert(ctx context.Context, databaseName string, mutations []*spanner.Mutation) error { + // Create a Spanner client. + client, err := spanner.NewClient(ctx, databaseName) + if err != nil { + return fmt.Errorf("failed to create Spanner client: %w", err) + } + defer client.Close() // Ensure the client is closed after the operation. + + // Apply the batch of mutations to the database. + _, err = client.Apply(ctx, mutations) + return err +} + +// createDynamoClient initializes a DynamoDB client using default AWS configuration. +func createDynamoClient() *dynamodb.Client { + // Load the default AWS configuration. + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + log.Fatalf("failed to load AWS config: %v", err) + } + return dynamodb.NewFromConfig(cfg) // Return the configured client. +} + +// fetchSpannerSchema retrieves the schema of a Spanner table by querying the INFORMATION_SCHEMA. +func fetchSpannerSchema(ctx context.Context, db, tableName string) (map[string]string, error) { + // Create a Spanner client. + client, err := spanner.NewClient(ctx, db) + if err != nil { + return nil, fmt.Errorf("failed to create Spanner client: %v", err) + } + defer client.Close() // Ensure the client is closed after the operation. + + // Query the schema information for the specified table. + stmt := spanner.Statement{ + SQL: `SELECT COLUMN_NAME, SPANNER_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = @tableName`, + Params: map[string]interface{}{ + "tableName": tableName, + }, + } + iter := client.Single().Query(ctx, stmt) + defer iter.Stop() // Ensure the iterator is stopped after use. + + // Map to store the schema information. + schema := make(map[string]string) + err = iter.Do(func(row *spanner.Row) error { + var columnName, spannerType string + // Extract column name and type from the row. + if err := row.Columns(&columnName, &spannerType); err != nil { + return err + } + schema[columnName] = spannerType + return nil + }) + if err != nil { + return nil, err + } + return schema, nil +} + +// applySpannerDDL executes DDL statements on a Spanner database. +func applySpannerDDL(ctx context.Context, db string, ddlStatements []string) error { + // Create a Spanner Admin client. + adminClient, err := Admindatabase.NewDatabaseAdminClient(ctx) + if err != nil { + return fmt.Errorf("failed to create Spanner Admin client: %v", err) + } + defer adminClient.Close() // Ensure the client is closed after the operation. + + // Initiate the DDL update operation. + op, err := adminClient.UpdateDatabaseDdl(ctx, &database.UpdateDatabaseDdlRequest{ + Database: db, + Statements: ddlStatements, + }) + if err != nil { + return fmt.Errorf("failed to initiate DDL update: %v", err) + } + + // Wait for the DDL update operation to complete. + if err := op.Wait(ctx); err != nil { + return fmt.Errorf("error while waiting for DDL update to complete: %v", err) + } + return nil +} diff --git a/config-files/production/config.json b/config-files/production/config.json deleted file mode 100644 index 1ca5ab9..0000000 --- a/config-files/production/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "GoogleProjectID": "Your Google Project ID", - "SpannerDb": "Your Spanner Database Name", - "QueryLimit": 5000 -} \ No newline at end of file diff --git a/config-files/production/spanner.json b/config-files/production/spanner.json deleted file mode 100644 index 397cc31..0000000 --- a/config-files/production/spanner.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dynamodb_adapter_table_ddl": "instance-ID of dynamodb_adapter_table_ddl table", - "dynamodb_adapter_config_manager": "instance-ID of dynamodb_adapter_config_manager table", - "tableName": "instance-ID of Table" -} \ No newline at end of file diff --git a/config-files/production/tables.json b/config-files/production/tables.json deleted file mode 100644 index c56cbf5..0000000 --- a/config-files/production/tables.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "tableName":{ - "partitionKey":"primary key or Partition key", - "sortKey": "sorting key of dynamoDB adapter", - "attributeTypes": { - "ColInt64": "N", - "ColString": "S", - "ColBytes": "B", - "ColBool": "BOOL", - "ColDate": "S", - "ColTimestamp": "S" - }, - "indices": { - "indexName1": { - "sortKey": "sort key for indexName1", - "partitionKey": "partition key for indexName1" - } - } - } -} \ No newline at end of file diff --git a/config-files/staging/config.json b/config-files/staging/config.json deleted file mode 100644 index 1ca5ab9..0000000 --- a/config-files/staging/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "GoogleProjectID": "Your Google Project ID", - "SpannerDb": "Your Spanner Database Name", - "QueryLimit": 5000 -} \ No newline at end of file diff --git a/config-files/staging/spanner.json b/config-files/staging/spanner.json deleted file mode 100644 index 397cc31..0000000 --- a/config-files/staging/spanner.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dynamodb_adapter_table_ddl": "instance-ID of dynamodb_adapter_table_ddl table", - "dynamodb_adapter_config_manager": "instance-ID of dynamodb_adapter_config_manager table", - "tableName": "instance-ID of Table" -} \ No newline at end of file diff --git a/config-files/staging/tables.json b/config-files/staging/tables.json deleted file mode 100644 index 99d015f..0000000 --- a/config-files/staging/tables.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "tableName":{ - "partitionKey":"primary key or Partition key", - "sortKey": "sorting key of dynamoDB adapter", - "attributeTypes": { - "ColInt64": "N", - "ColString": "S", - "ColBytes": "B", - "ColBool": "BOOL", - "ColDate": "S", - "ColTimestamp": "S" - }, - "indices": { - "indexName1": { - "sortKey": "sort key for indexName1", - "partitionKey": "partition key for indexName1" - } - } - } -} \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..9c7785d --- /dev/null +++ b/config.yaml @@ -0,0 +1,6 @@ +spanner: + project_id: ${PROJECT_ID} + instance_id: ${INSTANCE_ID} + database_name: ${DATABASE_ID} + query_limit: ${QUERY_LIMIT} + dynamo_query_limit: ${DYNAMODB_QUERY_LIMIT} \ No newline at end of file diff --git a/config/config.go b/config/config.go index 8b8fbd0..2cda7bc 100644 --- a/config/config.go +++ b/config/config.go @@ -17,16 +17,13 @@ package config import ( - "encoding/json" + "fmt" + "log" "os" - "strings" - "sync" - rice "github.com/GeertJohan/go.rice" "github.com/cloudspannerecosystem/dynamodb-adapter/models" "github.com/cloudspannerecosystem/dynamodb-adapter/pkg/errors" - "github.com/cloudspannerecosystem/dynamodb-adapter/pkg/logger" - "github.com/cloudspannerecosystem/dynamodb-adapter/utils" + "gopkg.in/yaml.v2" ) // Configuration struct @@ -39,63 +36,37 @@ type Configuration struct { // ConfigurationMap pointer var ConfigurationMap *Configuration -// DbConfigMap dynamo to Spanner -var DbConfigMap map[string]models.TableConfig - -var once sync.Once - func init() { ConfigurationMap = new(Configuration) } -// InitConfig loads ConfigurationMap and DbConfigMap in memory based on -// ACTIVE_ENV. If ACTIVE_ENV is not set or and empty string the environment -// is defaulted to staging. -// -// These config files are read from rice-box -func InitConfig(box *rice.Box) { - once.Do(func() { - env := strings.ToLower(os.Getenv("ACTIVE_ENV")) - if env == "" { - env = "staging" - } +var readFile = os.ReadFile - ConfigurationMap = new(Configuration) +func InitConfig(filepath string) { + GlobalConfig, err := loadConfig(filepath) + if err != nil { + log.Printf("failed to read config file: %v", err) + } + models.GlobalConfig = GlobalConfig +} + +func loadConfig(filename string) (*models.Config, error) { + data, err := readFile(filename) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + // Unmarshal YAML data into config struct + var config models.Config + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal config: %w", err) + } - ba, err := box.Bytes(env + "/tables.json") - if err != nil { - logger.LogFatal(err) - } - err = json.Unmarshal(ba, &DbConfigMap) - if err != nil { - logger.LogFatal(err) - } - ba, err = box.Bytes(env + "/config.json") - if err != nil { - logger.LogFatal(err) - } - err = json.Unmarshal(ba, ConfigurationMap) - if err != nil { - logger.LogFatal(err) - } - ba, err = box.Bytes(env + "/spanner.json") - if err != nil { - logger.LogFatal(err) - } - tmp := make(map[string]string) - err = json.Unmarshal(ba, &tmp) - if err != nil { - logger.LogFatal(err) - } - for k, v := range tmp { - models.SpannerTableMap[utils.ChangeTableNameForSpanner(k)] = v - } - }) + return &config, nil } -//GetTableConf returns table configuration from global map object +// GetTableConf returns table configuration from global map object func GetTableConf(tableName string) (models.TableConfig, error) { - tableConf, ok := DbConfigMap[tableName] + tableConf, ok := models.DbConfigMap[tableName] if !ok { return models.TableConfig{}, errors.New("ResourceNotFoundException", tableName) } @@ -104,7 +75,7 @@ func GetTableConf(tableName string) (models.TableConfig, error) { return tableConf, nil } else if tableConf.ActualTable != "" { actualTable := tableConf.ActualTable - tableConf = DbConfigMap[actualTable] + tableConf = models.DbConfigMap[actualTable] tableConf.ActualTable = actualTable return tableConf, nil } diff --git a/config/config_test.go b/config/config_test.go index 2be07e0..7bfc3ba 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -22,7 +22,7 @@ import ( ) func TestGetTableConf(t *testing.T) { - DbConfigMap = map[string]models.TableConfig{ + models.DbConfigMap = map[string]models.TableConfig{ "employee_data": { PartitionKey: "emp_id", SortKey: "emp_name", diff --git a/go.mod b/go.mod index fcd2080..6eaed2b 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/ajg/form v1.5.1 // indirect github.com/andybalholm/brotli v1.0.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.5 github.com/daaku/go.zipexe v1.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -84,7 +85,30 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/grpc v1.40.0 // indirect google.golang.org/protobuf v1.27.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e // indirect ) + +require ( + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.37.1 + github.com/aws/smithy-go v1.22.1 // indirect +) + +require ( + github.com/aws/aws-sdk-go-v2/config v1.28.5 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.5 // indirect +) + +require ( + github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect +) diff --git a/go.sum b/go.sum index f40d8d6..b56adfb 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,36 @@ github.com/antonmedv/expr v1.9.0 h1:j4HI3NHEdgDnN9p6oI6Ndr0G5QryMY0FNxT4ONrFDGU= github.com/antonmedv/expr v1.9.0/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= github.com/aws/aws-sdk-go v1.40.43 h1:froMtO2//9kCu1sK+dOfAcwxUu91p5KgUP4AL7SDwUQ= github.com/aws/aws-sdk-go v1.40.43/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= +github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= +github.com/aws/aws-sdk-go-v2/config v1.28.5/go.mod h1:4VsPbHP8JdcdUDmbTVgNL/8w9SqOkM5jyY8ljIxLO3o= +github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgbz1E7Mm95ieIRDNUg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.37.1 h1:vucMirlM6D+RDU8ncKaSZ/5dGrXNajozVwpmWNPn2gQ= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.37.1/go.mod h1:fceORfs010mNxZbQhfqUjUeHlTwANmIT4mvHamuUaUg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.5 h1:3Y457U2eGukmjYjeHG6kanZpDzJADa2m0ADqnuePYVQ= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.5/go.mod h1:CfwEHGkTjYZpkQ/5PvcbEtT7AJlG68KkEvmtwU8z3/U= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.6/go.mod h1:WJSZH2ZvepM6t6jwu4w/Z45Eoi75lPN7DcydSRtJg6Y= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3yZp/S9OaoLU33S7vPS8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/initializer/initializer.go b/initializer/initializer.go index 7f99035..b462510 100644 --- a/initializer/initializer.go +++ b/initializer/initializer.go @@ -17,7 +17,6 @@ package initializer import ( - rice "github.com/GeertJohan/go.rice" "github.com/cloudspannerecosystem/dynamodb-adapter/config" "github.com/cloudspannerecosystem/dynamodb-adapter/service/services" "github.com/cloudspannerecosystem/dynamodb-adapter/service/spanner" @@ -26,8 +25,8 @@ import ( // InitAll - this will initialize all the project object // Config, storage and all other global objects are initialize -func InitAll(box *rice.Box) error { - config.InitConfig(box) +func InitAll(filepath string) error { + config.InitConfig(filepath) storage.InitializeDriver() err := spanner.ParseDDL(true) if err != nil { diff --git a/integrationtest/README.md b/integrationtest/README.md index baaced6..df2603c 100644 --- a/integrationtest/README.md +++ b/integrationtest/README.md @@ -1,64 +1,30 @@ # Integration Tests -Running the integration tests will require the files present in the -[staging](./config-files/staging) folder to be configured as below: +## config.yaml -## Config Files +This file defines the necessary settings for the adapter. +A sample configuration might look like this: -`config-files/staging/config.json` +spanner: + project_id: "my-project-id" + instance_id: "my-instance-id" + database_name: "my-database-name" + query_limit: "query_limit" + dynamo_query_limit: "dynamo_query_limit" -```json -{ - "GoogleProjectID": "", - "SpannerDb": "", - "QueryLimit": 5000 -} -``` - -`config-files/staging/spanner.json` - -```json -{ - "dynamodb_adapter_table_ddl": "", - "dynamodb_adapter_config_manager": "", - "department": "", - "employee": "" -} -``` - -`config-files/staging/tables.json` - -```json -{ - "employee": { - "partitionKey": "emp_id", - "sortKey": "", - "attributeTypes": { - "emp_id": "N", - "first_name": "S", - "last_name": "S", - "address": "S", - "age": "N" - }, - "indices": {} - }, - "department": { - "partitionKey": "d_id", - "sortKey": "", - "attributeTypes": { - "d_id": "N", - "d_name": "S", - "d_specialization": "S" - }, - "indices": {} - } -} -``` +The fields are: +project_id: The Google Cloud project ID. +instance_id: The Spanner instance ID. +database_name: The database name in Spanner. +query_limit: Database query limit. +dynamo_query_limit: DynamoDb query limit. ## Execute tests ```sh + go run integrationtest/setup.go setup go test integrationtest/api_test.go go run integrationtest/setup.go teardown + ``` diff --git a/integrationtest/api_test.go b/integrationtest/api_test.go index 136debe..798a820 100644 --- a/integrationtest/api_test.go +++ b/integrationtest/api_test.go @@ -16,26 +16,26 @@ package main import ( "context" - "encoding/json" "fmt" "log" "net/http" + "os" "testing" - rice "github.com/GeertJohan/go.rice" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/cloudspannerecosystem/dynamodb-adapter/api" "github.com/cloudspannerecosystem/dynamodb-adapter/apitesting" - "github.com/cloudspannerecosystem/dynamodb-adapter/config" "github.com/cloudspannerecosystem/dynamodb-adapter/initializer" "github.com/cloudspannerecosystem/dynamodb-adapter/models" httpexpect "github.com/gavv/httpexpect/v2" "github.com/gin-gonic/gin" + "gopkg.in/yaml.v2" ) // database name used in all the test cases var databaseName string +var readConfigFile = os.ReadFile // params for TestGetItemAPI var ( @@ -471,7 +471,7 @@ var ( queryTestCaseOutput16 = `{"Count":1,"Items":[]}` ) -//Test Data for Scan API +// Test Data for Scan API var ( ScanTestCase1Name = "1: Wrong URL" ScanTestCase1 = models.ScanMeta{ @@ -589,7 +589,7 @@ var ( ScanTestCase13Output = `{"Count":5,"Items":[]}` ) -//Test Data for UpdateItem API +// Test Data for UpdateItem API var ( //200 Status check @@ -728,7 +728,7 @@ var ( } ) -//Test Data for PutItem API +// Test Data for PutItem API var ( //400 bad request PutItemTestCase1Name = "1: only tablename passed" @@ -839,7 +839,7 @@ var ( } ) -//Test Data DeleteItem API +// Test Data DeleteItem API var ( DeleteItemTestCase1Name = "1: Only TableName passed" DeleteItemTestCase1 = models.Delete{ @@ -926,7 +926,7 @@ var ( } ) -//test Data for BatchWriteItem API +// test Data for BatchWriteItem API var ( BatchWriteItemTestCase1Name = "1: Only Table name passed" BatchWriteItemTestCase1 = models.BatchWriteItem{ @@ -1307,9 +1307,7 @@ var ( ) func handlerInitFunc() *gin.Engine { - box := rice.MustFindBox("../config-files") - - initErr := initializer.InitAll(box) + initErr := initializer.InitAll("../config.yaml") if initErr != nil { log.Fatalln(initErr) } @@ -1366,32 +1364,18 @@ func createStatusCheckPostTestCase(name, url, dynamoAction string, httpStatus in } } -func init() { - box := rice.MustFindBox("../config-files") - - // read the config variables - ba, err := box.Bytes("staging/config.json") +func LoadConfig(filename string) (*models.Config, error) { + data, err := readConfigFile(filename) if err != nil { - log.Fatal("error reading staging config json: ", err.Error()) - } - var conf = &config.Configuration{} - if err = json.Unmarshal(ba, &conf); err != nil { - log.Fatal(err) + return nil, fmt.Errorf("failed to read config file: %w", err) } - // read the spanner table configurations - var m = make(map[string]string) - ba, err = box.Bytes("staging/spanner.json") - if err != nil { - log.Fatal("error reading spanner config json: ", err.Error()) - } - if err = json.Unmarshal(ba, &m); err != nil { - log.Fatal(err) + var config models.Config + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal config: %w", err) } - databaseName = fmt.Sprintf( - "projects/%s/instances/%s/databases/%s", conf.GoogleProjectID, m["dynamodb_adapter_table_ddl"], conf.SpannerDb, - ) + return &config, nil } func testGetItemAPI(t *testing.T) { @@ -1743,6 +1727,16 @@ func TestApi(t *testing.T) { t.Skip("skipping integration tests in short mode") } + config, err := LoadConfig("../config.yaml") + if err != nil { + log.Fatalf("Error loading configuration: %v", err) + } + // Build the Spanner database name + databaseName = fmt.Sprintf( + "projects/%s/instances/%s/databases/%s", + config.Spanner.ProjectID, config.Spanner.InstanceID, config.Spanner.DatabaseName, + ) + // this is done to maintain the order of the test cases var testNames = []string{ "GetItemAPI", diff --git a/integrationtest/setup.go b/integrationtest/setup.go index 5fe94b9..cee0268 100644 --- a/integrationtest/setup.go +++ b/integrationtest/setup.go @@ -16,80 +16,38 @@ package main import ( "context" - "encoding/json" - "errors" "fmt" "io" "log" "os" "regexp" - "strings" "cloud.google.com/go/spanner" database "cloud.google.com/go/spanner/admin/database/apiv1" - rice "github.com/GeertJohan/go.rice" - "github.com/cloudspannerecosystem/dynamodb-adapter/config" - "google.golang.org/api/iterator" + "github.com/cloudspannerecosystem/dynamodb-adapter/models" adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1" + "gopkg.in/yaml.v2" ) -const ( - expectedRowCount = 18 -) - -var ( - colNameRg = regexp.MustCompile("^[a-zA-Z0-9_]*$") - chars = []string{"]", "^", "\\\\", "/", "[", ".", "(", ")", "-"} - ss = strings.Join(chars, "") - specialCharRg = regexp.MustCompile("[" + ss + "]+") -) +var readFile = os.ReadFile func main() { - box := rice.MustFindBox("../config-files") - - // read the config variables - ba, err := box.Bytes("staging/config.json") - if err != nil { - log.Fatal("error reading staging config json: ", err.Error()) - } - var conf = &config.Configuration{} - if err = json.Unmarshal(ba, &conf); err != nil { - log.Fatal(err) - } - - // read the spanner table configurations - var m = make(map[string]string) - ba, err = box.Bytes("staging/spanner.json") + config, err := loadConfig("config.yaml") if err != nil { - log.Fatal("error reading spanner config json: ", err.Error()) - } - if err = json.Unmarshal(ba, &m); err != nil { - log.Fatal(err) + log.Fatalf("Error loading configuration: %v", err) } - var databaseName = fmt.Sprintf( - "projects/%s/instances/%s/databases/%s", conf.GoogleProjectID, m["dynamodb_adapter_table_ddl"], conf.SpannerDb, + // Build the Spanner database name + databaseName := fmt.Sprintf( + "projects/%s/instances/%s/databases/%s", + config.Spanner.ProjectID, config.Spanner.InstanceID, config.Spanner.DatabaseName, ) - switch cmd := os.Args[1]; cmd { case "setup": w := log.Writer() if err := createDatabase(w, databaseName); err != nil { log.Fatal(err) } - - if err := updateDynamodbAdapterTableDDL(w, databaseName); err != nil { - log.Fatal(err) - } - - count, err := verifySpannerSetup(databaseName) - if err != nil { - log.Fatal(err) - } - if count != expectedRowCount { - log.Fatalf("Rows found: %d, exepected %d\n", count, expectedRowCount) - } - if err := initData(w, databaseName); err != nil { log.Fatal(err) } @@ -103,10 +61,24 @@ func main() { } } +func loadConfig(filename string) (*models.Config, error) { + data, err := readFile(filename) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + var config models.Config + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal config: %w", err) + } + + return &config, nil +} + func createDatabase(w io.Writer, db string) error { matches := regexp.MustCompile("^(.*)/databases/(.*)$").FindStringSubmatch(db) if matches == nil || len(matches) != 3 { - return fmt.Errorf("Invalid database id %s", db) + return fmt.Errorf("invalid database id %s", db) } ctx := context.Background() @@ -121,19 +93,16 @@ func createDatabase(w io.Writer, db string) error { CreateStatement: "CREATE DATABASE `" + matches[2] + "`", ExtraStatements: []string{ `CREATE TABLE dynamodb_adapter_table_ddl ( - column STRING(MAX), - tableName STRING(MAX), - dataType STRING(MAX), - originalColumn STRING(MAX), + tableName STRING(MAX) NOT NULL, + column STRING(MAX) NOT NULL, + dynamoDataType STRING(MAX) NOT NULL, + originalColumn STRING(MAX) NOT NULL, + partitionKey STRING(MAX), + sortKey STRING(MAX), + spannerIndexName STRING(MAX), + actualTable STRING(MAX), + spannerDataType STRING(MAX) ) PRIMARY KEY (tableName, column)`, - `CREATE TABLE dynamodb_adapter_config_manager ( - tableName STRING(MAX), - config STRING(MAX), - cronTime STRING(MAX), - enabledStream STRING(MAX), - pubsubTopic STRING(MAX), - uniqueValue STRING(MAX), - ) PRIMARY KEY (tableName)`, `CREATE TABLE employee ( emp_id FLOAT64, address STRING(MAX), @@ -173,125 +142,36 @@ func deleteDatabase(w io.Writer, db string) error { return nil } -func updateDynamodbAdapterTableDDL(w io.Writer, db string) error { - stmt, err := readDatabaseSchema(db) - if err != nil { - return err - } - - var mutations []*spanner.Mutation - for i := 0; i < len(stmt); i++ { - tokens := strings.Split(stmt[i], "\n") - if len(tokens) == 1 { - continue - } - var currentTable, colName, colType, originalColumn string - - for j := 0; j < len(tokens); j++ { - if strings.Contains(tokens[j], "PRIMARY KEY") { - continue - } - if strings.Contains(tokens[j], "CREATE TABLE") { - currentTable = getTableName(tokens[j]) - continue - } - colName, colType = getColNameAndType(tokens[j]) - originalColumn = colName - - if !colNameRg.MatchString(colName) { - colName = specialCharRg.ReplaceAllString(colName, "_") - } - colType = strings.Replace(colType, ",", "", 1) - var mut = spanner.InsertOrUpdateMap( - "dynamodb_adapter_table_ddl", - map[string]interface{}{ - "tableName": currentTable, - "column": colName, - "dataType": colType, - "originalColumn": originalColumn, - }, - ) - fmt.Fprintf(w, "[%s, %s, %s, %s]\n", currentTable, colName, colType, originalColumn) - mutations = append(mutations, mut) - } - } - - return spannerBatchPut(context.Background(), db, mutations) -} - -func readDatabaseSchema(db string) ([]string, error) { +func initData(w io.Writer, db string) error { ctx := context.Background() - cli, err := database.NewDatabaseAdminClient(ctx) - if err != nil { - return nil, err - } - defer cli.Close() - - ddlResp, err := cli.GetDatabaseDdl(ctx, &adminpb.GetDatabaseDdlRequest{Database: db}) - if err != nil { - return nil, err - } - return ddlResp.GetStatements(), nil -} - -func getTableName(stmt string) string { - tokens := strings.Split(stmt, " ") - return tokens[2] -} - -func getColNameAndType(stmt string) (string, string) { - stmt = strings.TrimSpace(stmt) - tokens := strings.Split(stmt, " ") - tokens[0] = strings.Trim(tokens[0], "`") - return tokens[0], tokens[1] -} - -// spannerBatchPut - this insert or update data in batch -func spannerBatchPut(ctx context.Context, db string, m []*spanner.Mutation) error { client, err := spanner.NewClient(ctx, db) if err != nil { - log.Fatalf("Failed to create client %v", err) return err } defer client.Close() - if _, err = client.Apply(ctx, m); err != nil { - return errors.New("ResourceNotFoundException: " + err.Error()) - } - return nil -} - -func verifySpannerSetup(db string) (int, error) { - ctx := context.Background() - client, err := spanner.NewClient(ctx, db) - if err != nil { - return 0, err - } - defer client.Close() - - var iter = client.Single().Read(ctx, "dynamodb_adapter_table_ddl", spanner.AllKeys(), - []string{"column", "tableName", "dataType", "originalColumn"}) - - var count int - for { - if _, err := iter.Next(); err != nil { - if err == iterator.Done { - break - } - return 0, err + _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { + stmt := spanner.Statement{ + SQL: `INSERT dynamodb_adapter_table_ddl (tableName, column, dynamoDataType, originalColumn, partitionKey,sortKey, spannerIndexName, actualTable, spannerDataType) VALUES + ('employee', 'emp_id', 'N', 'emp_id', 'emp_id','', 'emp_id','employee', 'FLOAT64'), + ('employee', 'address', 'S', 'address', 'emp_id','', 'address','employee', 'STRING(MAX)'), + ('employee', 'age', 'N', 'age', 'emp_id','', 'age','employee', 'FLOAT64'), + ('employee', 'first_name', 'S', 'first_name', 'emp_id','', 'first_name','employee', 'STRING(MAX)'), + ('employee', 'last_name', 'S', 'last_name', 'emp_id','', 'last_name','employee', 'STRING(MAX)'), + ('department', 'd_id', 'N', 'd_id', 'd_id','', 'd_id','department', 'FLOAT64'), + ('department', 'd_name', 'S', 'd_name', 'd_id','', 'd_name','department', 'STRING(MAX)'), + ('department', 'd_specialization', 'S', 'd_specialization', 'd_id','', 'd_specialization','department', 'STRING(MAX)')`, } - count++ - } - return count, nil -} - -func initData(w io.Writer, db string) error { - ctx := context.Background() - client, err := spanner.NewClient(ctx, db) + rowCount, err := txn.Update(ctx, stmt) + if err != nil { + return err + } + fmt.Fprintf(w, "%d record(s) inserted.\n", rowCount) + return err + }) if err != nil { return err } - defer client.Close() _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { stmt := spanner.Statement{ @@ -312,7 +192,7 @@ func initData(w io.Writer, db string) error { if err != nil { return err } - + _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { stmt := spanner.Statement{ SQL: `INSERT department (d_id, d_name, d_specialization) VALUES diff --git a/main.go b/main.go index 74f304e..1630fd7 100644 --- a/main.go +++ b/main.go @@ -18,7 +18,6 @@ import ( "log" "net/http" - rice "github.com/GeertJohan/go.rice" "github.com/cloudspannerecosystem/dynamodb-adapter/api" "github.com/cloudspannerecosystem/dynamodb-adapter/docs" "github.com/cloudspannerecosystem/dynamodb-adapter/initializer" @@ -38,11 +37,7 @@ import ( // @BasePath /v1 func main() { - // This will pack config-files folder inside binary - // you need rice utility for it - box := rice.MustFindBox("config-files") - - initErr := initializer.InitAll(box) + initErr := initializer.InitAll("config.yaml") if initErr != nil { log.Fatalln(initErr) } diff --git a/models/model.go b/models/model.go index ae00328..32ed24b 100644 --- a/models/model.go +++ b/models/model.go @@ -22,6 +22,20 @@ import ( "github.com/aws/aws-sdk-go/service/dynamodb" ) +type SpannerConfig struct { + ProjectID string `yaml:"project_id"` + InstanceID string `yaml:"instance_id"` + DatabaseName string `yaml:"database_name"` + QueryLimit int64 `yaml:"query_limit"` + DynamoQueryLimit int32 `yaml:"dynamo_query_limit"` //dynamo_query_limit +} + +type Config struct { + Spanner SpannerConfig `yaml:"spanner"` +} + +var GlobalConfig *Config + // Meta struct type Meta struct { TableName string `json:"TableName"` @@ -72,7 +86,7 @@ type GetItemMeta struct { Key map[string]*dynamodb.AttributeValue `json:"Key"` } -//BatchGetMeta struct +// BatchGetMeta struct type BatchGetMeta struct { RequestItems map[string]BatchGetWithProjectionMeta `json:"RequestItems"` } @@ -135,7 +149,7 @@ type UpdateAttr struct { ExpressionAttributeValues map[string]*dynamodb.AttributeValue `json:"ExpressionAttributeValues"` } -//ScanMeta for Scan request +// ScanMeta for Scan request type ScanMeta struct { TableName string `json:"TableName"` IndexName string `json:"IndexName"` @@ -158,39 +172,41 @@ type TableConfig struct { Indices map[string]TableConfig `json:"Indices,omitempty"` GCSSourcePath string `json:"GcsSourcePath,omitempty"` DDBIndexName string `json:"DdbIndexName,omitempty"` - SpannerIndexName string `json:"Table,omitempty"` + SpannerIndexName string `json:"SpannerIndexName,omitempty"` IsPadded bool `json:"IsPadded,omitempty"` IsComplement bool `json:"IsComplement,omitempty"` TableSource string `json:"TableSource,omitempty"` ActualTable string `json:"ActualTable,omitempty"` } -//BatchWriteItem for Batch Operation +// BatchWriteItem for Batch Operation type BatchWriteItem struct { RequestItems map[string][]BatchWriteSubItems `json:"RequestItems"` } -//BatchWriteItemResponse for Batch Operation +// BatchWriteItemResponse for Batch Operation type BatchWriteItemResponse struct { UnprocessedItems map[string][]BatchWriteSubItems `json:"UnprocessedItems"` } -//BatchWriteSubItems is for BatchWriteItem +// BatchWriteSubItems is for BatchWriteItem type BatchWriteSubItems struct { DelReq BatchDeleteItem `json:"DeleteRequest"` PutReq BatchPutItem `json:"PutRequest"` } -//BatchDeleteItem is for BatchWriteSubItems +// BatchDeleteItem is for BatchWriteSubItems type BatchDeleteItem struct { Key map[string]*dynamodb.AttributeValue `json:"Key"` } -//BatchPutItem is for BatchWriteSubItems +// BatchPutItem is for BatchWriteSubItems type BatchPutItem struct { Item map[string]*dynamodb.AttributeValue `json:"Item"` } +var DbConfigMap map[string]TableConfig + // TableDDL - this contains the DDL var TableDDL map[string]map[string]string @@ -208,10 +224,10 @@ var OriginalColResponse map[string]string func init() { TableDDL = make(map[string]map[string]string) - TableDDL["dynamodb_adapter_table_ddl"] = map[string]string{"tableName": "STRING(MAX)", "column": "STRING(MAX)", "dataType": "STRING(MAX)", "originalColumn": "STRING(MAX)"} + TableDDL["dynamodb_adapter_table_ddl"] = map[string]string{"tableName": "S", "column": "S", "dynamoDataType": "S", "originalColumn": "S", "partitionKey": "S", "sortKey": "S", "spannerIndexName": "S", "actualTable": "S", "spannerDataType": "S"} TableDDL["dynamodb_adapter_config_manager"] = map[string]string{"tableName": "STRING(MAX)", "config": "STRING(MAX)", "cronTime": "STRING(MAX)", "uniqueValue": "STRING(MAX)", "enabledStream": "STRING(MAX)", "pubsubTopic": "STRING(MAX)"} TableColumnMap = make(map[string][]string) - TableColumnMap["dynamodb_adapter_table_ddl"] = []string{"tableName", "column", "dataType", "originalColumn"} + TableColumnMap["dynamodb_adapter_table_ddl"] = []string{"tableName", "column", "dynamoDataType", "originalColumn", "partitionKey", "sortKey", "spannerIndexName", "actualTable", "spannerDataType"} TableColumnMap["dynamodb_adapter_config_manager"] = []string{"tableName", "config", "cronTime", "uniqueValue", "enabledStream", "pubsubTopic"} TableColChangeMap = make(map[string]struct{}) ColumnToOriginalCol = make(map[string]string) diff --git a/service/services/services.go b/service/services/services.go index 9d41351..1d3625e 100644 --- a/service/services/services.go +++ b/service/services/services.go @@ -539,7 +539,7 @@ func Scan(ctx context.Context, scanData models.ScanMeta) (map[string]interface{} query.TableName = scanData.TableName query.Limit = scanData.Limit if query.Limit == 0 { - query.Limit = config.ConfigurationMap.QueryLimit + query.Limit = models.GlobalConfig.Spanner.QueryLimit } query.StartFrom = scanData.StartFrom query.RangeValMap = scanData.ExpressionAttributeMap diff --git a/service/spanner/spanner.go b/service/spanner/spanner.go index 51df05d..c323ad8 100644 --- a/service/spanner/spanner.go +++ b/service/spanner/spanner.go @@ -16,33 +16,54 @@ package spanner import ( "context" + "strings" + "cloud.google.com/go/spanner" "github.com/cloudspannerecosystem/dynamodb-adapter/models" "github.com/cloudspannerecosystem/dynamodb-adapter/storage" - - "cloud.google.com/go/spanner" ) // ParseDDL - this will parse DDL of spannerDB and set all the table configs in models // This fetches the spanner schema config from dynamodb_adapter_table_ddl table and stored it in // global map object which is used to read and write data into spanner tables -func ParseDDL(updateDB bool) error { +// InitConfig loads ConfigurationMap and DbConfigMap in memory based on +// ACTIVE_ENV. If ACTIVE_ENV is not set or and empty string the environment +// is defaulted to staging. +// +// These config files are read from rice-box + +func ParseDDL(updateDB bool) error { stmt := spanner.Statement{} + stmt.SQL = "SELECT * FROM dynamodb_adapter_table_ddl" - ms, err := storage.GetStorageInstance().ExecuteSpannerQuery(context.Background(), "dynamodb_adapter_table_ddl", []string{"tableName", "column", "dataType", "originalColumn"}, false, stmt) + ms, err := storage.GetStorageInstance().ExecuteSpannerQuery(context.Background(), "dynamodb_adapter_table_ddl", []string{"tableName", "column", "dynamoDataType", "originalColumn", "partitionKey", "sortKey", "spannerIndexName", "actualTable", "spannerDataType"}, false, stmt) + if err != nil { return err } + if models.DbConfigMap == nil { + models.DbConfigMap = make(map[string]models.TableConfig) + } if len(ms) > 0 { for i := 0; i < len(ms); i++ { tableName := ms[i]["tableName"].(string) column := ms[i]["column"].(string) column = strings.Trim(column, "`") - dataType := ms[i]["dataType"].(string) + dataType := ms[i]["dynamoDataType"].(string) originalColumn, ok := ms[i]["originalColumn"].(string) + partitionKey := ms[i]["partitionKey"].(string) + sortKey, _ := ms[i]["sortKey"].(string) // Optional, check if available + spannerIndexName, _ := ms[i]["spannerIndexName"].(string) + models.DbConfigMap[tableName] = models.TableConfig{ + PartitionKey: partitionKey, + SortKey: sortKey, + SpannerIndexName: spannerIndexName, + ActualTable: tableName, + } + if ok { originalColumn = strings.Trim(originalColumn, "`") if column != originalColumn && originalColumn != "" { diff --git a/storage/spanner.go b/storage/spanner.go index 544f857..80e91c0 100755 --- a/storage/spanner.go +++ b/storage/spanner.go @@ -115,11 +115,15 @@ func (s Storage) SpannerGet(ctx context.Context, tableName string, pKeys, sKeys // ExecuteSpannerQuery - this will execute query on spanner database func (s Storage) ExecuteSpannerQuery(ctx context.Context, table string, cols []string, isCountQuery bool, stmt spanner.Statement) ([]map[string]interface{}, error) { + colDLL, ok := models.TableDDL[utils.ChangeTableNameForSpanner(table)] + if !ok { return nil, errors.New("ResourceNotFoundException", table) } + itr := s.getSpannerClient(table).Single().WithTimestampBound(spanner.ExactStaleness(time.Second*10)).Query(ctx, stmt) + defer itr.Stop() allRows := []map[string]interface{}{} for { @@ -146,6 +150,7 @@ func (s Storage) ExecuteSpannerQuery(ctx context.Context, table string, cols []s } allRows = append(allRows, singleRow) } + return allRows, nil } @@ -733,7 +738,7 @@ func evaluateStatementFromRowMap(conditionalExpression, colName string, rowMap m return true } _, ok := rowMap[colName] - return !ok + return !ok } if strings.HasPrefix(conditionalExpression, "attribute_exists") || strings.HasPrefix(conditionalExpression, "if_exists") { if len(rowMap) == 0 { @@ -745,7 +750,7 @@ func evaluateStatementFromRowMap(conditionalExpression, colName string, rowMap m return rowMap[conditionalExpression] } -//parseRow - Converts Spanner row and datatypes to a map removing null columns from the result. +// parseRow - Converts Spanner row and datatypes to a map removing null columns from the result. func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{}, error) { singleRow := make(map[string]interface{}) if r == nil { @@ -762,7 +767,7 @@ func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{}, return nil, errors.New("ResourceNotFoundException", k) } switch v { - case "STRING(MAX)": + case "S": var s spanner.NullString err := r.Column(i, &s) if err != nil { @@ -774,7 +779,7 @@ func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{}, if !s.IsNull() { singleRow[k] = s.StringVal } - case "BYTES(MAX)": + case "B": var s []byte err := r.Column(i, &s) if err != nil { @@ -828,19 +833,7 @@ func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{}, } singleRow[k] = m } - case "INT64": - var s spanner.NullInt64 - err := r.Column(i, &s) - if err != nil { - if strings.Contains(err.Error(), "ambiguous column name") { - continue - } - return nil, errors.New("ValidationException", err, k) - } - if !s.IsNull() { - singleRow[k] = s.Int64 - } - case "FLOAT64": + case "N": var s spanner.NullFloat64 err := r.Column(i, &s) if err != nil { diff --git a/storage/spanner_test.go b/storage/spanner_test.go index d2a40be..d0ebef3 100644 --- a/storage/spanner_test.go +++ b/storage/spanner_test.go @@ -14,7 +14,6 @@ package storage import ( - "math/big" "reflect" "testing" @@ -22,91 +21,124 @@ import ( ) func Test_parseRow(t *testing.T) { - simpleStringRow, _ := spanner.NewRow([]string{"strCol"}, []interface{}{"my-text"}) - simpleIntRow, _ := spanner.NewRow([]string{"intCol"}, []interface{}{int64(314)}) - simpleFloatRow, _ := spanner.NewRow([]string{"floatCol"}, []interface{}{3.14}) - simpleNumericIntRow, _ := spanner.NewRow([]string{"numericCol"}, []interface{}{big.NewRat(314, 1)}) - simpleNumericFloatRow, _ := spanner.NewRow([]string{"numericCol"}, []interface{}{big.NewRat(13, 4)}) - simpleBoolRow, _ := spanner.NewRow([]string{"boolCol"}, []interface{}{true}) - removeNullRow, _ := spanner.NewRow([]string{"strCol", "nullCol"}, []interface{}{"my-text", spanner.NullString{}}) - skipCommitTimestampRow, _ := spanner.NewRow([]string{"strCol", "commit_timestamp"}, []interface{}{"my-text", "2021-01-01"}) - multipleValuesRow, _ := spanner.NewRow([]string{"strCol", "intCol", "nullCol", "boolCol"}, []interface{}{"my-text", int64(32), spanner.NullString{}, true}) - - - type args struct { - r *spanner.Row - colDDL map[string]string - } tests := []struct { - name string - args args - want map[string]interface{} - wantErr bool + name string + row *spanner.Row + colDDL map[string]string + want map[string]interface{} + wantError bool }{ { - "ParseStringValue", - args{simpleStringRow, map[string]string{"strCol": "STRING(MAX)"}}, - map[string]interface{}{"strCol": "my-text"}, - false, - }, - { - "ParseIntValue", - args{simpleIntRow, map[string]string{"intCol": "INT64"}}, - map[string]interface{}{"intCol": int64(314)}, - false, - }, - { - "ParseFloatValue", - args{simpleFloatRow, map[string]string{"floatCol": "FLOAT64"}}, - map[string]interface{}{"floatCol": 3.14}, - false, + name: "ParseStringValue", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"strCol"}, []interface{}{ + spanner.NullString{StringVal: "my-text", Valid: true}, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"strCol": "S"}, + want: map[string]interface{}{"strCol": "my-text"}, }, { - "ParseNumericIntValue", - args{simpleNumericIntRow, map[string]string{"numericCol": "NUMERIC"}}, - map[string]interface{}{"numericCol": int64(314)}, - false, + name: "ParseIntValue", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"intCol"}, []interface{}{ + spanner.NullFloat64{Float64: 314, Valid: true}, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"intCol": "N"}, + want: map[string]interface{}{"intCol": 314.0}, }, { - "ParseNumericFloatValue", - args{simpleNumericFloatRow, map[string]string{"numericCol": "NUMERIC"}}, - map[string]interface{}{"numericCol": 3.25}, - false, + name: "ParseFloatValue", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"floatCol"}, []interface{}{ + spanner.NullFloat64{Float64: 3.14, Valid: true}, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"floatCol": "N"}, + want: map[string]interface{}{"floatCol": 3.14}, }, { - "ParseBoolValue", - args{simpleBoolRow, map[string]string{"boolCol": "BOOL"}}, - map[string]interface{}{"boolCol": true}, - false, + name: "ParseBoolValue", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"boolCol"}, []interface{}{ + spanner.NullBool{Bool: true, Valid: true}, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"boolCol": "BOOL"}, + want: map[string]interface{}{"boolCol": true}, }, { - "RemoveNulls", - args{removeNullRow, map[string]string{"strCol": "STRING(MAX)", "nullCol": "STRING(MAX)"}}, - map[string]interface{}{"strCol": "my-text"}, - false, + name: "RemoveNulls", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"strCol"}, []interface{}{ + spanner.NullString{StringVal: "", Valid: false}, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"strCol": "S"}, + want: map[string]interface{}{}, // Null value should be removed }, { - "SkipCommitTimestamp", - args{skipCommitTimestampRow, map[string]string{"strCol": "STRING(MAX)", "commit_timestamp": "TIMESTAMP"}}, - map[string]interface{}{"strCol": "my-text"}, - false, + name: "SkipCommitTimestamp", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"commit_timestamp"}, []interface{}{ + nil, // Commit timestamp should be skipped + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"commit_timestamp": "S"}, + want: map[string]interface{}{}, // Commit timestamp should not appear in the result }, { - "MultiValueRow", - args{multipleValuesRow, map[string]string{"strCol": "STRING(MAX)", "intCol": "INT64", "nullCol": "STRING(MAX)", "boolCol": "BOOL"}}, - map[string]interface{}{"strCol": "my-text", "intCol": int64(32), "boolCol": true}, - false, + name: "MultiValueRow", + row: func() *spanner.Row { + row, err := spanner.NewRow([]string{"boolCol", "intCol", "strCol"}, []interface{}{ + spanner.NullBool{Bool: true, Valid: true}, + spanner.NullFloat64{Float64: 32, Valid: true}, + spanner.NullString{StringVal: "my-text", Valid: true}, + }) + if err != nil { + t.Fatalf("failed to create row: %v", err) + } + return row + }(), + colDDL: map[string]string{"boolCol": "BOOL", "intCol": "N", "strCol": "S"}, + want: map[string]interface{}{"boolCol": true, "intCol": 32.0, "strCol": "my-text"}, }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseRow(tt.args.r, tt.args.colDDL) - if (err != nil) != tt.wantErr { - t.Errorf("parseRowForNull() error = %v, wantErr %v", err, tt.wantErr) + got, err := parseRow(tt.row, tt.colDDL) + if (err != nil) != tt.wantError { + t.Errorf("parseRow() error = %v, wantError %v", err, tt.wantError) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("parseRowForNull() = %v, want %v", got, tt.want) + t.Errorf("parseRow() = %v, want %v", got, tt.want) } }) } diff --git a/storage/storage.go b/storage/storage.go index ab81de5..8e4fee8 100755 --- a/storage/storage.go +++ b/storage/storage.go @@ -23,10 +23,8 @@ import ( "syscall" "cloud.google.com/go/spanner" - "github.com/cloudspannerecosystem/dynamodb-adapter/config" "github.com/cloudspannerecosystem/dynamodb-adapter/models" "github.com/cloudspannerecosystem/dynamodb-adapter/pkg/logger" - "github.com/cloudspannerecosystem/dynamodb-adapter/utils" ) // Storage object for intracting with storage package @@ -37,10 +35,9 @@ type Storage struct { // storage - global instance of storage var storage *Storage -func initSpannerDriver(instance string) *spanner.Client { +func initSpannerDriver() *spanner.Client { conf := spanner.ClientConfig{} - - str := "projects/" + config.ConfigurationMap.GoogleProjectID + "/instances/" + instance + "/databases/" + config.ConfigurationMap.SpannerDb + str := "projects/" + models.GlobalConfig.Spanner.ProjectID + "/instances/" + models.GlobalConfig.Spanner.InstanceID + "/databases/" + models.GlobalConfig.Spanner.DatabaseName Client, err := spanner.NewClientWithConfig(context.Background(), str, conf) if err != nil { logger.LogFatal(err) @@ -50,14 +47,10 @@ func initSpannerDriver(instance string) *spanner.Client { // InitializeDriver - this will Initialize databases object in global map func InitializeDriver() { - storage = new(Storage) storage.spannerClient = make(map[string]*spanner.Client) - for _, v := range models.SpannerTableMap { - if _, ok := storage.spannerClient[v]; !ok { - storage.spannerClient[v] = initSpannerDriver(v) - } - } + storage.spannerClient[models.GlobalConfig.Spanner.InstanceID] = initSpannerDriver() + } // Close - This gracefully returns the session pool objects, when driver gets exit signal @@ -85,6 +78,6 @@ func GetStorageInstance() *Storage { return storage } -func (s Storage) getSpannerClient(tableName string) *spanner.Client { - return s.spannerClient[models.SpannerTableMap[utils.ChangeTableNameForSpanner(tableName)]] +func (s Storage) getSpannerClient(_ string) *spanner.Client { + return s.spannerClient[models.GlobalConfig.Spanner.InstanceID] } diff --git a/utils/utils.go b/utils/utils.go index 63f5c9a..d882c4c 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -172,3 +172,31 @@ func ChangeTableNameForSpanner(tableName string) string { tableName = strings.ReplaceAll(tableName, "-", "_") return tableName } + +// Convert DynamoDB data types to equivalent Spanner types +func ConvertDynamoTypeToSpannerType(dynamoType string) string { + switch dynamoType { + case "S": + return "STRING(MAX)" + case "N": + return "FLOAT64" + case "B": + return "BYTES(MAX)" + case "BOOL": + return "BOOL" + case "NULL": + return "NULL" + case "SS": + return "ARRAY" + case "NS": + return "ARRAY" + case "BS": + return "ARRAY" + case "M": + return "JSON" + case "L": + return "JSON" + default: + return "STRING(MAX)" + } +} From 4621112400e0bc0c52e2733f23db81e499ea647a Mon Sep 17 00:00:00 2001 From: taherkl Date: Tue, 11 Feb 2025 00:32:57 +0530 Subject: [PATCH 13/15] integration test for set --- integrationtest/api_test.go | 117 +++++++++++++++++++----------------- integrationtest/setup.go | 63 +++++++++++++------ 2 files changed, 106 insertions(+), 74 deletions(-) diff --git a/integrationtest/api_test.go b/integrationtest/api_test.go index 798a820..bfbb073 100644 --- a/integrationtest/api_test.go +++ b/integrationtest/api_test.go @@ -54,26 +54,29 @@ var ( "emp_id": {N: aws.String("2")}, }, } - getItemTest2Output = `{"Item":{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}}}` - getItemTest3 = models.GetItemMeta{ + getItemTest2Output = `{"Item":{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"},"phone_numbers":{"SS":["+1333333333"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]},"salaries":{"NS":["3000"]}}}` + + getItemTest3 = models.GetItemMeta{ TableName: "employee", Key: map[string]*dynamodb.AttributeValue{ "emp_id": {N: aws.String("2")}, }, - ProjectionExpression: "emp_id, address", + ProjectionExpression: "emp_id, address, phone_numbers, salaries", } - getItemTest3Output = `{"Item":{"address":{"S":"Ney York"},"emp_id":{"N":"2"}}}` + getItemTest3Output = `{"Item":{"address":{"S":"New York"},"emp_id":{"N":"2"},"phone_numbers":{"SS":["+1333333333"]},"salaries":{"NS":["3000"]}}}` getItemTest4 = models.GetItemMeta{ TableName: "employee", Key: map[string]*dynamodb.AttributeValue{ "emp_id": {N: aws.String("2")}, }, - ProjectionExpression: "#emp, address", + ProjectionExpression: "#emp, address, profile_pics", ExpressionAttributeNames: map[string]string{ "#emp": "emp_id", }, } + getItemTest4Output = `{"Item":{"address":{"S":"New York"},"emp_id":{"N":"2"},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]}}}` + getItemTest5 = models.GetItemMeta{ TableName: "employee", Key: map[string]*dynamodb.AttributeValue{ @@ -81,7 +84,7 @@ var ( }, ProjectionExpression: "#emp, address", } - getItemTest5Output = `{"Item":{"address":{"S":"Ney York"}}}` + getItemTest5Output = `{"Item":{"address":{"S":"New York"}}}` ) // params for TestGetBatchAPI @@ -113,7 +116,7 @@ var ( }, }, } - TestGetBatch3Output = `{"Responses":{"employee":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}}` + TestGetBatch3Output = `{"Responses":{"employee":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]},"salaries":{"NS":["9000.5"]}}]}}` TestGetBatch4Name = "4: Keys present for 2 table" TestGetBatch4 = models.BatchGetMeta{ @@ -133,7 +136,7 @@ var ( }, }, } - TestGetBatch4Output = `{"Responses":{"department":[{"d_id":{"N":"100"},"d_name":{"S":"Engineering"},"d_specialization":{"S":"CSE, ECE, Civil"}},{"d_id":{"N":"300"},"d_name":{"S":"Culture"},"d_specialization":{"S":"History"}}],"employee":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}}` + TestGetBatch4Output = `{"Responses":{"department":[{"d_id":{"N":"100"},"d_name":{"S":"Engineering"},"d_specialization":{"S":"CSE, ECE, Civil"}},{"d_id":{"N":"300"},"d_name":{"S":"Culture"},"d_specialization":{"S":"History"}}],"employee":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]},"salaries":{"NS":["9000.5"]}}]}}` TestGetBatch5Name = "5: ProjectionExpression without ExpressionAttributeNames for 1 table" TestGetBatch5 = models.BatchGetMeta{ @@ -144,11 +147,11 @@ var ( {"emp_id": {N: aws.String("5")}}, {"emp_id": {N: aws.String("3")}}, }, - ProjectionExpression: "emp_id, address, first_name, last_name", + ProjectionExpression: "emp_id, address, first_name, last_name, phone_numbers, profile_pics, address", }, }, } - TestGetBatch5Output = `{"Responses":{"employee":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"London"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}}` + TestGetBatch5Output = `{"Responses":{"employee":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]}},{"address":{"S":"London"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]}}]}}` TestGetBatch6Name = "6: ProjectionExpression without ExpressionAttributeNames for 2 table" TestGetBatch6 = models.BatchGetMeta{ @@ -159,7 +162,7 @@ var ( {"emp_id": {N: aws.String("5")}}, {"emp_id": {N: aws.String("3")}}, }, - ProjectionExpression: "emp_id, address, first_name, last_name", + ProjectionExpression: "emp_id, address, first_name, last_name, phone_numbers, profile_pics, address", }, "department": { Keys: []map[string]*dynamodb.AttributeValue{ @@ -170,7 +173,7 @@ var ( }, }, } - TestGetBatch6Output = `{"Responses":{"department":[{"d_id":{"N":"100"},"d_name":{"S":"Engineering"},"d_specialization":{"S":"CSE, ECE, Civil"}},{"d_id":{"N":"300"},"d_name":{"S":"Culture"},"d_specialization":{"S":"History"}}],"employee":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"London"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}}` + TestGetBatch6Output = `{"Responses":{"department":[{"d_id":{"N":"100"},"d_name":{"S":"Engineering"},"d_specialization":{"S":"CSE, ECE, Civil"}},{"d_id":{"N":"300"},"d_name":{"S":"Culture"},"d_specialization":{"S":"History"}}],"employee":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]}},{"address":{"S":"London"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]}}]}}` TestGetBatch7Name = "7: ProjectionExpression with ExpressionAttributeNames for 1 table" TestGetBatch7 = models.BatchGetMeta{ @@ -181,7 +184,7 @@ var ( {"emp_id": {N: aws.String("5")}}, {"emp_id": {N: aws.String("3")}}, }, - ProjectionExpression: "#emp, #add, first_name, last_name", + ProjectionExpression: "#emp, #add, first_name, last_name, phone_numbers, profile_pics, address", ExpressionAttributeNames: map[string]string{ "#emp": "emp_id", "#add": "address", @@ -189,7 +192,7 @@ var ( }, }, } - TestGetBatch7Output = `{"Responses":{"employee":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"London"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}}` + TestGetBatch7Output = `{"Responses":{"employee":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]}},{"address":{"S":"London"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]}}]}}` TestGetBatch8Name = "8: ProjectionExpression with ExpressionAttributeNames for 2 table" TestGetBatch8 = models.BatchGetMeta{ @@ -200,7 +203,7 @@ var ( {"emp_id": {N: aws.String("5")}}, {"emp_id": {N: aws.String("3")}}, }, - ProjectionExpression: "#emp, #add, first_name, last_name", + ProjectionExpression: "#emp, #add, first_name, last_name, phone_numbers, profile_pics, address", ExpressionAttributeNames: map[string]string{ "#emp": "emp_id", "#add": "address", @@ -219,7 +222,7 @@ var ( }, }, } - TestGetBatch8Output = `{"Responses":{"department":[{"d_id":{"N":"100"},"d_name":{"S":"Engineering"},"d_specialization":{"S":"CSE, ECE, Civil"}},{"d_id":{"N":"300"},"d_name":{"S":"Culture"},"d_specialization":{"S":"History"}}],"employee":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"London"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}}` + TestGetBatch8Output = `{"Responses":{"department":[{"d_id":{"N":"100"},"d_name":{"S":"Engineering"},"d_specialization":{"S":"CSE, ECE, Civil"}},{"d_id":{"N":"300"},"d_name":{"S":"Culture"},"d_specialization":{"S":"History"}}],"employee":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]}},{"address":{"S":"London"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]}}]}}` TestGetBatch9Name = "9: ProjectionExpression but ExpressionAttributeNames not present" TestGetBatch9 = models.BatchGetMeta{ @@ -230,11 +233,11 @@ var ( {"emp_id": {N: aws.String("5")}}, {"emp_id": {N: aws.String("3")}}, }, - ProjectionExpression: "#emp, #add, first_name, last_name", + ProjectionExpression: "#emp, #add, first_name, last_name phone_numbers, profile_pics, address", }, }, } - TestGetBatch9Output = `{"Responses":{"employee":[{"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}}` + TestGetBatch9Output = `{"Responses":{"employee":[{"address":{"S":"Shamli"},"first_name":{"S":"Marc"},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]}},{"address":{"S":"Pune"},"first_name":{"S":"Alice"},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]}},{"address":{"S":"London"},"first_name":{"S":"David"},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]}}]}}` TestGetBatch10Name = "10: Wrong Keys" TestGetBatch10 = models.BatchGetMeta{ @@ -442,7 +445,7 @@ var ( Limit: 4, } - queryTestCaseOutput1 = `{"Count":5,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}` + queryTestCaseOutput1 = `{"Count":5,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}},{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"},"phone_numbers":{"SS":["+1333333333"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]},"salaries":{"NS":["3000"]}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"},"phone_numbers":{"SS":["+1666666666"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTY="]},"salaries":{"NS":["7000","8000.25"]}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]},"salaries":{"NS":["9000.5"]}}]}` queryTestCaseOutput2 = `{"Count":5,"Items":[{"emp_id":{"N":"1"},"first_name":{"S":"Marc"}},{"emp_id":{"N":"2"},"first_name":{"S":"Catalina"}},{"emp_id":{"N":"3"},"first_name":{"S":"Alice"}},{"emp_id":{"N":"4"},"first_name":{"S":"Lea"}},{"emp_id":{"N":"5"},"first_name":{"S":"David"}}]}` @@ -456,11 +459,11 @@ var ( queryTestCaseOutput9 = `{"Count":1,"Items":[{"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}}]}` - queryTestCaseOutput10 = `{"Count":5,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}` + queryTestCaseOutput10 = `{"Count":5,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}},{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"},"phone_numbers":{"SS":["+1333333333"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]},"salaries":{"NS":["3000"]}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"},"phone_numbers":{"SS":["+1666666666"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTY="]},"salaries":{"NS":["7000","8000.25"]}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]},"salaries":{"NS":["9000.5"]}}]}` - queryTestCaseOutput11 = `{"Count":4,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"}}],"LastEvaluatedKey":{"emp_id":{"N":"4"},"offset":{"N":"4"}}}` + queryTestCaseOutput11 = `{"Count":4,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}},{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"},"phone_numbers":{"SS":["+1333333333"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]},"salaries":{"NS":["3000"]}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"},"phone_numbers":{"SS":["+1666666666"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTY="]},"salaries":{"NS":["7000","8000.25"]}}],"LastEvaluatedKey":{"emp_id":{"N":"4"},"offset":{"N":"4"}}}` - queryTestCaseOutput12 = `{"Count":4,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"}}],"LastEvaluatedKey":{"emp_id":{"N":"4"},"offset":{"N":"4"}}}` + queryTestCaseOutput12 = `{"Count":4,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}},{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"},"phone_numbers":{"SS":["+1333333333"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]},"salaries":{"NS":["3000"]}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"},"phone_numbers":{"SS":["+1666666666"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTY="]},"salaries":{"NS":["7000","8000.25"]}}],"LastEvaluatedKey":{"emp_id":{"N":"4"},"offset":{"N":"4"}}}` queryTestCaseOutput13 = `{"Count":5,"Items":[]}` @@ -482,21 +485,21 @@ var ( ScanTestCase2 = models.ScanMeta{ TableName: "employee", } - ScanTestCase2Output = `{"Count":5,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}` + ScanTestCase2Output = `{"Count":5,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}},{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"},"phone_numbers":{"SS":["+1333333333"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]},"salaries":{"NS":["3000"]}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"},"phone_numbers":{"SS":["+1666666666"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTY="]},"salaries":{"NS":["7000","8000.25"]}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]},"salaries":{"NS":["9000.5"]}}]}` ScanTestCase3Name = "3: With Limit Attribute" ScanTestCase3 = models.ScanMeta{ TableName: "employee", Limit: 3, } - ScanTestCase3Output = `{"Count":3,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}}],"LastEvaluatedKey":{"emp_id":{"N":"3"},"offset":{"N":"3"}}}` + ScanTestCase3Output = `{"Count":3,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}},{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"},"phone_numbers":{"SS":["+1333333333"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]},"salaries":{"NS":["3000"]}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}}],"LastEvaluatedKey":{"emp_id":{"N":"3"},"offset":{"N":"3"}}}` ScanTestCase4Name = "4: With Projection Expression" ScanTestCase4 = models.ScanMeta{ TableName: "employee", ProjectionExpression: "address, emp_id, first_name", } - ScanTestCase4Output = `{"Count":5,"Items":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"}},{"address":{"S":"Ney York"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"}},{"address":{"S":"Silicon Valley"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"}},{"address":{"S":"London"},"emp_id":{"N":"5"},"first_name":{"S":"David"}}]}` + ScanTestCase4Output = `{"Count":5,"Items":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"}},{"address":{"S":"New York"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"}},{"address":{"S":"Silicon Valley"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"}},{"address":{"S":"London"},"emp_id":{"N":"5"},"first_name":{"S":"David"}}]}` ScanTestCase5Name = "5: With Projection Expression & limit" ScanTestCase5 = models.ScanMeta{ @@ -504,7 +507,7 @@ var ( Limit: 3, ProjectionExpression: "address, emp_id, first_name", } - ScanTestCase5Output = `{"Count":3,"Items":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"}},{"address":{"S":"Ney York"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"}}],"LastEvaluatedKey":{"emp_id":{"N":"3"},"offset":{"N":"3"}}}` + ScanTestCase5Output = `{"Count":3,"Items":[{"address":{"S":"Shamli"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"}},{"address":{"S":"New York"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"}},{"address":{"S":"Pune"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"}}],"LastEvaluatedKey":{"emp_id":{"N":"3"},"offset":{"N":"3"}}}` ScanTestCase6Name = "6: Projection Expression without ExpressionAttributeNames" ScanTestCase6 = models.ScanMeta{ @@ -525,7 +528,7 @@ var ( Limit: 3, ProjectionExpression: "address, #ag, emp_id, first_name, last_name", } - ScanTestCase7Output = `{"Count":3,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}}],"LastEvaluatedKey":{"emp_id":{"N":"3"},"offset":{"N":"3"}}}` + ScanTestCase7Output = `{"Count":3,"Items":[{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}},{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}}],"LastEvaluatedKey":{"emp_id":{"N":"3"},"offset":{"N":"3"}}}` //400 Bad request ScanTestCase8Name = "8: Filter Expression without ExpressionAttributeValues" @@ -546,7 +549,7 @@ var ( }, FilterExpression: "age > :val1", } - ScanTestCase9Output = `{"Count":4,"Items":[{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}` + ScanTestCase9Output = `{"Count":4,"Items":[{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"},"phone_numbers":{"SS":["+1333333333"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]},"salaries":{"NS":["3000"]}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"},"phone_numbers":{"SS":["+1666666666"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTY="]},"salaries":{"NS":["7000","8000.25"]}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]},"salaries":{"NS":["9000.5"]}}]}` //400 bad request ScanTestCase10Name = "10: FilterExpression & ExpressionAttributeValues without ExpressionAttributeNames" @@ -567,7 +570,7 @@ var ( }, FilterExpression: "age > :val1", } - ScanTestCase11Output = `{"Count":4,"Items":[{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}` + ScanTestCase11Output = `{"Count":4,"Items":[{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"},"phone_numbers":{"SS":["+1333333333"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]},"salaries":{"NS":["3000"]}},{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}},{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"},"phone_numbers":{"SS":["+1666666666"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTY="]},"salaries":{"NS":["7000","8000.25"]}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]},"salaries":{"NS":["9000.5"]}}]}` ScanTestCase12Name = "12: With ExclusiveStartKey" ScanTestCase12 = models.ScanMeta{ @@ -578,7 +581,7 @@ var ( }, Limit: 3, } - ScanTestCase12Output = `{"Count":2,"Items":[{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"}}]}` + ScanTestCase12Output = `{"Count":2,"Items":[{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"},"phone_numbers":{"SS":["+1666666666"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTY="]},"salaries":{"NS":["7000","8000.25"]}},{"address":{"S":"London"},"age":{"N":"50"},"emp_id":{"N":"5"},"first_name":{"S":"David"},"last_name":{"S":"Lomond"},"phone_numbers":{"SS":["+1777777777","+1888888888","+1999999999"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTc=","U29tZUJ5dGVzRGF0YTg="]},"salaries":{"NS":["9000.5"]}}]}` ScanTestCase13Name = "13: With Count" ScanTestCase13 = models.ScanMeta{ @@ -610,9 +613,9 @@ var ( }, ReturnValues: "ALL_NEW", } - UpdateItemTestCase2Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}}}` + UpdateItemTestCase2Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}}}` - UpdateItemTestCase3Name = "2: UpdateExpression, ExpressionAttributeValues with ExpressionAttributeNames" + UpdateItemTestCase3Name = "3: UpdateExpression, ExpressionAttributeValues with ExpressionAttributeNames" UpdateItemTestCase3 = models.UpdateAttr{ TableName: "employee", Key: map[string]*dynamodb.AttributeValue{ @@ -626,7 +629,7 @@ var ( "#ag": "age", }, } - UpdateItemTestCase3Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}}}` + UpdateItemTestCase3Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}}}` UpdateItemTestCase4Name = "4: Update Expression without ExpressionAttributeValues" UpdateItemTestCase4 = models.UpdateAttr{ @@ -675,7 +678,7 @@ var ( "#ag": "age", }, } - UpdateItemTestCase7Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}}}` + UpdateItemTestCase7Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}}}` //400 bad request UpdateItemTestCase8Name = "8: Wrong ConditionExpression" @@ -744,7 +747,7 @@ var ( "age": {N: aws.String("11")}, }, } - PutItemTestCase2Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}}}` + PutItemTestCase2Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}}}` PutItemTestCase3Name = "3: ConditionExpression with ExpressionAttributeValues & ExpressionAttributeNames" PutItemTestCase3 = models.Meta{ @@ -761,7 +764,7 @@ var ( "#ag": "age", }, } - PutItemTestCase3Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"11"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}}}` + PutItemTestCase3Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"11"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}}}` PutItemTestCase4Name = "4: ConditionExpression with ExpressionAttributeValues" PutItemTestCase4 = models.Meta{ @@ -775,7 +778,7 @@ var ( ":val2": {N: aws.String("9")}, }, } - PutItemTestCase4Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"}}}` + PutItemTestCase4Output = `{"Attributes":{"address":{"S":"Shamli"},"age":{"N":"10"},"emp_id":{"N":"1"},"first_name":{"S":"Marc"},"last_name":{"S":"Richards"},"phone_numbers":{"SS":["+1111111111","+1222222222"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTE=","U29tZUJ5dGVzRGF0YTI="]},"salaries":{"NS":["1000.5","2000.75"]}}}` //400 bad request PutItemTestCase5Name = "5: ConditionExpression without ExpressionAttributeValues" @@ -853,7 +856,7 @@ var ( "emp_id": {N: aws.String("2")}, }, } - DeleteItemTestCase2Output = `{"Attributes":{"address":{"S":"Ney York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"}}}` + DeleteItemTestCase2Output = `{"Attributes":{"address":{"S":"New York"},"age":{"N":"20"},"emp_id":{"N":"2"},"first_name":{"S":"Catalina"},"last_name":{"S":"Smith"},"phone_numbers":{"SS":["+1333333333"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTM="]},"salaries":{"NS":["3000"]}}}` DeleteItemTestCase3Name = "3: Icorrect Key passed" DeleteItemTestCase3 = models.Delete{ @@ -874,7 +877,7 @@ var ( ":val2": {N: aws.String("9")}, }, } - DeleteItemTestCase4Output = `{"Attributes":{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}}}` + DeleteItemTestCase4Output = `{"Attributes":{"address":{"S":"Pune"},"age":{"N":"30"},"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"},"phone_numbers":{"SS":["+1444444444","+1555555555"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTQ=","U29tZUJ5dGVzRGF0YTU="]},"salaries":{"NS":["4000.25","5000.5","6000.75"]}}}` DeleteItemTestCase5Name = "5: ConditionExpressionNames with ExpressionAttributeNames & ExpressionAttributeValues" DeleteItemTestCase5 = models.Delete{ @@ -884,13 +887,13 @@ var ( }, ConditionExpression: "#ag > :val2", ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{ - ":val2": {N: aws.String("19")}, + ":val2": {N: aws.String("19.0")}, }, ExpressionAttributeNames: map[string]string{ "#ag": "age", }, } - DeleteItemTestCase5Output = `{"Attributes":{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"}}}` + DeleteItemTestCase5Output = `{"Attributes":{"address":{"S":"Silicon Valley"},"age":{"N":"40"},"emp_id":{"N":"4"},"first_name":{"S":"Lea"},"last_name":{"S":"Martin"},"phone_numbers":{"SS":["+1666666666"]},"profile_pics":{"BS":["U29tZUJ5dGVzRGF0YTY="]},"salaries":{"NS":["7000","8000.25"]}}}` DeleteItemTestCase6Name = "6: ConditionExpressionNames without ExpressionAttributeValues" DeleteItemTestCase6 = models.Delete{ @@ -936,34 +939,40 @@ var ( } BatchWriteItemTestCase2Name = "2: Batch Put Request for one table" - BatchWriteItemTestCase2 = models.BatchWriteItem{ + BatchWriteItemTestCase2 = models.BatchWriteItem{ RequestItems: map[string][]models.BatchWriteSubItems{ "employee": { { PutReq: models.BatchPutItem{ Item: map[string]*dynamodb.AttributeValue{ - "emp_id": {N: aws.String("6")}, - "age": {N: aws.String("60")}, - "address": {S: aws.String("London")}, - "first_name": {S: aws.String("David")}, - "last_name": {S: aws.String("Root")}, + "emp_id": {N: aws.String("6")}, + "age": {N: aws.String("60")}, + "address": {S: aws.String("London")}, + "first_name": {S: aws.String("David")}, + "last_name": {S: aws.String("Root")}, + "phone_numbers": {SS: []*string{aws.String("+1777777777"), aws.String("+1888888888")}}, + "profile_pics": {BS: [][]byte{[]byte("U29tZUJ5dGVzRGF0YTc="), []byte("U29tZUJ5dGVzRGF0YTg=")}}, + "salaries": {NS: []*string{aws.String("9000.50"), aws.String("10000.75")}}, }, }, }, { PutReq: models.BatchPutItem{ Item: map[string]*dynamodb.AttributeValue{ - "emp_id": {N: aws.String("7")}, - "age": {N: aws.String("70")}, - "address": {S: aws.String("Paris")}, - "first_name": {S: aws.String("Marc")}, - "last_name": {S: aws.String("Ponting")}, + "emp_id": {N: aws.String("7")}, + "age": {N: aws.String("70")}, + "address": {S: aws.String("Paris")}, + "first_name": {S: aws.String("Marc")}, + "last_name": {S: aws.String("Ponting")}, + "phone_numbers": {SS: []*string{aws.String("+1999999999"), aws.String("+2111111111")}}, + "profile_pics": {BS: [][]byte{[]byte("U29tZUJ5dGVzRGF0YTk="), []byte("U29tZUJ5dGVzRGF0YTEw=")}}, + "salaries": {NS: []*string{aws.String("11000"), aws.String("12000.25")}}, }, }, }, }, }, - } + } BatchWriteItemTestCase3Name = "3: Batch Delete Request for one Table" BatchWriteItemTestCase3 = models.BatchWriteItem{ @@ -1433,7 +1442,7 @@ func testGetItemAPI(t *testing.T) { }, createPostTestCase("Crorect Data TestCase", "/v1", "GetItem", getItemTest2Output, getItemTest2), createPostTestCase("Crorect data with Projection param Testcase", "/v1", "GetItem", getItemTest3Output, getItemTest3), - createPostTestCase("Crorect data with ExpressionAttributeNames Testcase", "/v1", "GetItem", getItemTest3Output, getItemTest4), + createPostTestCase("Crorect data with ExpressionAttributeNames Testcase", "/v1", "GetItem", getItemTest4Output, getItemTest4), createPostTestCase("Crorect data with ExpressionAttributeNames values not passed Testcase", "/v1", "GetItem", getItemTest5Output, getItemTest5), } apitest.RunTests(t, tests) diff --git a/integrationtest/setup.go b/integrationtest/setup.go index 318407f..0b640cc 100644 --- a/integrationtest/setup.go +++ b/integrationtest/setup.go @@ -112,11 +112,14 @@ func createDatabase(w io.Writer, db string) error { spannerDataType STRING(MAX) ) PRIMARY KEY (tableName, column)`, `CREATE TABLE employee ( - emp_id FLOAT64, - address STRING(MAX), - age FLOAT64, - first_name STRING(MAX), - last_name STRING(MAX), + emp_id FLOAT64, + address STRING(MAX), + age FLOAT64, + first_name STRING(MAX), + last_name STRING(MAX), + phone_numbers ARRAY, + profile_pics ARRAY, + salaries ARRAY ) PRIMARY KEY (emp_id)`, `CREATE TABLE department ( d_id FLOAT64, @@ -160,15 +163,20 @@ func initData(w io.Writer, db string) error { _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { stmt := spanner.Statement{ - SQL: `INSERT dynamodb_adapter_table_ddl (tableName, column, dynamoDataType, originalColumn, partitionKey,sortKey, spannerIndexName, actualTable, spannerDataType) VALUES - ('employee', 'emp_id', 'N', 'emp_id', 'emp_id','', 'emp_id','employee', 'FLOAT64'), - ('employee', 'address', 'S', 'address', 'emp_id','', 'address','employee', 'STRING(MAX)'), - ('employee', 'age', 'N', 'age', 'emp_id','', 'age','employee', 'FLOAT64'), - ('employee', 'first_name', 'S', 'first_name', 'emp_id','', 'first_name','employee', 'STRING(MAX)'), - ('employee', 'last_name', 'S', 'last_name', 'emp_id','', 'last_name','employee', 'STRING(MAX)'), - ('department', 'd_id', 'N', 'd_id', 'd_id','', 'd_id','department', 'FLOAT64'), - ('department', 'd_name', 'S', 'd_name', 'd_id','', 'd_name','department', 'STRING(MAX)'), - ('department', 'd_specialization', 'S', 'd_specialization', 'd_id','', 'd_specialization','department', 'STRING(MAX)')`, + SQL: `INSERT INTO dynamodb_adapter_table_ddl + (tableName, column, dynamoDataType, originalColumn, partitionKey, sortKey, spannerIndexName, actualTable, spannerDataType) + VALUES + ('employee', 'emp_id', 'N', 'emp_id', 'emp_id', '', 'emp_id', 'employee', 'FLOAT64'), + ('employee', 'address', 'S', 'address', 'emp_id', '', 'address', 'employee', 'STRING(MAX)'), + ('employee', 'age', 'N', 'age', 'emp_id', '', 'age', 'employee', 'FLOAT64'), + ('employee', 'first_name', 'S', 'first_name', 'emp_id', '', 'first_name', 'employee', 'STRING(MAX)'), + ('employee', 'last_name', 'S', 'last_name', 'emp_id', '', 'last_name', 'employee', 'STRING(MAX)'), + ('employee', 'phone_numbers', 'SS', 'phone_numbers', 'emp_id', '', 'phone_numbers', 'employee', 'ARRAY'), + ('employee', 'profile_pics', 'BS', 'profile_pics', 'emp_id', '', 'profile_pics', 'employee', 'ARRAY'), + ('employee', 'salaries', 'NS', 'salaries', 'emp_id', '', 'salaries', 'employee', 'ARRAY'), + ('department', 'd_id', 'N', 'd_id', 'd_id', '', 'd_id', 'department', 'FLOAT64'), + ('department', 'd_name', 'S', 'd_name', 'd_id', '', 'd_name', 'department', 'STRING(MAX)'), + ('department', 'd_specialization', 'S', 'd_specialization', 'd_id', '', 'd_specialization', 'department', 'STRING(MAX)');`, } rowCount, err := txn.Update(ctx, stmt) if err != nil { @@ -183,12 +191,27 @@ func initData(w io.Writer, db string) error { _, err = client.ReadWriteTransaction(ctx, func(ctx context.Context, txn *spanner.ReadWriteTransaction) error { stmt := spanner.Statement{ - SQL: `INSERT employee (emp_id, address, age, first_name, last_name) VALUES - (1, 'Shamli', 10, 'Marc', 'Richards'), - (2, 'Ney York', 20, 'Catalina', 'Smith'), - (3, 'Pune', 30, 'Alice', 'Trentor'), - (4, 'Silicon Valley', 40, 'Lea', 'Martin'), - (5, 'London', 50, 'David', 'Lomond')`, + SQL: `INSERT INTO employee (emp_id, address, age, first_name, last_name, phone_numbers, profile_pics, salaries) VALUES + (1, 'Shamli', 10, 'Marc', 'Richards', + ['+1111111111', '+1222222222'], + [FROM_BASE64('U29tZUJ5dGVzRGF0YTE='), FROM_BASE64('U29tZUJ5dGVzRGF0YTI=')], + [1000.50, 2000.75]), + (2, 'New York', 20, 'Catalina', 'Smith', + ['+1333333333'], + [FROM_BASE64('U29tZUJ5dGVzRGF0YTM=')], + [3000.00]), + (3, 'Pune', 30, 'Alice', 'Trentor', + ['+1444444444', '+1555555555'], + [FROM_BASE64('U29tZUJ5dGVzRGF0YTQ='), FROM_BASE64('U29tZUJ5dGVzRGF0YTU=')], + [4000.25, 5000.50, 6000.75]), + (4, 'Silicon Valley', 40, 'Lea', 'Martin', + ['+1666666666'], + [FROM_BASE64('U29tZUJ5dGVzRGF0YTY=')], + [7000.00, 8000.25]), + (5, 'London', 50, 'David', 'Lomond', + ['+1777777777', '+1888888888', '+1999999999'], + [FROM_BASE64('U29tZUJ5dGVzRGF0YTc='), FROM_BASE64('U29tZUJ5dGVzRGF0YTg=')], + [9000.50]);`, } rowCount, err := txn.Update(ctx, stmt) if err != nil { From f53c8040e6ccde1572aba0a91b816b4530afeafb Mon Sep 17 00:00:00 2001 From: taherkl Date: Tue, 11 Feb 2025 10:02:59 +0530 Subject: [PATCH 14/15] bugfix --- api/v1/condition.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/v1/condition.go b/api/v1/condition.go index 4b9b266..405235c 100644 --- a/api/v1/condition.go +++ b/api/v1/condition.go @@ -180,6 +180,8 @@ func parseActionValue(actionValue string, updateAtrr models.UpdateAttr, assignme resp[key] = append(strSlice, newValue...) } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { resp[key] = removeFromSlice(strSlice, newValue) + } else { + resp[key] = tmp } } else { resp[key] = tmp @@ -190,6 +192,8 @@ func parseActionValue(actionValue string, updateAtrr models.UpdateAttr, assignme resp[key] = append(floatSlice, newValue...) } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { resp[key] = removeFromSlice(floatSlice, newValue) + } else { + resp[key] = tmp } } else { resp[key] = tmp @@ -200,6 +204,8 @@ func parseActionValue(actionValue string, updateAtrr models.UpdateAttr, assignme resp[key] = append(byteSlice, newValue...) } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { resp[key] = removeFromByteSlice(byteSlice, newValue) + } else { + resp[key] = tmp } } else { resp[key] = tmp From 0fc3ef08a36194783d980d40f6139f98ffaad3dc Mon Sep 17 00:00:00 2001 From: taherkl Date: Tue, 11 Feb 2025 14:07:15 +0530 Subject: [PATCH 15/15] bugfix for SET unique --- api/v1/condition.go | 19 +++++----- utils/utils.go | 43 +++++++++++++++++++++ utils/utils_test.go | 92 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 9 deletions(-) diff --git a/api/v1/condition.go b/api/v1/condition.go index 405235c..abf76e2 100644 --- a/api/v1/condition.go +++ b/api/v1/condition.go @@ -32,6 +32,7 @@ import ( "github.com/cloudspannerecosystem/dynamodb-adapter/pkg/errors" "github.com/cloudspannerecosystem/dynamodb-adapter/pkg/logger" "github.com/cloudspannerecosystem/dynamodb-adapter/service/services" + "github.com/cloudspannerecosystem/dynamodb-adapter/utils" ) var operations = map[string]string{"SET": "(?i) SET ", "DELETE": "(?i) DELETE ", "ADD": "(?i) ADD ", "REMOVE": "(?i) REMOVE "} @@ -177,38 +178,38 @@ func parseActionValue(actionValue string, updateAtrr models.UpdateAttr, assignme case []string: // String Set if strSlice, ok := oldRes[key].([]string); ok { if strings.Contains(updateAtrr.UpdateExpression, "ADD") { - resp[key] = append(strSlice, newValue...) + resp[key] = utils.RemoveDuplicatesString(append(strSlice, newValue...)) } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { resp[key] = removeFromSlice(strSlice, newValue) } else { - resp[key] = tmp + resp[key] = utils.RemoveDuplicatesString(newValue) // Ensure uniqueness even in replace } } else { - resp[key] = tmp + resp[key] = utils.RemoveDuplicatesString(newValue) } case []float64: // Number Set if floatSlice, ok := oldRes[key].([]float64); ok { if strings.Contains(updateAtrr.UpdateExpression, "ADD") { - resp[key] = append(floatSlice, newValue...) + resp[key] = utils.RemoveDuplicatesFloat(append(floatSlice, newValue...)) } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { resp[key] = removeFromSlice(floatSlice, newValue) } else { - resp[key] = tmp + resp[key] = utils.RemoveDuplicatesFloat(newValue) } } else { - resp[key] = tmp + resp[key] = utils.RemoveDuplicatesFloat(newValue) } case [][]byte: // Binary Set if byteSlice, ok := oldRes[key].([][]byte); ok { if strings.Contains(updateAtrr.UpdateExpression, "ADD") { - resp[key] = append(byteSlice, newValue...) + resp[key] = utils.RemoveDuplicatesByteSlice(append(byteSlice, newValue...)) } else if strings.Contains(updateAtrr.UpdateExpression, "DELETE") { resp[key] = removeFromByteSlice(byteSlice, newValue) } else { - resp[key] = tmp + resp[key] = utils.RemoveDuplicatesByteSlice(newValue) } } else { - resp[key] = tmp + resp[key] = utils.RemoveDuplicatesByteSlice(newValue) } default: resp[key] = tmp diff --git a/utils/utils.go b/utils/utils.go index d882c4c..6e3f5ab 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -200,3 +200,46 @@ func ConvertDynamoTypeToSpannerType(dynamoType string) string { return "STRING(MAX)" } } + +// RemoveDuplicatesString removes duplicates from a []string +func RemoveDuplicatesString(input []string) []string { + seen := make(map[string]struct{}) + var result []string + + for _, val := range input { + if _, exists := seen[val]; !exists { + seen[val] = struct{}{} + result = append(result, val) + } + } + return result +} + +// RemoveDuplicatesFloat removes duplicates from a []float64 +func RemoveDuplicatesFloat(input []float64) []float64 { + seen := make(map[float64]struct{}) + var result []float64 + + for _, val := range input { + if _, exists := seen[val]; !exists { + seen[val] = struct{}{} + result = append(result, val) + } + } + return result +} + +// RemoveDuplicatesByteSlice removes duplicates from a [][]byte +func RemoveDuplicatesByteSlice(input [][]byte) [][]byte { + seen := make(map[string]struct{}) + var result [][]byte + + for _, val := range input { + key := string(val) // Convert byte slice to string for map key + if _, exists := seen[key]; !exists { + seen[key] = struct{}{} + result = append(result, val) + } + } + return result +} diff --git a/utils/utils_test.go b/utils/utils_test.go index 80828bc..dd1978c 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -15,6 +15,8 @@ package utils import ( + "bytes" + "reflect" "testing" "github.com/antonmedv/expr" @@ -246,3 +248,93 @@ func TestChangeTableNameForSpanner(t *testing.T) { assert.Equal(t, got, tc.want) } } + +func TestRemoveDuplicatesString(t *testing.T) { + tests := []struct { + input []string + expected []string + }{ + {[]string{"apple", "banana", "apple", "orange"}, []string{"apple", "banana", "orange"}}, + {[]string{"a", "b", "a", "c", "b"}, []string{"a", "b", "c"}}, + {[]string{"one", "two", "three"}, []string{"one", "two", "three"}}, // No duplicates + {[]string{}, []string{}}, // Empty slice + } + + for _, test := range tests { + result := RemoveDuplicatesString(test.input) + if len(result) == 0 && len(test.expected) == 0 { + continue + } + if !reflect.DeepEqual(result, test.expected) { + t.Errorf("RemoveDuplicatesString(%v) = %v; want %v", test.input, result, test.expected) + } + + } +} + +func TestRemoveDuplicatesFloat(t *testing.T) { + tests := []struct { + input []float64 + expected []float64 + }{ + {[]float64{1.1, 2.2, 3.3, 1.1, 2.2}, []float64{1.1, 2.2, 3.3}}, + {[]float64{0.5, 0.5, 0.5}, []float64{0.5}}, + {[]float64{10.0, 20.0, 30.0}, []float64{10.0, 20.0, 30.0}}, // No duplicates + {[]float64{}, []float64{}}, // Empty slice + } + + for _, test := range tests { + result := RemoveDuplicatesFloat(test.input) + if len(result) == 0 && len(test.expected) == 0 { + continue + } + if !reflect.DeepEqual(result, test.expected) { + t.Errorf("RemoveDuplicatesString(%v) = %v; want %v", test.input, result, test.expected) + } + + } +} + +func TestRemoveDuplicatesByteSlice(t *testing.T) { + tests := []struct { + input [][]byte + expected [][]byte + }{ + { + [][]byte{[]byte("foo"), []byte("bar"), []byte("foo"), []byte("baz")}, + [][]byte{[]byte("foo"), []byte("bar"), []byte("baz")}, + }, + { + [][]byte{[]byte("apple"), []byte("banana"), []byte("apple")}, + [][]byte{[]byte("apple"), []byte("banana")}, + }, + { + [][]byte{[]byte("one"), []byte("two"), []byte("three")}, + [][]byte{[]byte("one"), []byte("two"), []byte("three")}, + }, + { + [][]byte{}, + [][]byte{}, + }, + } + + for _, test := range tests { + result := RemoveDuplicatesByteSlice(test.input) + if !equalByteSlices(result, test.expected) { + t.Errorf("RemoveDuplicatesByteSlice(%v) = %v; want %v", test.input, result, test.expected) + } + } +} + +// Helper function to compare [][]byte slices +func equalByteSlices(a, b [][]byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if !bytes.Equal(a[i], b[i]) { + return false + } + } + return true +}