diff --git a/README.md b/README.md index 1505e21..a486412 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,9 @@ If you prefer ORM style, and define column-field relationship via struct field t - `type` is to define the data type. if type is timestamp, `precision` is supported - the metadata separator is `;` and the key value separator is `:` -type supported is the same as described [Datatypes supported](#datatypes-supported), and case insensitive +type supported is the same as described [Datatypes supported](#datatypes-supported), and case insensitive. + +When fields marked with `greptime:"-"`, writing field will be ignored. ##### define struct with tags diff --git a/schema/field.go b/schema/field.go index d1f4080..4926af2 100644 --- a/schema/field.go +++ b/schema/field.go @@ -52,6 +52,10 @@ func newColumnSchema(columnName string, semanticType gpb.SemanticType, datatype func parseField(structField reflect.StructField) (*Field, error) { tags := parseTag(structField) + if _, ok := tags[IgnoreFiledTag]; ok && len(tags) == 1 { + return nil, nil + } + columnName, err := util.SanitateName(structField.Name) if err != nil { return nil, err @@ -85,7 +89,7 @@ func parseField(structField reflect.StructField) (*Field, error) { func parseTag(structField reflect.StructField) map[string]string { tags := map[string]string{} - str, ok := structField.Tag.Lookup("greptime") + str, ok := structField.Tag.Lookup(GreptimeFieldTagKey) if !ok { return tags } diff --git a/schema/schema.go b/schema/schema.go index 4d9775d..94931b4 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -25,6 +25,11 @@ import ( "github.com/GreptimeTeam/greptimedb-ingester-go/util" ) +const ( + IgnoreFiledTag = "-" + GreptimeFieldTagKey = "greptime" +) + type Schema struct { tableName string @@ -52,7 +57,7 @@ func getTableName(typ reflect.Type) (string, error) { func Parse(input any) (*table.Table, error) { if input == nil { - return nil, fmt.Errorf("unsupported empty data. %#v", input) + return nil, fmt.Errorf("unsupported empty data: %#v", input) } schema_, err := parseSchema(input) @@ -114,7 +119,9 @@ func parseSchema(input any) (*Schema, error) { if err != nil { return nil, err } - fields = append(fields, field.ToColumnSchema()) + if field != nil { + fields = append(fields, field.ToColumnSchema()) + } } return &Schema{tableName: tableName, fields: fields}, nil @@ -142,15 +149,19 @@ func (s *Schema) parseValues(input any) error { return fmt.Errorf("unsupported type %T of %+v", input, input) } - size := len(reflect.VisibleFields(typ)) - values := make([]*gpb.Value, 0, size) - for i, structField := range reflect.VisibleFields(typ) { - if !structField.IsExported() { + visibleFields := reflect.VisibleFields(typ) + processingFields := make([]reflect.StructField, 0, len(visibleFields)) + values := make([]*gpb.Value, 0, len(processingFields)) + + for _, structField := range visibleFields { + if !structField.IsExported() || structField.Tag.Get(GreptimeFieldTagKey) == IgnoreFiledTag { continue } + processingFields = append(processingFields, structField) + } + for i, structField := range processingFields { field := s.fields[i] - value, err := parseValue(field.Datatype, val.FieldByName(structField.Name)) if err != nil { return err diff --git a/schema/schema_test.go b/schema/schema_test.go index 520289e..095e145 100644 --- a/schema/schema_test.go +++ b/schema/schema_test.go @@ -975,3 +975,96 @@ func TestParseWithUnmatchedDatatype(t *testing.T) { assert.Nil(t, tbl) } } + +func TestParseSchemaWithIgnoreFields(t *testing.T) { + INT16 := int16(1) + UINT64 := uint64(2) + FLOAT32 := float32(3) + STRING := "string" + JSON := `{"key1":"value1","key2":10}` + + TIMESTAMP := time.Now() + DATETIME_INT := TIMESTAMP.UnixMilli() + + length := 6 + assertSchema := func(cols []*gpb.ColumnSchema) { + assert.Len(t, cols, length) + + assert.EqualValues(t, newColumnSchema("uint64_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_UINT64), cols[0]) + assert.EqualValues(t, newColumnSchema("string_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_STRING), cols[1]) + assert.EqualValues(t, newColumnSchema("json_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_JSON), cols[2]) + + offset := length / 2 + assert.EqualValues(t, newColumnSchema("ptr_uint64_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_UINT64), cols[0+offset]) + assert.EqualValues(t, newColumnSchema("ptr_string_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_STRING), cols[1+offset]) + assert.EqualValues(t, newColumnSchema("ptr_json_column", gpb.SemanticType_FIELD, gpb.ColumnDataType_JSON), cols[2+offset]) + } + + assertValue := func(row *gpb.Row) { + vals := row.Values + assert.Len(t, vals, length) + + assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_U64Value{U64Value: UINT64}}, vals[0]) + assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: STRING}}, vals[1]) + assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: JSON}}, vals[2]) + + offset := length / 2 + + assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_U64Value{U64Value: UINT64}}, vals[0+offset]) + assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: STRING}}, vals[1+offset]) + assert.EqualValues(t, &gpb.Value{ValueData: &gpb.Value_StringValue{StringValue: JSON}}, vals[2+offset]) + } + + type Monitor struct { + INT16 int16 `greptime:"-"` + UINT64 uint64 `greptime:"field;column:uint64_column;type:uint64"` + FLOAT32 float32 `greptime:"-"` + STRING string `greptime:"field;column:string_column;type:string"` + DATETIME_INT int64 `greptime:"-"` + JSON string `greptime:"field;column:json_column;type:json"` + + PtrINT16 *int16 `greptime:"-"` + PtrUINT64 *uint64 `greptime:"field;column:ptr_uint64_column;type:uint64"` + PtrFLOAT32 *float32 `greptime:"-"` + PtrSTRING *string `greptime:"field;column:ptr_string_column;type:string"` + PtrDATETIME_INT *int64 `greptime:"-"` + PtrJSON *string `greptime:"field;column:ptr_json_column;type:json"` + + privateField string // will be ignored + } + + monitor := Monitor{ + INT16: INT16, + UINT64: UINT64, + FLOAT32: FLOAT32, + STRING: STRING, + DATETIME_INT: DATETIME_INT, + JSON: JSON, + + PtrINT16: &INT16, + PtrUINT64: &UINT64, + PtrFLOAT32: &FLOAT32, + PtrSTRING: &STRING, + PtrDATETIME_INT: &DATETIME_INT, + PtrJSON: &JSON, + + privateField: "private", + } + + { + tbl, err := Parse(monitor) + assert.Nil(t, err) + assert.NotNil(t, tbl) + + rows := tbl.GetRows() + assert.NotNil(t, rows) + + assertSchema(rows.Schema) + + assert.Len(t, rows.Rows, 1) + assert.Len(t, rows.Rows[0].Values, length) + for _, row := range rows.Rows { + assertValue(row) + } + } +}