diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d499565..3509c51 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -43,6 +43,7 @@ jobs: protoc -I. -Iexamples --plugin=./protoc-gen-bq-schema --bq-schema_out=examples examples/foo.proto protoc -I. -Iexamples --plugin=./protoc-gen-bq-schema --bq-schema_out=examples examples/foo-proto3.proto protoc -I. -Iexamples --plugin=./protoc-gen-bq-schema --bq-schema_out=examples --bq-schema_opt=single-message examples/single_message.proto + protoc -I. -Iexamples --plugin=./protoc-gen-bq-schema --bq-schema_out=examples --bq-schema_opt=enum-as-string examples/enum_as_string.proto - name: Verify examples are working run: | if [ -n "$(git status --porcelain)" ]; then diff --git a/README.md b/README.md index 9eefe3e..9af6443 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ So you can reuse existing data definitions in .proto for BigQuery with this plug ## Installation ```sh -go install github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v2@latest +go install github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v3@latest ``` ## Usage @@ -72,6 +72,8 @@ The message `foo.Baz` is ignored because it doesn't have option `gen_bq_schema.b `protoc --bq-schema_out=. --bq-schema_opt=single-message single_message.proto` will generate a file named `foo/single_message.schema`. The message `foo.Baz` is also ignored because it is not the first message in the file. +`protoc --bq-schema_out=. --bq-schema_opt=enum-as-string foo.proto` will generate a file named `foo/bar_table.schema`. +Fields that are of type enum will be converted to strings in the generated BigQuery schema. ### Support for PolicyTags `protoc-gen-bq-schema` now supports [policyTags](https://cloud.google.com/bigquery/docs/column-level-security-intro). diff --git a/examples/enum_as_string.proto b/examples/enum_as_string.proto new file mode 100644 index 0000000..f2563cc --- /dev/null +++ b/examples/enum_as_string.proto @@ -0,0 +1,12 @@ +syntax = "proto2"; +package foo; +import "bq_table.proto"; + +message Bar { + option (gen_bq_schema.bigquery_opts).table_name = "enum_as_string_table"; + optional Baz f = 6; + + enum Baz { + foo = 1; + } +} \ No newline at end of file diff --git a/examples/foo.proto b/examples/foo.proto index 1481493..d9c1ca8 100644 --- a/examples/foo.proto +++ b/examples/foo.proto @@ -32,6 +32,12 @@ message Bar { } ]; + optional Qux f = 6; + + enum Qux { + foo = 1; + } + optional google.protobuf.Int32Value wkt1 = 11; optional google.protobuf.Timestamp wkt2 = 12; } diff --git a/examples/foo/bar_table.schema b/examples/foo/bar_table.schema index 3cece7a..de97e1d 100644 --- a/examples/foo/bar_table.schema +++ b/examples/foo/bar_table.schema @@ -30,6 +30,11 @@ "mode": "REQUIRED", "description": "TIMESTAMP (uint64 in proto) - required in BigQuery" }, + { + "name": "f", + "type": "INTEGER", + "mode": "NULLABLE" + }, { "name": "wkt1", "type": "INTEGER", diff --git a/examples/foo/enum_as_string_table.schema b/examples/foo/enum_as_string_table.schema new file mode 100644 index 0000000..3c0ddb3 --- /dev/null +++ b/examples/foo/enum_as_string_table.schema @@ -0,0 +1,7 @@ +[ + { + "name": "f", + "type": "STRING", + "mode": "NULLABLE" + } +] \ No newline at end of file diff --git a/go.mod b/go.mod index b2b8d47..62985d5 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ -module github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v2 +module github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v3 go 1.21 require ( - github.com/golang/glog v1.2.3 - google.golang.org/protobuf v1.35.2 + github.com/golang/glog v1.2.4 + google.golang.org/protobuf v1.36.3 ) diff --git a/go.sum b/go.sum index 7dc6d99..c8e12b1 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ -github.com/golang/glog v1.2.3 h1:oDTdz9f5VGVVNGu/Q7UXKWYsD0873HXLHdJUNBsSEKM= -github.com/golang/glog v1.2.3/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= +google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= diff --git a/main.go b/main.go index c60eb60..9e49991 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,7 @@ import ( "fmt" "os" - "github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v2/pkg/converter" + "github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v3/pkg/converter" "github.com/golang/glog" "google.golang.org/protobuf/proto" plugin "google.golang.org/protobuf/types/pluginpb" diff --git a/pkg/converter/convert.go b/pkg/converter/convert.go index 9a3bedf..5937545 100644 --- a/pkg/converter/convert.go +++ b/pkg/converter/convert.go @@ -8,7 +8,7 @@ import ( "sort" "strings" - "github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v2/protos" + "github.com/GoogleCloudPlatform/protoc-gen-bq-schema/v3/protos" "github.com/golang/glog" "google.golang.org/protobuf/encoding/prototext" "google.golang.org/protobuf/proto" @@ -47,7 +47,7 @@ var ( descriptor.FieldDescriptorProto_TYPE_STRING: "STRING", descriptor.FieldDescriptorProto_TYPE_BYTES: "BYTES", - descriptor.FieldDescriptorProto_TYPE_ENUM: "STRING", + descriptor.FieldDescriptorProto_TYPE_ENUM: "INTEGER", descriptor.FieldDescriptorProto_TYPE_BOOL: "BOOLEAN", @@ -401,6 +401,15 @@ func handleSingleMessageOpt(file *descriptor.FileDescriptorProto, requestParam s }) } +// enumAsStringOpt handles --bq-schema_opt=enum-as-string in protoc params. +// providing that param tesll protoc-gen-bq-schema to treat enums as strings. +func enumAsStringOpt(requestParam string) { + if !strings.Contains(requestParam, "enum-as-string") { + return + } + typeFromFieldType[descriptor.FieldDescriptorProto_TYPE_ENUM] = "STRING" +} + func Convert(req *plugin.CodeGeneratorRequest) (*plugin.CodeGeneratorResponse, error) { generateTargets := make(map[string]bool) for _, file := range req.GetFileToGenerate() { @@ -412,6 +421,7 @@ func Convert(req *plugin.CodeGeneratorRequest) (*plugin.CodeGeneratorResponse, e MinimumEdition: proto.Int32(int32(descriptor.Edition_EDITION_PROTO2)), MaximumEdition: proto.Int32(int32(descriptor.Edition_EDITION_MAX)), } + enumAsStringOpt(req.GetParameter()) for _, file := range req.GetProtoFile() { for msgIndex, msg := range file.GetMessageType() { glog.V(1).Infof("Loading a message type %s from package %s", msg.GetName(), file.GetPackage()) diff --git a/pkg/converter/plugin_test.go b/pkg/converter/plugin_test.go index 083c2ed..9b955e6 100644 --- a/pkg/converter/plugin_test.go +++ b/pkg/converter/plugin_test.go @@ -309,8 +309,8 @@ func TestTypes(t *testing.T) { { "name": "bool", "type": "BOOLEAN", "mode": "NULLABLE" }, { "name": "str", "type": "STRING", "mode": "NULLABLE" }, { "name": "bytes", "type": "BYTES", "mode": "NULLABLE" }, - { "name": "enum1", "type": "STRING", "mode": "NULLABLE" }, - { "name": "enum2", "type": "STRING", "mode": "NULLABLE" }, + { "name": "enum1", "type": "INTEGER", "mode": "NULLABLE" }, + { "name": "enum2", "type": "INTEGER", "mode": "NULLABLE" }, { "name": "grp1", "type": "RECORD", "mode": "NULLABLE", "fields": [{ "name": "i1", "type": "INTEGER", "mode": "NULLABLE" }] diff --git a/protos/bq_field.pb.go b/protos/bq_field.pb.go index c4817c2..93bb235 100644 --- a/protos/bq_field.pb.go +++ b/protos/bq_field.pb.go @@ -14,8 +14,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.1 -// protoc v5.26.1 +// protoc-gen-go v1.36.3 +// protoc v5.29.2 // source: bq_field.proto package protos @@ -38,10 +38,7 @@ const ( // Message containing options related to BigQuery schema generation // and management via Protobuf. type BigQueryFieldOptions struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Flag to specify that a field should be marked as 'REQUIRED' when // used to generate schema for BigQuery. Require bool `protobuf:"varint,1,opt,name=require,proto3" json:"require,omitempty"` @@ -63,6 +60,8 @@ type BigQueryFieldOptions struct { // See https://cloud.google.com/bigquery/docs/default-values for possible // values. DefaultValueExpression string `protobuf:"bytes,7,opt,name=default_value_expression,json=defaultValueExpression,proto3" json:"default_value_expression,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BigQueryFieldOptions) Reset() { diff --git a/protos/bq_table.pb.go b/protos/bq_table.pb.go index f13bd91..b538ca9 100644 --- a/protos/bq_table.pb.go +++ b/protos/bq_table.pb.go @@ -14,8 +14,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.1 -// protoc v5.26.1 +// protoc-gen-go v1.36.3 +// protoc v5.29.2 // source: bq_table.proto package protos @@ -83,10 +83,7 @@ func (BigQueryMessageOptions_FieldOrder) EnumDescriptor() ([]byte, []int) { } type BigQueryMessageOptions struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Specifies a name of table in BigQuery for the message. // // If not blank, indicates the message is a type of record to be stored into BigQuery. @@ -100,6 +97,8 @@ type BigQueryMessageOptions struct { // "NULLABLE" by default, different mode may be set via optional suffix ":" ExtraFields []string `protobuf:"bytes,3,rep,name=extra_fields,json=extraFields,proto3" json:"extra_fields,omitempty"` OutputFieldOrder BigQueryMessageOptions_FieldOrder `protobuf:"varint,4,opt,name=output_field_order,json=outputFieldOrder,proto3,enum=gen_bq_schema.BigQueryMessageOptions_FieldOrder" json:"output_field_order,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BigQueryMessageOptions) Reset() {