Dev/virost/feat/proto3 optional (#466)

README.md

@@ -162,10 +162,6 @@ $ protoc googleapis/google/cloud/vision/v1/*.proto \
     --ruby_gapic_opt=configuration=vision.yml --ruby_gapic_out=dest
 ```
 
-Note that generated clients depend on the `gapic-common` 2.0 gem, which is not
-yet released. To test a client, you may need to modify it to pull `gapic-common`
-from the github master branch.
-
 ### Create a custom generator
 If you want to customize the output, you may create a new generator using the
 `gapic-generator` binary installed by the main generator. A new generator takes

gapic-common/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Release History
 
+### 0.3.0 / 2020-06-18
+
+* Update the dependency on google-protobuf to 3.12.2
+
 ### 0.2.1 / 2020-06-02
 
 * Fix a crash when resetting a config field to nil when it has a parent but no default

gapic-common/gapic-common.gemspec

@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = ">= 2.4"
 
-  spec.add_dependency "google-protobuf", "~> 3.2"
+  spec.add_dependency "google-protobuf", "~> 3.12", ">= 3.12.2"
   spec.add_dependency "googleapis-common-protos", ">= 1.3.9", "< 2.0"
   spec.add_dependency "googleapis-common-protos-types", ">= 1.0.4", "< 2.0"
   spec.add_dependency "googleauth", "~> 0.9"

gapic-common/lib/gapic/common/version.rb

@@ -14,6 +14,6 @@
 
 module Gapic
   module Common
-    VERSION = "0.2.1".freeze
+    VERSION = "0.3.0".freeze
   end
 end

gapic-generator-ads/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Release History for gapic-generator-ads
 
+### 0.6.2 / 2020-06-18
+
+* Includes changes from gapic-generator 0.6.2.
+
 ### 0.6.1 / 2020-06-16
 
 * Includes changes from gapic-generator 0.6.1.

gapic-generator-ads/Dockerfile

@@ -54,7 +54,7 @@ RUN if [ -f gapic-generator.gem ]; then \
     ;fi
 
 # Install the subgenerator and other needed tools.
-RUN gem install grpc-tools gapic-generator-ads.gem \
+RUN gem install --pre grpc-tools gapic-generator-ads.gem \
     && rm gapic-generator-ads.gem \
     && mkdir -p --mode=777 /.cache
 

gapic-generator-ads/docker-entrypoint.sh

@@ -27,6 +27,7 @@ done
 
 mkdir -p /workspace/out/lib
 grpc_tools_ruby_protoc \
+  --experimental_allow_proto3_optional=1 \
   --proto_path=/workspace/common-protos/ --proto_path=/workspace/in/ \
   --ruby_out=/workspace/out/lib \
   --grpc_out=/workspace/out/lib \

gapic-generator-ads/gapic-generator-ads.gemspec

@@ -40,12 +40,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "actionpack", "~> 5.2"
-  spec.add_dependency "gapic-generator", "= 0.6.1"
+  spec.add_dependency "gapic-generator", "= 0.6.2"
   spec.add_dependency "protobuf", "~> 3.8"
 
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "google-style", "~> 1.24.0"
-  spec.add_development_dependency "grpc-tools", "~> 1.19"
+  spec.add_development_dependency "grpc-tools", "~> 1.30.0.pre1"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "minitest-focus", "~> 1.0"
   spec.add_development_dependency "rake", ">= 12.0"

gapic-generator-ads/lib/gapic/generator/ads/version.rb

@@ -18,7 +18,7 @@
 module Gapic
   module Generator
     module Ads
-      VERSION = "0.6.1"
+      VERSION = "0.6.2"
     end
   end
 end

gapic-generator-cloud/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Release History for gapic-generator-cloud
 
+### 0.6.2 / 2020-06-18
+
+* Includes changes from gapic-generator 0.6.2.
+
 ### 0.6.1 / 2020-06-16
 
 * Includes changes from gapic-generator 0.6.1.

gapic-generator-cloud/gapic-generator-cloud.gemspec

@@ -39,12 +39,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "actionpack", "~> 5.2"
-  spec.add_dependency "gapic-generator", "= 0.6.1"
+  spec.add_dependency "gapic-generator", "= 0.6.2"
   spec.add_dependency "google-style", "~> 1.24.0"
   spec.add_dependency "protobuf", "~> 3.8"
 
   spec.add_development_dependency "bundler", "~> 2.1"
-  spec.add_development_dependency "grpc-tools", "~> 1.19"
+  spec.add_development_dependency "grpc-tools", "~> 1.30.0.pre1"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "minitest-autotest", "~> 1.0"
   spec.add_development_dependency "minitest-focus", "~> 1.0"

gapic-generator-cloud/lib/gapic/generator/cloud/version.rb

@@ -18,7 +18,7 @@
 module Gapic
   module Generator
     module Cloud
-      VERSION = "0.6.1"
+      VERSION = "0.6.2"
     end
   end
 end

gapic-generator-cloud/templates/cloud/wrapper_gem/gemfile.erb

@@ -6,6 +6,3 @@ gemspec
 <%- gem.versioned_gems.each do |name| -%>
 gem "<%= name %>", path: "../<%= name %>"
 <%- end -%>
-
-# google-protobuf 3.12.0 requires Ruby 2.5 or later, so pin to 3.11 on older Rubies
-gem "google-protobuf", (RUBY_VERSION < "2.5" ? "~> 3.11.4" : "~> 3.12")

gapic-generator/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Release History for gapic-generator
 
+### 0.6.2 / 2020-06-18
+
+* Support for the proto3_optional
+* Fixed an issue where tests for the oneof fields were not generating correctly
+* Removed ruby <2.5 pin for the protobuf dependency since new protobuf supports it again
+* Generated libraries now depend on gapic-common 0.3
+
 ### 0.6.1 / 2020-06-16
 
 * Add auto-generated disclaimer to generated tests.

gapic-generator/Dockerfile

@@ -41,7 +41,7 @@ COPY --from=gcr.io/gapic-images/api-common-protos:latest /protos/ /workspace/com
 COPY --from=builder /workspace/*.gem /workspace/
 
 # Install the generator and other needed tools.
-RUN gem install grpc-tools ./gapic-generator.gem \
+RUN gem install --pre grpc-tools ./gapic-generator.gem \
     && rm gapic-generator.gem \
     && mkdir -p --mode=777 /.cache
 

gapic-generator/Rakefile

@@ -169,6 +169,7 @@ def generate_default_templates service
 
   protoc_cmd = [
     "grpc_tools_ruby_protoc",
+    "--experimental_allow_proto3_optional=1",
     "-I ../shared/api-common-protos",
     "-I ../shared/googleapis",
     "-I ../shared/protos",

gapic-generator/docker-entrypoint.sh

@@ -27,6 +27,7 @@ done
 
 mkdir -p /workspace/out/lib
 exec grpc_tools_ruby_protoc \
+        --experimental_allow_proto3_optional=1 \
         --proto_path=/workspace/common-protos/ --proto_path=/workspace/in/ \
         --ruby_out=/workspace/out/lib \
         --grpc_out=/workspace/out/lib \

gapic-generator/gapic-generator.gemspec

@@ -43,7 +43,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "google-style", "~> 1.24.0"
-  spec.add_development_dependency "grpc-tools", "~> 1.19"
+  spec.add_development_dependency "grpc-tools", "~> 1.30.0.pre1"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "minitest-autotest", "~> 1.0"
   spec.add_development_dependency "minitest-focus", "~> 1.0"

gapic-generator/gem_templates/dockerfile.erb

@@ -30,7 +30,7 @@ COPY --from=gcr.io/gapic-images/api-common-protos:beta /protos/ /workspace/commo
 COPY --from=builder /workspace/*.gem /workspace/
 
 # Install the subgenerator and other needed tools.
-RUN gem install grpc-tools gapic-generator-<%= gem_name %>.gem \
+RUN gem install --pre grpc-tools gapic-generator-<%= gem_name %>.gem \
     && rm gapic-generator-<%= gem_name %>.gem \
     && mkdir -p --mode=777 /.cache
 

gapic-generator/gem_templates/entrypoint.erb

@@ -16,6 +16,7 @@ done
 
 mkdir -p /workspace/out/lib
 exec grpc_tools_ruby_protoc \
+        --experimental_allow_proto3_optional=1 \
         --proto_path=/workspace/common-protos/ --proto_path=/workspace/in/ \
         --ruby_out=/workspace/out/lib \
         --grpc_out=/workspace/out/lib \

gapic-generator/lib/gapic/generator/version.rb

@@ -16,6 +16,6 @@
 
 module Gapic
   module Generator
-    VERSION = "0.6.1"
+    VERSION = "0.6.2"
   end
 end

gapic-generator/lib/gapic/presenters/field_presenter.rb

@@ -80,6 +80,10 @@ def enum?
         @field.enum?
       end
 
+      def proto3_optional?
+        @field.proto3_optional?
+      end
+
       def repeated?
         @field.repeated?
       end
@@ -92,6 +96,10 @@ def oneof?
         @field.oneof?
       end
 
+      def oneof_name
+        @message.oneof_decl[@field.oneof_index].name
+      end
+
       protected
 
       def field_doc_types field, output

gapic-generator/lib/gapic/presenters/gem_presenter.rb

@@ -174,7 +174,7 @@ def extra_files
       end
 
       def dependencies
-        deps = { "gapic-common" => "~> 0.2" }
+        deps = { "gapic-common" => "~> 0.3" }
         deps["grpc-google-iam-v1"] = [">= 0.6.10", "< 2.0"] if iam_dependency?
         extra_deps = gem_config :extra_dependencies
         deps.merge! extra_deps if extra_deps

gapic-generator/lib/gapic/runner.rb

@@ -48,8 +48,10 @@ def run generator_type: nil
       output_files = generator.new(api).generate
 
       # Create and write the response
-      Google::Protobuf::Compiler::CodeGeneratorResponse.new \
+      response = Google::Protobuf::Compiler::CodeGeneratorResponse.new \
         file: output_files
+      response.supported_features = Google::Protobuf::Compiler::CodeGeneratorResponse::FEATURE_PROTO3_OPTIONAL
+      response
     end
 
     # Run protoc generation.

gapic-generator/lib/gapic/schema/wrappers.rb

@@ -740,7 +740,8 @@ def immutable?
         :default_value,
         :oneof_index,
         :json_name,
-        :options
+        :options,
+        :proto3_optional?
       )
     end
 

gapic-generator/lib/google/protobuf/compiler/plugin.pb.rb

@@ -1,7 +1,8 @@
 # encoding: utf-8
 
 ##
-# This file is auto-generated. DO NOT EDIT!
+# This file was auto-generated initially but now is hand-edited to add support 
+# for response features and proto3_optional
 #
 require 'protobuf'
 
@@ -54,13 +55,16 @@ class CodeGeneratorRequest
       end
 
       class CodeGeneratorResponse
+        FEATURE_PROTO3_OPTIONAL = 1
+
         class File
           optional :string, :name, 1
           optional :string, :insertion_point, 2
           optional :string, :content, 15
         end
 
         optional :string, :error, 1
+        optional :int32, :supported_features, 2
         repeated ::Google::Protobuf::Compiler::CodeGeneratorResponse::File, :file, 15
       end
 

gapic-generator/lib/google/protobuf/descriptor.pb.rb

@@ -190,6 +190,7 @@ class FieldDescriptorProto
       optional :int32, :oneof_index, 9
       optional :string, :json_name, 10
       optional ::Google::Protobuf::FieldOptions, :options, 8
+      optional :bool, :proto3_optional, 17
     end
 
     class OneofDescriptorProto

gapic-generator/templates/default/gem/gemfile.erb

@@ -2,6 +2,3 @@
 source "https://rubygems.org"
 
 gemspec
-
-# google-protobuf 3.12.0 requires Ruby 2.5 or later, so pin to 3.11 on older Rubies
-gem "google-protobuf", (RUBY_VERSION < "2.5" ? "~> 3.11.4" : "~> 3.12")

gapic-generator/templates/default/service/test/method/_bidi.erb

@@ -93,6 +93,12 @@ def test_<%= method.name %>
         <%- else -%>
         assert_equal <%= field.default_value %>, r.<%= field.name %>
         <%- end -%>
+        <%- if field.oneof? && !field.proto3_optional? -%>
+        assert_equal :<%= field.name %>, r.<%= field.oneof_name %>
+        <%- end -%>
+        <%- if field.proto3_optional? -%>
+        assert r.has_<%= field.name %>?
+        <%- end -%>
         <%- end -%>
       end
     end

gapic-generator/templates/default/service/test/method/_client.erb

@@ -77,6 +77,12 @@ def test_<%= method.name %>
         <%- else -%>
         assert_equal <%= field.default_value %>, r.<%= field.name %>
         <%- end -%>
+        <%- if field.oneof? && !field.proto3_optional? -%>
+        assert_equal :<%= field.name %>, r.<%= field.oneof_name %>
+        <%- end -%>
+        <%- if field.proto3_optional? -%>
+        assert r.has_<%= field.name %>?
+        <%- end -%>
         <%- end -%>
       end
     end

gapic-generator/templates/default/service/test/method/_normal.erb

@@ -26,6 +26,12 @@ def test_<%= method.name %>
     <%- else -%>
     assert_equal <%= field.default_value %>, request.<%= field.name %>
     <%- end -%>
+    <%- if field.oneof? && !field.proto3_optional? -%>
+    assert_equal :<%= field.name %>, request.<%= field.oneof_name %>
+    <%- end -%>
+    <%- if field.proto3_optional? -%>
+    assert request.has_<%= field.name %>?
+    <%- end -%>
     <%- end -%>
     refute_nil options
   end

gapic-generator/templates/default/service/test/method/_server.erb

@@ -24,6 +24,12 @@ def test_<%= method.name %>
     <%- else -%>
     assert_equal <%= field.default_value %>, request.<%= field.name %>
     <%- end -%>
+    <%- if field.oneof? && !field.proto3_optional? -%>
+    assert_equal :<%= field.name %>, request.<%= field.oneof_name %>
+    <%- end -%>
+    <%- if field.proto3_optional? -%>
+    assert request.has_<%= field.name %>?
+    <%- end -%>
     <%- end -%>
     refute_nil options
   end

gapic-generator/test/gapic/annotations/enum_test.rb

@@ -22,7 +22,7 @@ def test_enum
     message = garbage.messages.find { |s| s.name == "TypicalGarbage" }
     refute_nil message
 
-    assert_equal 21, message.fields.count
+    assert_equal 22, message.fields.count
 
     assert_equal "enum", message.fields[12].name
     assert_equal Google::Protobuf::FieldDescriptorProto::Type::TYPE_ENUM, message.fields[12].type

gapic-generator/test/gapic/annotations/field_test.rb

@@ -22,7 +22,7 @@ def test_garbage_TypicalGarbage
     message = garbage.messages.find { |s| s.name == "TypicalGarbage" }
     refute_nil message
 
-    assert_equal 21, message.fields.count
+    assert_equal 22, message.fields.count
 
     assert_equal "name", message.fields[0].name
     refute_nil message.fields[0].options

gapic-generator/test/gapic/presenters/field_presenter_showcase_test.rb

@@ -37,6 +37,21 @@ def test_showcase_User_name_field
     assert_nil fp.type_name_full
   end
 
+  def test_showcase_User_name_field
+    fp = field_presenter :showcase, "google.showcase.v1beta1.User", "age"
+
+    assert_equal "age", fp.name
+    assert_equal "@!attribute [rw] age", fp.doc_attribute_type
+    assert_equal "::Integer", fp.output_doc_types
+    assert_equal "The age of the use in years.\n", fp.doc_description
+    assert_equal "42", fp.default_value
+    assert_equal "", fp.type_name
+    assert_nil fp.type_name_full
+    assert fp.proto3_optional?
+    assert fp.oneof?
+    assert_equal "_age", fp.oneof_name
+  end
+
   def test_showcase_User_create_time_field
     fp = field_presenter :showcase, "google.showcase.v1beta1.User", "create_time"