From 795d219a226add8730396188fc274b05b95cf02b Mon Sep 17 00:00:00 2001 From: Eduardo Ramirez Date: Wed, 20 Jun 2018 01:38:47 -0700 Subject: [PATCH] support parsing proto syntax 3 --- lib/descriptors/ProtoDescriptor.js | 1 + lib/parser.js | 34 +++++++---- tests/parsing_test.js | 94 +++++++++++++++++++++++++++--- tests/protos/common.proto | 1 + tests/protos/conventions.proto | 3 +- tests/protos/inner.proto | 3 +- tests/protos/kitchen-sink-v3.proto | 78 +++++++++++++++++++++++++ tests/protos/loop.proto | 1 + tests/protos/options.proto | 3 +- tests/protos/otherOptions.proto | 2 + tests/protos/person.proto | 2 + tests/protos/services.proto | 3 +- tests/protos/super-person.proto | 2 + tests/protos/vehicle.proto | 2 + 14 files changed, 207 insertions(+), 22 deletions(-) create mode 100644 tests/protos/kitchen-sink-v3.proto diff --git a/lib/descriptors/ProtoDescriptor.js b/lib/descriptors/ProtoDescriptor.js index 3555e1f..e634390 100644 --- a/lib/descriptors/ProtoDescriptor.js +++ b/lib/descriptors/ProtoDescriptor.js @@ -17,6 +17,7 @@ function ProtoDescriptor(filePath) { Descriptor.call(this) this._filePath = filePath + this._syntax = 'proto2' this._package = '' this._options = {} this._importNames = [] diff --git a/lib/parser.js b/lib/parser.js index 49ff041..b8b03b1 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -14,8 +14,8 @@ var FieldDescriptor = require('./descriptors/FieldDescriptor') var MethodDescriptor = require('./descriptors/MethodDescriptor') var ParseError = require('./errors').ParseError var FieldType = require('./FieldType') -var fieldTypeValues = {} +var fieldTypeValues = {} Object.keys(FieldType).forEach(function (k) { fieldTypeValues[FieldType[k]] = true }) @@ -139,6 +139,12 @@ module.exports = function parser(identifier, string) { case 'enum': parseEnum(message) break + case 'oneof': + parseOneof(message) + break + case 'extensions': + parseExtensions(proto) + break case 'required': parseRequiredField(message) break @@ -148,14 +154,13 @@ module.exports = function parser(identifier, string) { case 'repeated': parseRepeatedField(message) break - case 'oneof': - parseOneof(message) - break - case 'extensions': - parseExtensions(proto) - break default: - throw new ParseError(identifier, token) + if (proto.getSyntax() == 'proto3') { + tokens.unshift(token) + parseOptionalField(message) + } else { + throw new ParseError(identifier, token) + } } }) } @@ -251,7 +256,10 @@ module.exports = function parser(identifier, string) { expect(Token.Type.TERMINATOR) } - // Parses an optional field: optional protoType name = tag [default = 123]; + // Parses an optional field: + // optional protoType name = tag [default = 123]; + // or in proto 3 + // protoType name = tag; function parseOptionalField(parent) { var field = parseField(parent) field.setOptional(true) @@ -298,7 +306,12 @@ module.exports = function parser(identifier, string) { parseRepeatedField(extend) break default: - throw new ParseError(identifier, token) + if (proto.getSyntax() == 'proto3') { + tokens.unshift(token) + parseOptionalField(extend) + } else { + throw new ParseError(identifier, token) + } } }) } @@ -418,5 +431,4 @@ module.exports = function parser(identifier, string) { function peek() { return tokens[0] } - } diff --git a/tests/parsing_test.js b/tests/parsing_test.js index b80da40..cb501f9 100644 --- a/tests/parsing_test.js +++ b/tests/parsing_test.js @@ -74,14 +74,14 @@ exports.testKitchenSinkParsing = function (test) { // Test service parsing test.equal(proto.getServices().length, 1) - var service = proto.getService("WhatTheSinkCanDo") + var service = proto.getService('WhatTheSinkCanDo') test.equal(service.getMethods().length, 3) - test.equal(service.getMethods()[0].getName(), "DisposeLoudly") - test.equal(service.getMethods()[0].getRawInputType(), "ThisIsTheKitchenSink") - test.equal(service.getMethods()[0].getRawOutputType(), "examples.Color") - test.equal(service.getMethods()[0].getBaseOutputType(), "Color") - test.equal(service.getMethods()[1].getName(), "RinseQuietly") - test.equal(service.getMethods()[2].getName(), "HoldDishes") + test.equal(service.getMethods()[0].getName(), 'DisposeLoudly') + test.equal(service.getMethods()[0].getRawInputType(), 'ThisIsTheKitchenSink') + test.equal(service.getMethods()[0].getRawOutputType(), 'examples.Color') + test.equal(service.getMethods()[0].getBaseOutputType(), 'Color') + test.equal(service.getMethods()[1].getName(), 'RinseQuietly') + test.equal(service.getMethods()[2].getName(), 'HoldDishes') test.equal(service.getMethods()[0].getOption('method_option'), 'gargle') test.equal(service.getMethods()[1].getOption('method_option'), 'shhhhh') test.ok(!service.getMethods()[2].getOption('method_option')) @@ -90,6 +90,86 @@ exports.testKitchenSinkParsing = function (test) { } +exports.testKitchenSinkParsingV3 = function (test) { + var proto = parseFile('kitchen-sink-v3.proto') + + test.equal(proto.getPackage(), 'some_newer_package') + + test.equal(proto.getSyntax(), 'proto3') + + // Test imports. + test.equal(proto.getImportNames().length, 4) + test.equal(proto.getImportNames()[0], 'protos/options.proto') + + // Test proto level options. + test.equal(proto.getOptionKeys().length, 2) + test.equal(proto.getOption('file_level_option'), 'string value') + test.equal(proto.getOption('another_option'), 'Just "testing" that strings parse.') + + // Test message level options. + test.equals(proto.getMessage('SomeCoolMessage').getOption('message_level_option'), 'XYZ') + + // Test messages. + test.equal(proto.getMessages().length, 2) + test.ok(!!proto.getMessage('SomeCoolMessage').getMessage('MessagesWithinMessages')) + test.ok(!!proto.getMessage('SomeCoolMessage') + .getMessage('MessagesWithinMessages') + .getEnum('EnumInsideMessageInsideMessage')) + + // Test fields. + var msg = proto.getMessage('ThisIsTheKitchenSinkV3') + test.equal(msg.getFields().length, 12) + test.ok(msg.getField('some_field').isOptional()) + test.ok(!msg.getField('some_field').isRepeated()) + test.ok(!msg.getField('repeated_field').isOptional()) + test.ok(msg.getField('repeated_field').isRepeated()) + + test.equal(msg.getField('some_field').getType(), 'string') + test.equal(msg.getField('number_field').getType(), 'number') + test.equal(msg.getField('repeated_field').getType(), 'boolean') + test.equal(msg.getField('using_another_message').getType(), 'SomeCoolMessage') + test.equal(msg.getField('color_field').getType(), 'examples.Color') + test.equal(msg.getField('color_field').getBaseType(), 'Color') + test.equal(msg.getField('other_field_option').getOption('other.field.field_name'), 'special') + + test.equal(msg.getOneof('oneof_name').getOneofIndex(), 0) + test.ok(msg.getField('oneof_field_normal').isOptional()) + test.ok(msg.getField('oneof_field_with_option').isOptional()) + test.ok(msg.getField('oneof_color_field').isOptional()) + test.equal(msg.getField('oneof_field_normal').getType(), 'number') + test.equal(msg.getField('oneof_field_with_option').getOption('other.field.field_name'), 'specialty') + test.equal(msg.getField('oneof_color_field').getType(), 'examples.Color') + test.equal(msg.getField('oneof_color_field').getBaseType(), 'Color') + + test.equal(msg.getField('sherlock_lives_at_221b').getType(), 'boolean') + test.equal(msg.getField('call_867_5309').getType(), 'boolean') + + // Test service parsing + test.equal(proto.getServices().length, 1) + var service = proto.getService('WhatTheSinkV3CanDo') + test.equal(service.getMethods().length, 3) + test.equal(service.getMethods()[0].getName(), 'DisposeLoudly') + test.equal(service.getMethods()[0].getRawInputType(), 'ThisIsTheKitchenSinkV3') + test.equal(service.getMethods()[0].getRawOutputType(), 'examples.Color') + test.equal(service.getMethods()[0].getBaseOutputType(), 'Color') + test.equal(service.getMethods()[1].getName(), 'RinseQuietly') + test.equal(service.getMethods()[2].getName(), 'HoldDishes') + test.equal(service.getMethods()[0].getOption('method_option'), 'gargle') + test.equal(service.getMethods()[1].getOption('method_option'), 'shhhhh') + test.ok(!service.getMethods()[2].getOption('method_option')) + test.equal(service.getOption('service_level_option'), 'serviceOption') + + test.done() +} + + +exports.testBadProto_incorrectSyntax = function (test) { + assertFails('syntax = "proto2"; message SyntaxThree { string first = 1; string second = 1; }', + 'Unexpected token in', + 'Protos using incorrect syntax according to version should fail', test) +} + + exports.testBadProto_duplicateTags = function (test) { assertFails('message Dupe { required string first = 1; required string second = 1; }', 'duplicate tag in', diff --git a/tests/protos/common.proto b/tests/protos/common.proto index 52ba3bf..4aa2521 100644 --- a/tests/protos/common.proto +++ b/tests/protos/common.proto @@ -1,3 +1,4 @@ +syntax = "proto2"; package examples; diff --git a/tests/protos/conventions.proto b/tests/protos/conventions.proto index 8d59f07..dd08759 100644 --- a/tests/protos/conventions.proto +++ b/tests/protos/conventions.proto @@ -1,3 +1,4 @@ +syntax = "proto2"; package conventions; @@ -13,4 +14,4 @@ message Casing { enum CasingEnum { TWO_WORDS = 1; -} \ No newline at end of file +} diff --git a/tests/protos/inner.proto b/tests/protos/inner.proto index 6b509c5..5c4a12a 100644 --- a/tests/protos/inner.proto +++ b/tests/protos/inner.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + package burrito; message Tortilla { @@ -19,4 +21,3 @@ message Tortilla { message Avocado { } - diff --git a/tests/protos/kitchen-sink-v3.proto b/tests/protos/kitchen-sink-v3.proto new file mode 100644 index 0000000..d0ae382 --- /dev/null +++ b/tests/protos/kitchen-sink-v3.proto @@ -0,0 +1,78 @@ +// Proto file that tries to exercise all valid proto features. + +/** + * Both comment types should be supported and ignore other tokens such as message FooBar { }. + */ + +syntax = "proto3"; + +package some_newer_package; + +import "protos/options.proto"; +import "protos/otherOptions.proto"; +import "protos/common.proto"; +import "google/protobuf/descriptor.proto"; + +option (file_level_option) = "string value"; +option (another_option) = "Just \"testing\" that strings parse."; + +message ThisIsTheKitchenSinkV3 { + string some_field = 1; + repeated bool repeated_field = 2; + SomeCoolMessage using_another_message = 3; + + int32 number_field = 4; + string string_field = 5; + examples.Color color_field = 6; + + oneof oneof_name { + int32 oneof_field_normal = 7; + string oneof_field_with_option = 8 [(other.field).field_name='specialty']; + examples.Color oneof_color_field = 9; + } + + bool sherlock_lives_at_221b = 10; + bool call_867_5309 = 11; + string other_field_option = 12 [(other.field).field_name='special']; +} + +message SomeCoolMessage { + option (message_level_option) = "XYZ"; + int64 id = 1000 [(option)=1, (something_else)="foobar"]; + string x = 1; string y = 2; string z = 3; + + message MessagesWithinMessages { + bool done = 1; + + enum EnumInsideMessageInsideMessage { + V3_MAYBE = 0; + V3_YES = 1; + V3_NO = 2; + } + } +} + +service WhatTheSinkV3CanDo { + option (service_level_option) = "serviceOption"; + + rpc DisposeLoudly (ThisIsTheKitchenSinkV3) returns (examples.Color) { + option (method_option) = "gargle"; + } + + rpc RinseQuietly (ThisIsTheKitchenSinkV3) returns (examples.Color) { + option (method_option) = "shhhhh"; + } + + rpc HoldDishes (ThisIsTheKitchenSinkV3) returns (examples.Color); +} + +extend google.protobuf.MessageOptions { + string another_message_type = 50005; +} + +enum ColorsInWheel { + COLOR_UNKNOWN = 0; + COLOR_BLACK = 1; + COLOR_RED = 2; + COLOR_BLUE = 3; +} diff --git a/tests/protos/loop.proto b/tests/protos/loop.proto index 9701df2..32272ba 100644 --- a/tests/protos/loop.proto +++ b/tests/protos/loop.proto @@ -1,3 +1,4 @@ +syntax = "proto2"; message TweedleDee { optional TweedleDum dum = 1; diff --git a/tests/protos/options.proto b/tests/protos/options.proto index 94799e4..2e67c0c 100644 --- a/tests/protos/options.proto +++ b/tests/protos/options.proto @@ -1,4 +1,5 @@ // Copyright 2015. A Medium Corporation. +syntax = "proto2"; import "google/protobuf/descriptor.proto"; @@ -22,4 +23,4 @@ extend google.protobuf.MethodOptions { extend google.protobuf.ServiceOptions { optional string service_level_option = 20170125; -} \ No newline at end of file +} diff --git a/tests/protos/otherOptions.proto b/tests/protos/otherOptions.proto index 7cce854..3ba9067 100644 --- a/tests/protos/otherOptions.proto +++ b/tests/protos/otherOptions.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + package other; import "google/protobuf/descriptor.proto"; diff --git a/tests/protos/person.proto b/tests/protos/person.proto index 541a7dc..5c6cf63 100644 --- a/tests/protos/person.proto +++ b/tests/protos/person.proto @@ -2,6 +2,8 @@ * Proto definition for a Person */ +syntax = "proto2"; + import "protos/common.proto"; package examples; diff --git a/tests/protos/services.proto b/tests/protos/services.proto index 2bbc24a..113bd80 100644 --- a/tests/protos/services.proto +++ b/tests/protos/services.proto @@ -1,3 +1,4 @@ +syntax = "proto2"; package shoes; @@ -14,4 +15,4 @@ message FullShoe { service RunningShoe { rpc LaceShoe (Shoe) returns (FullShoe); rpc Stride (Shoe) returns (FullShoe); -} \ No newline at end of file +} diff --git a/tests/protos/super-person.proto b/tests/protos/super-person.proto index 9c0f4e0..a151b86 100644 --- a/tests/protos/super-person.proto +++ b/tests/protos/super-person.proto @@ -2,6 +2,8 @@ * Extension of person */ +syntax = "proto2"; + import "protos/person.proto"; package examples; diff --git a/tests/protos/vehicle.proto b/tests/protos/vehicle.proto index b1fb4e3..e062297 100644 --- a/tests/protos/vehicle.proto +++ b/tests/protos/vehicle.proto @@ -1,5 +1,7 @@ // Proto definition for a Car +syntax = "proto2"; + import "protos/common.proto"; import "protos/person.proto";