Merge pull request #69 from Medium/eduardoramirez/parse-proto-3

support parsing proto syntax 3

lib/descriptors/ProtoDescriptor.js

@@ -17,6 +17,7 @@ function ProtoDescriptor(filePath) {
   Descriptor.call(this)
 
   this._filePath = filePath
+  this._syntax = 'proto2'
   this._package = ''
   this._options = {}
   this._importNames = []

lib/parser.js

@@ -14,8 +14,8 @@ var FieldDescriptor = require('./descriptors/FieldDescriptor')
 var MethodDescriptor = require('./descriptors/MethodDescriptor')
 var ParseError = require('./errors').ParseError
 var FieldType = require('./FieldType')
-var fieldTypeValues = {}
 
+var fieldTypeValues = {}
 Object.keys(FieldType).forEach(function (k) {
   fieldTypeValues[FieldType[k]] = true
 })
@@ -139,6 +139,12 @@ module.exports = function parser(identifier, string) {
         case 'enum':
           parseEnum(message)
           break
+        case 'oneof':
+          parseOneof(message)
+          break
+        case 'extensions':
+          parseExtensions(proto)
+          break
         case 'required':
           parseRequiredField(message)
           break
@@ -148,14 +154,13 @@ module.exports = function parser(identifier, string) {
         case 'repeated':
           parseRepeatedField(message)
           break
-        case 'oneof':
-          parseOneof(message)
-          break
-        case 'extensions':
-          parseExtensions(proto)
-          break
         default:
-          throw new ParseError(identifier, token)
+          if (proto.getSyntax() == 'proto3') {
+            tokens.unshift(token)
+            parseOptionalField(message)
+          } else {
+            throw new ParseError(identifier, token)
+          }
       }
     })
   }
@@ -251,7 +256,10 @@ module.exports = function parser(identifier, string) {
     expect(Token.Type.TERMINATOR)
   }
 
-  // Parses an optional field: optional protoType name = tag [default = 123];
+  // Parses an optional field:
+  //   optional protoType name = tag [default = 123];
+  // or in proto 3
+  //   protoType name = tag;
   function parseOptionalField(parent) {
     var field = parseField(parent)
     field.setOptional(true)
@@ -298,7 +306,12 @@ module.exports = function parser(identifier, string) {
           parseRepeatedField(extend)
           break
         default:
-          throw new ParseError(identifier, token)
+          if (proto.getSyntax() == 'proto3') {
+            tokens.unshift(token)
+            parseOptionalField(extend)
+          } else {
+            throw new ParseError(identifier, token)
+          }
       }
     })
   }
@@ -418,5 +431,4 @@ module.exports = function parser(identifier, string) {
   function peek() {
     return tokens[0]
   }
-
 }

tests/parsing_test.js

@@ -74,14 +74,14 @@ exports.testKitchenSinkParsing = function (test) {
 
   // Test service parsing
   test.equal(proto.getServices().length, 1)
-  var service = proto.getService("WhatTheSinkCanDo")
+  var service = proto.getService('WhatTheSinkCanDo')
   test.equal(service.getMethods().length, 3)
-  test.equal(service.getMethods()[0].getName(), "DisposeLoudly")
-  test.equal(service.getMethods()[0].getRawInputType(), "ThisIsTheKitchenSink")
-  test.equal(service.getMethods()[0].getRawOutputType(), "examples.Color")
-  test.equal(service.getMethods()[0].getBaseOutputType(), "Color")
-  test.equal(service.getMethods()[1].getName(), "RinseQuietly")
-  test.equal(service.getMethods()[2].getName(), "HoldDishes")
+  test.equal(service.getMethods()[0].getName(), 'DisposeLoudly')
+  test.equal(service.getMethods()[0].getRawInputType(), 'ThisIsTheKitchenSink')
+  test.equal(service.getMethods()[0].getRawOutputType(), 'examples.Color')
+  test.equal(service.getMethods()[0].getBaseOutputType(), 'Color')
+  test.equal(service.getMethods()[1].getName(), 'RinseQuietly')
+  test.equal(service.getMethods()[2].getName(), 'HoldDishes')
   test.equal(service.getMethods()[0].getOption('method_option'), 'gargle')
   test.equal(service.getMethods()[1].getOption('method_option'), 'shhhhh')
   test.ok(!service.getMethods()[2].getOption('method_option'))
@@ -90,6 +90,86 @@ exports.testKitchenSinkParsing = function (test) {
 }
 
 
+exports.testKitchenSinkParsingV3 = function (test) {
+  var proto = parseFile('kitchen-sink-v3.proto')
+
+  test.equal(proto.getPackage(), 'some_newer_package')
+
+  test.equal(proto.getSyntax(), 'proto3')
+
+  // Test imports.
+  test.equal(proto.getImportNames().length, 4)
+  test.equal(proto.getImportNames()[0], 'protos/options.proto')
+
+  // Test proto level options.
+  test.equal(proto.getOptionKeys().length, 2)
+  test.equal(proto.getOption('file_level_option'), 'string value')
+  test.equal(proto.getOption('another_option'), 'Just "testing" that strings parse.')
+
+  // Test message level options.
+  test.equals(proto.getMessage('SomeCoolMessage').getOption('message_level_option'), 'XYZ')
+
+  // Test messages.
+  test.equal(proto.getMessages().length, 2)
+  test.ok(!!proto.getMessage('SomeCoolMessage').getMessage('MessagesWithinMessages'))
+  test.ok(!!proto.getMessage('SomeCoolMessage')
+      .getMessage('MessagesWithinMessages')
+      .getEnum('EnumInsideMessageInsideMessage'))
+
+  // Test fields.
+  var msg = proto.getMessage('ThisIsTheKitchenSinkV3')
+  test.equal(msg.getFields().length, 12)
+  test.ok(msg.getField('some_field').isOptional())
+  test.ok(!msg.getField('some_field').isRepeated())
+  test.ok(!msg.getField('repeated_field').isOptional())
+  test.ok(msg.getField('repeated_field').isRepeated())
+
+  test.equal(msg.getField('some_field').getType(), 'string')
+  test.equal(msg.getField('number_field').getType(), 'number')
+  test.equal(msg.getField('repeated_field').getType(), 'boolean')
+  test.equal(msg.getField('using_another_message').getType(), 'SomeCoolMessage')
+  test.equal(msg.getField('color_field').getType(), 'examples.Color')
+  test.equal(msg.getField('color_field').getBaseType(), 'Color')
+  test.equal(msg.getField('other_field_option').getOption('other.field.field_name'), 'special')
+
+  test.equal(msg.getOneof('oneof_name').getOneofIndex(), 0)
+  test.ok(msg.getField('oneof_field_normal').isOptional())
+  test.ok(msg.getField('oneof_field_with_option').isOptional())
+  test.ok(msg.getField('oneof_color_field').isOptional())
+  test.equal(msg.getField('oneof_field_normal').getType(), 'number')
+  test.equal(msg.getField('oneof_field_with_option').getOption('other.field.field_name'), 'specialty')
+  test.equal(msg.getField('oneof_color_field').getType(), 'examples.Color')
+  test.equal(msg.getField('oneof_color_field').getBaseType(), 'Color')
+
+  test.equal(msg.getField('sherlock_lives_at_221b').getType(), 'boolean')
+  test.equal(msg.getField('call_867_5309').getType(), 'boolean')
+
+  // Test service parsing
+  test.equal(proto.getServices().length, 1)
+  var service = proto.getService('WhatTheSinkV3CanDo')
+  test.equal(service.getMethods().length, 3)
+  test.equal(service.getMethods()[0].getName(), 'DisposeLoudly')
+  test.equal(service.getMethods()[0].getRawInputType(), 'ThisIsTheKitchenSinkV3')
+  test.equal(service.getMethods()[0].getRawOutputType(), 'examples.Color')
+  test.equal(service.getMethods()[0].getBaseOutputType(), 'Color')
+  test.equal(service.getMethods()[1].getName(), 'RinseQuietly')
+  test.equal(service.getMethods()[2].getName(), 'HoldDishes')
+  test.equal(service.getMethods()[0].getOption('method_option'), 'gargle')
+  test.equal(service.getMethods()[1].getOption('method_option'), 'shhhhh')
+  test.ok(!service.getMethods()[2].getOption('method_option'))
+  test.equal(service.getOption('service_level_option'), 'serviceOption')
+
+  test.done()
+}
+
+
+exports.testBadProto_incorrectSyntax = function (test) {
+  assertFails('syntax = "proto2"; message SyntaxThree { string first = 1; string second = 1; }',
+      'Unexpected token in',
+      'Protos using incorrect syntax according to version should fail', test)
+}
+
+
 exports.testBadProto_duplicateTags = function (test) {
   assertFails('message Dupe { required string first = 1; required string second = 1; }',
       'duplicate tag in',

tests/protos/common.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 package examples;
 

tests/protos/conventions.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 package conventions;
 
@@ -13,4 +14,4 @@ message Casing {
 
 enum CasingEnum {
   TWO_WORDS = 1;
-}
+}

tests/protos/inner.proto

@@ -1,3 +1,5 @@
+syntax = "proto2";
+
 package burrito;
 
 message Tortilla {
@@ -19,4 +21,3 @@ message Tortilla {
 
 message Avocado {
 }
-

tests/protos/kitchen-sink-v3.proto

@@ -0,0 +1,78 @@
+// Proto file that tries to exercise all valid proto features.
+
+/**
+ * Both comment types should be supported and ignore other tokens such as message FooBar { }.
+ */
+
+syntax = "proto3";
+
+package some_newer_package;
+
+import "protos/options.proto";
+import "protos/otherOptions.proto";
+import "protos/common.proto";
+import "google/protobuf/descriptor.proto";
+
+option (file_level_option) = "string value";
+option (another_option) = "Just \"testing\" that strings parse.";
+
+message ThisIsTheKitchenSinkV3 {
+  string some_field = 1;
+  repeated bool repeated_field = 2;
+  SomeCoolMessage using_another_message = 3;
+
+  int32 number_field = 4;
+  string string_field = 5;
+  examples.Color color_field = 6;
+
+  oneof oneof_name {
+    int32 oneof_field_normal = 7;
+    string oneof_field_with_option = 8 [(other.field).field_name='specialty'];
+    examples.Color oneof_color_field = 9;
+  }
+
+  bool sherlock_lives_at_221b = 10;
+  bool call_867_5309 = 11;
+  string other_field_option = 12 [(other.field).field_name='special'];
+}
+
+message SomeCoolMessage {
+  option (message_level_option) = "XYZ";
+  int64 id = 1000 [(option)=1, (something_else)="foobar"];
+  string x = 1; string y = 2; string z = 3;
+
+  message MessagesWithinMessages {
+    bool done = 1;
+
+    enum EnumInsideMessageInsideMessage {
+      V3_MAYBE = 0;
+      V3_YES = 1;
+      V3_NO = 2;
+    }
+  }
+}
+
+service WhatTheSinkV3CanDo {
+  option (service_level_option) = "serviceOption";
+
+  rpc DisposeLoudly (ThisIsTheKitchenSinkV3) returns (examples.Color) {
+    option (method_option) = "gargle";
+  }
+
+  rpc RinseQuietly (ThisIsTheKitchenSinkV3) returns (examples.Color) {
+    option (method_option) = "shhhhh";
+  }
+
+  rpc HoldDishes (ThisIsTheKitchenSinkV3) returns (examples.Color);
+}
+
+extend google.protobuf.MessageOptions {
+  string another_message_type = 50005;
+}
+
+enum ColorsInWheel {
+  COLOR_UNKNOWN = 0;
+  COLOR_BLACK = 1;
+  COLOR_RED = 2;
+  COLOR_BLUE = 3;
+}

tests/protos/loop.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 message TweedleDee {
   optional TweedleDum dum = 1;

tests/protos/options.proto

@@ -1,4 +1,5 @@
 // Copyright 2015. A Medium Corporation.
+syntax = "proto2";
 
 import "google/protobuf/descriptor.proto";
 
@@ -22,4 +23,4 @@ extend google.protobuf.MethodOptions {
 
 extend google.protobuf.ServiceOptions {
   optional string service_level_option = 20170125;
-}
+}

tests/protos/otherOptions.proto

@@ -1,3 +1,5 @@
+syntax = "proto2";
+
 package other;
 
 import "google/protobuf/descriptor.proto";

tests/protos/person.proto

@@ -2,6 +2,8 @@
  * Proto definition for a Person
  */
 
+syntax = "proto2";
+
 import "protos/common.proto";
 
 package examples;

tests/protos/services.proto

@@ -1,3 +1,4 @@
+syntax = "proto2";
 
 package shoes;
 
@@ -14,4 +15,4 @@ message FullShoe {
 service RunningShoe {
   rpc LaceShoe (Shoe) returns (FullShoe);
   rpc Stride (Shoe) returns (FullShoe);
-}
+}

tests/protos/super-person.proto

@@ -2,6 +2,8 @@
  * Extension of person
  */
 
+syntax = "proto2";
+
 import "protos/person.proto";
 
 package examples;

tests/protos/vehicle.proto

@@ -1,5 +1,7 @@
 // Proto definition for a Car
 
+syntax = "proto2";
+
 import "protos/common.proto";
 import "protos/person.proto";