Skip to content

Commit

Permalink
#116 - make compatible json decoders handle wrong numeric literals (#117
Browse files Browse the repository at this point in the history 
)

Co-authored-by: Radai Rosenblatt <[email protected]>
  • Loading branch information
@radai-rosenblatt
radai-rosenblatt and Radai Rosenblatt authored Dec 7, 2020
1 parent 77415f6 commit 11dee45
Show file tree
Hide file tree
Showing 8 changed files with 154 additions and 54 deletions.
7 changes: 6 additions & 1 deletion ...er/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,12 @@ public static JsonDecoder newJsonDecoder(Schema schema, String in) throws IOExce

/**
* constructs a {@link JsonDecoder} on top of the given {@link InputStream} for the given {@link Schema}
* that can decode json in either "modern" or old (avro &lt;= 1.4) formats
* that is more widely compatible than the "native" avro decoder:
* <ul>
* <li>avro json format has changed between 1.4 and 1.5 around encoding of union branches - simple name vs full names for named types</li>
* <li>avro json decoders are not tolerant of int literals in json for float fields and vice versa under avro &lt; 1.7</li>
* </ul>
* the decoder returned by this method is expected to handle these cases with no errors
* @param schema a schema
* @param in an input stream containing a json-serialized avro payload
* @return a decoder
Expand Down
30 changes: 15 additions & 15 deletions ...rc/main/java/com/linkedin/avroutil1/compatibility/avro14/codec/CompatibleJsonDecoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,14 @@ public class CompatibleJsonDecoder extends ParsingDecoder
implements Parser.ActionHandler {
private JsonParser in;
private static JsonFactory jsonFactory = new JsonFactory();

static final String CHARSET = "ISO-8859-1";

CompatibleJsonDecoder(Symbol root, InputStream in) throws IOException {
super(root);
init(in);
}

CompatibleJsonDecoder(Symbol root, String in) throws IOException {
super(root);
init(in);
Expand All @@ -61,7 +61,7 @@ public class CompatibleJsonDecoder extends ParsingDecoder
public CompatibleJsonDecoder(Schema schema, InputStream in) throws IOException {
this(new JsonGrammarGenerator().generate(schema, false), in);
}

/** Creates a new JsonDecoder based on a String input. */
public CompatibleJsonDecoder(Schema schema, String in) throws IOException {
this(new JsonGrammarGenerator().generate(schema, false), in);
Expand All @@ -80,7 +80,7 @@ public void init(InputStream in) throws IOException {
this.in = jsonFactory.createJsonParser(in);
this.in.nextToken();
}

/** Re-initializes to start reading from a new String input. */
public void init(String in) throws IOException {
parser.reset();
Expand All @@ -101,7 +101,7 @@ public void readNull() throws IOException {
@Override
public boolean readBoolean() throws IOException {
advance(Symbol.BOOLEAN);
JsonToken t = in.getCurrentToken();
JsonToken t = in.getCurrentToken();
if (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE) {
in.nextToken();
return t == JsonToken.VALUE_TRUE;
Expand All @@ -113,19 +113,19 @@ public boolean readBoolean() throws IOException {
@Override
public int readInt() throws IOException {
advance(Symbol.INT);
if (in.getCurrentToken() == JsonToken.VALUE_NUMBER_INT) {
if (in.getCurrentToken().isNumeric()) {
int result = in.getIntValue();
in.nextToken();
return result;
} else {
throw error("int");
}
}

@Override
public long readLong() throws IOException {
advance(Symbol.LONG);
if (in.getCurrentToken() == JsonToken.VALUE_NUMBER_INT) {
if (in.getCurrentToken().isNumeric()) {
long result = in.getLongValue();
in.nextToken();
return result;
Expand All @@ -137,7 +137,7 @@ public long readLong() throws IOException {
@Override
public float readFloat() throws IOException {
advance(Symbol.FLOAT);
if (in.getCurrentToken() == JsonToken.VALUE_NUMBER_FLOAT) {
if (in.getCurrentToken().isNumeric()) {
float result = in.getFloatValue();
in.nextToken();
return result;
Expand All @@ -149,15 +149,15 @@ public float readFloat() throws IOException {
@Override
public double readDouble() throws IOException {
advance(Symbol.DOUBLE);
if (in.getCurrentToken() == JsonToken.VALUE_NUMBER_FLOAT) {
if (in.getCurrentToken().isNumeric()) {
double result = in.getDoubleValue();
in.nextToken();
return result;
} else {
throw error("double");
}
}

@Override
public Utf8 readString(Utf8 old) throws IOException {
advance(Symbol.STRING);
Expand Down Expand Up @@ -228,7 +228,7 @@ private void checkFixed(int size) throws IOException {
top.size + " but received " + size + " bytes.");
}
}

@Override
public void readFixed(byte[] bytes, int start, int len) throws IOException {
checkFixed(len);
Expand Down Expand Up @@ -321,7 +321,7 @@ public long skipArray() throws IOException {
if (in.getCurrentToken() == JsonToken.START_ARRAY) {
in.skipChildren();
in.nextToken();
advance(Symbol.ARRAY_END);
advance(Symbol.ARRAY_END);
} else {
throw error("array-start");
}
Expand Down Expand Up @@ -361,7 +361,7 @@ public long skipMap() throws IOException {
if (in.getCurrentToken() == JsonToken.START_OBJECT) {
in.skipChildren();
in.nextToken();
advance(Symbol.MAP_END);
advance(Symbol.MAP_END);
} else {
throw error("map-start");
}
Expand All @@ -372,7 +372,7 @@ public long skipMap() throws IOException {
public int readIndex() throws IOException {
advance(Symbol.UNION);
Symbol.Alternative a = (Symbol.Alternative) parser.popSymbol();

String label;
if (in.getCurrentToken() == JsonToken.VALUE_NULL) {
label = "null";
Expand Down
32 changes: 16 additions & 16 deletions ...rc/main/java/com/linkedin/avroutil1/compatibility/avro15/codec/CompatibleJsonDecoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,14 @@ public class CompatibleJsonDecoder extends ParsingDecoder
implements Parser.ActionHandler {
private JsonParser in;
private static JsonFactory jsonFactory = new JsonFactory();

static final String CHARSET = "ISO-8859-1";

private CompatibleJsonDecoder(Symbol root, InputStream in) throws IOException {
super(root);
configure(in);
}

private CompatibleJsonDecoder(Symbol root, String in) throws IOException {
super(root);
configure(in);
Expand All @@ -66,11 +66,11 @@ private CompatibleJsonDecoder(Symbol root, String in) throws IOException {
public CompatibleJsonDecoder(Schema schema, InputStream in) throws IOException {
this(getSymbol(schema), in);
}

public CompatibleJsonDecoder(Schema schema, String in) throws IOException {
this(getSymbol(schema), in);
}

private static Symbol getSymbol(Schema schema) {
if (null == schema) {
throw new NullPointerException("Schema cannot be null!");
Expand Down Expand Up @@ -99,7 +99,7 @@ public CompatibleJsonDecoder configure(InputStream in) throws IOException {
this.in.nextToken();
return this;
}

/**
* Reconfigures this JsonDecoder to use the String provided for input.
* <p>
Expand Down Expand Up @@ -142,7 +142,7 @@ public void readNull() throws IOException {
@Override
public boolean readBoolean() throws IOException {
advance(Symbol.BOOLEAN);
JsonToken t = in.getCurrentToken();
JsonToken t = in.getCurrentToken();
if (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE) {
in.nextToken();
return t == JsonToken.VALUE_TRUE;
Expand All @@ -154,19 +154,19 @@ public boolean readBoolean() throws IOException {
@Override
public int readInt() throws IOException {
advance(Symbol.INT);
if (in.getCurrentToken() == JsonToken.VALUE_NUMBER_INT) {
if (in.getCurrentToken().isNumeric()) {
int result = in.getIntValue();
in.nextToken();
return result;
} else {
throw error("int");
}
}

@Override
public long readLong() throws IOException {
advance(Symbol.LONG);
if (in.getCurrentToken() == JsonToken.VALUE_NUMBER_INT) {
if (in.getCurrentToken().isNumeric()) {
long result = in.getLongValue();
in.nextToken();
return result;
Expand All @@ -178,7 +178,7 @@ public long readLong() throws IOException {
@Override
public float readFloat() throws IOException {
advance(Symbol.FLOAT);
if (in.getCurrentToken() == JsonToken.VALUE_NUMBER_FLOAT) {
if (in.getCurrentToken().isNumeric()) {
float result = in.getFloatValue();
in.nextToken();
return result;
Expand All @@ -190,15 +190,15 @@ public float readFloat() throws IOException {
@Override
public double readDouble() throws IOException {
advance(Symbol.DOUBLE);
if (in.getCurrentToken() == JsonToken.VALUE_NUMBER_FLOAT) {
if (in.getCurrentToken().isNumeric()) {
double result = in.getDoubleValue();
in.nextToken();
return result;
} else {
throw error("double");
}
}

@Override
public Utf8 readString(Utf8 old) throws IOException {
advance(Symbol.STRING);
Expand Down Expand Up @@ -269,7 +269,7 @@ private void checkFixed(int size) throws IOException {
top.size + " but received " + size + " bytes.");
}
}

@Override
public void readFixed(byte[] bytes, int start, int len) throws IOException {
checkFixed(len);
Expand Down Expand Up @@ -362,7 +362,7 @@ public long skipArray() throws IOException {
if (in.getCurrentToken() == JsonToken.START_ARRAY) {
in.skipChildren();
in.nextToken();
advance(Symbol.ARRAY_END);
advance(Symbol.ARRAY_END);
} else {
throw error("array-start");
}
Expand Down Expand Up @@ -402,7 +402,7 @@ public long skipMap() throws IOException {
if (in.getCurrentToken() == JsonToken.START_OBJECT) {
in.skipChildren();
in.nextToken();
advance(Symbol.MAP_END);
advance(Symbol.MAP_END);
} else {
throw error("map-start");
}
Expand All @@ -413,7 +413,7 @@ public long skipMap() throws IOException {
public int readIndex() throws IOException {
advance(Symbol.UNION);
Symbol.Alternative a = (Symbol.Alternative) parser.popSymbol();

String label;
if (in.getCurrentToken() == JsonToken.VALUE_NULL) {
label = "null";
Expand Down
Loading

0 comments on commit 11dee45

Please sign in to comment.