From 46b95d9e0e713f04472004e6590e31716efb7971 Mon Sep 17 00:00:00 2001 From: LiuBoyan Date: Tue, 6 Aug 2019 17:39:13 +0800 Subject: [PATCH] Modify the structure of the class DataEntry. Enhanced Semantic Expression of the DataEntry.In order to better characterize KV data structure, key and headers fields is added in DataEntry. At the same time, the function of value checking and string parsing is also added. Better yet, it provide basic and custom types for user selection and extension. The old structure as the STRUCT type of the new structure, you can use it as new way and more easier. We change SCHEMA to META because we think META is better than SCHEMA in approaching the meaning expressed by this object. We added the headers attribute to correspond to userProperty in Massage of RocketMQ. We provide a variety of static construction methods and static variables for all type of metas, to facilitate users to quickly select the required meta types. We also provides a fast conversion method for object data, so that users can get the required data safely and quickly. --- connector/pom.xml | 4 + .../connector/api/PositionStorageReader.java | 2 +- .../connector/api/data/DataEntry.java | 153 ++- .../connector/api/data/DataEntryBuilder.java | 318 ++++- .../connector/api/data/Date.java | 74 ++ .../connector/api/data/Decimal.java | 85 ++ .../connector/api/data/Field.java | 59 +- .../connector/api/data/FieldType.java | 60 - .../connector/api/data/Meta.java | 346 +++++ .../connector/api/data/MetaAndData.java | 1184 +++++++++++++++++ .../connector/api/data/MetaArray.java | 95 ++ .../connector/api/data/MetaArrayBuilder.java | 55 + .../connector/api/data/MetaBase.java | 75 ++ .../connector/api/data/MetaBaseBuilder.java | 54 + .../connector/api/data/MetaBuilder.java | 269 ++++ .../connector/api/data/MetaMap.java | 103 ++ .../connector/api/data/MetaMapBuilder.java | 57 + .../connector/api/data/MetaStruct.java | 110 ++ .../connector/api/data/MetaStructBuilder.java | 72 + .../connector/api/data/Schema.java | 76 -- .../connector/api/data/SinkDataEntry.java | 93 +- .../connector/api/data/SourceDataEntry.java | 166 ++- .../connector/api/data/Struct.java | 247 ++++ .../connector/api/data/Time.java | 77 ++ .../connector/api/data/Timestamp.java | 59 + .../connector/api/data/Type.java | 96 ++ .../connector/api/header/DataHeader.java | 89 ++ .../connector/api/header/DataHeaders.java | 502 +++++++ .../connector/api/header/Header.java | 60 + .../connector/api/header/Headers.java | 252 ++++ .../connector/api/data/HeadersTest.java | 49 + .../connector/api/data/MetaAndDataTest.java | 105 ++ .../connector/api/data/MetaTest.java | 508 +++++++ .../connector/api/data/SinkDataEntryTest.java | 117 ++ .../api/data/SourceDataEntryTest.java | 128 ++ pom.xml | 6 + 36 files changed, 5575 insertions(+), 230 deletions(-) create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/Date.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/Decimal.java delete mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/FieldType.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/Meta.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/MetaAndData.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/MetaArray.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/MetaArrayBuilder.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/MetaBase.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/MetaBaseBuilder.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/MetaBuilder.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/MetaMap.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/MetaMapBuilder.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/MetaStruct.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/MetaStructBuilder.java delete mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/Schema.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/Struct.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/Time.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/Timestamp.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/data/Type.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/header/DataHeader.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/header/DataHeaders.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/header/Header.java create mode 100644 connector/src/main/java/io/openmessaging/connector/api/header/Headers.java create mode 100644 connector/src/main/test/io/openmessaging/connector/api/data/HeadersTest.java create mode 100644 connector/src/main/test/io/openmessaging/connector/api/data/MetaAndDataTest.java create mode 100644 connector/src/main/test/io/openmessaging/connector/api/data/MetaTest.java create mode 100644 connector/src/main/test/io/openmessaging/connector/api/data/SinkDataEntryTest.java create mode 100644 connector/src/main/test/io/openmessaging/connector/api/data/SourceDataEntryTest.java diff --git a/connector/pom.xml b/connector/pom.xml index 3f085e3..66adfc0 100644 --- a/connector/pom.xml +++ b/connector/pom.xml @@ -28,6 +28,10 @@ io.openmessaging openmessaging-api + + junit + junit + \ No newline at end of file diff --git a/connector/src/main/java/io/openmessaging/connector/api/PositionStorageReader.java b/connector/src/main/java/io/openmessaging/connector/api/PositionStorageReader.java index 80df7f4..24157f7 100644 --- a/connector/src/main/java/io/openmessaging/connector/api/PositionStorageReader.java +++ b/connector/src/main/java/io/openmessaging/connector/api/PositionStorageReader.java @@ -30,7 +30,7 @@ public interface PositionStorageReader { /** - * Get the position for the specified partition. + * Get the position for the specified queueId. * * @param partition * @return diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/DataEntry.java b/connector/src/main/java/io/openmessaging/connector/api/data/DataEntry.java index 6b628db..d8e3c3c 100644 --- a/connector/src/main/java/io/openmessaging/connector/api/data/DataEntry.java +++ b/connector/src/main/java/io/openmessaging/connector/api/data/DataEntry.java @@ -17,6 +17,12 @@ package io.openmessaging.connector.api.data; +import java.util.Objects; + +import io.openmessaging.connector.api.header.DataHeaders; +import io.openmessaging.connector.api.header.Header; +import io.openmessaging.connector.api.header.Headers; + /** * Base class for records containing data to be copied to/from message queue. * @@ -24,43 +30,64 @@ * @since OMS 0.1.0 */ public abstract class DataEntry { - - public DataEntry(Long timestamp, - EntryType entryType, - String queueName, - Schema schema, - Object[] payload) { - this.timestamp = timestamp; - this.entryType = entryType; - this.queueName = queueName; - this.schema = schema; - this.payload = payload; - } - /** * Timestamp of the data entry. */ private Long timestamp; - /** - * Type of the data entry. + * Related queueName. + */ + private String queueName; + /** + * The id of queue. + */ + private Integer queueId; + /** + * {@link EntryType} of the {@link DataEntry} */ private EntryType entryType; - /** - * Related queueName. + * Definition data key. */ - private String queueName; - + private MetaAndData key; /** - * Schema of the data entry. + * Definition data value. */ - private Schema schema; - + private MetaAndData value; /** - * Payload of the data entry. + * The Headers of data. */ - private Object[] payload; + private Headers headers; + + + public DataEntry(Long timestamp, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value) { + this(timestamp, queueName, queueId, entryType, key, value, new DataHeaders()); + } + + public DataEntry(Long timestamp, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value, + Iterable
headers) { + this.timestamp = timestamp; + this.queueName = queueName; + this.queueId = queueId; + this.entryType = entryType; + this.key = key; + this.value = value; + if (headers instanceof DataHeaders) { + this.headers = (DataHeaders) headers; + } else { + this.headers = new DataHeaders(headers); + } + } public Long getTimestamp() { return timestamp; @@ -86,19 +113,81 @@ public void setQueueName(String queueName) { this.queueName = queueName; } - public Schema getSchema() { - return schema; + public MetaAndData getKey() { + return key; + } + + public void setKey(MetaAndData meta) { + this.key = meta; } - public void setSchema(Schema schema) { - this.schema = schema; + public MetaAndData getValue() { + return value; + } + + public void setValue(MetaAndData value) { + this.value = value; + } + + public Headers getHeaders() { + return headers; + } + + public void setHeaders(Headers headers) { + this.headers = headers; + } + + public int getQueueId() { + return queueId; + } + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + + @Override + public String toString() { + return "DataEntry{" + + "queueName='" + this.queueName + '\'' + + ", queueId='" + this.queueId + + ", entryType='" + this.entryType + '\'' + + ", key='" + this.key + '\'' + + ", value='" + this.value + '\'' + + ", timestamp='" + timestamp + '\'' + + ", headers'=" + headers + '\'' + + '}'; } - public Object[] getPayload() { - return payload; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DataEntry that = (DataEntry) o; + + return Objects.equals(this.queueId, that.queueId) + && Objects.equals(this.queueName, that.queueName) + && Objects.equals(this.entryType, that.entryType) + && Objects.equals(this.key, that.key) + && Objects.equals(this.value, that.value) + && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.headers, that.headers); } - public void setPayload(Object[] payload) { - this.payload = payload; + @Override + public int hashCode() { + int result = this.queueName != null ? this.queueName.hashCode() : 0; + result = 31 * result + (this.queueId != null ? this.queueId.hashCode() : 0); + result = 31 * result + (this.entryType != null ? entryType.hashCode() : 0); + result = 31 * result + (this.key != null ? this.key.hashCode() : 0); + result = 31 * result + (this.value != null ? this.value.hashCode() : 0); + result = 31 * result + (this.timestamp != null ? this.timestamp.hashCode() : 0); + result = 31 * result + this.headers.hashCode(); + return result; } } diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/DataEntryBuilder.java b/connector/src/main/java/io/openmessaging/connector/api/data/DataEntryBuilder.java index ba965ed..74f02e2 100644 --- a/connector/src/main/java/io/openmessaging/connector/api/data/DataEntryBuilder.java +++ b/connector/src/main/java/io/openmessaging/connector/api/data/DataEntryBuilder.java @@ -17,7 +17,14 @@ package io.openmessaging.connector.api.data; +import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import io.openmessaging.connector.api.header.DataHeaders; +import io.openmessaging.connector.api.header.Header; +import io.openmessaging.connector.api.header.Headers; /** * Use DataEntryBuilder to build SourceDataEntry or SinkDataEntry. @@ -32,29 +39,68 @@ public class DataEntryBuilder { */ private Long timestamp; + /** + * Related queue name. + */ + private String queueName; + + /** + * The queueId of queue. + */ + private Integer queueId; + /** * Type of the data entry. */ private EntryType entryType; /** - * Related queue name. + * Key of the data entry. */ - private String queueName; + private MetaAndData key; /** - * Schema of the data entry. + * Value of the data entry. */ - private Schema schema; + private MetaAndData value; /** - * Payload of the data entry. + * The Headers of data. */ - private Object[] payload; + private Headers headers; + + public DataEntryBuilder() { + this.headers = new DataHeaders(); + } + + public DataEntryBuilder(Meta valueMeta) { + this(null, new MetaAndData(valueMeta)); + } + + public DataEntryBuilder(MetaAndData valueMetaAndData) { + this(null, valueMetaAndData); + } - public DataEntryBuilder(Schema schema) { - this.schema = schema; - this.payload = new Object[schema.getFields().size()]; + public DataEntryBuilder(Meta keyMeta, Meta valueMeta) { + this(new MetaAndData(keyMeta), new MetaAndData(valueMeta)); + } + + public DataEntryBuilder(MetaAndData keyMetaAndData, MetaAndData valueMetaAndData) { + this.key = keyMetaAndData; + this.value = valueMetaAndData; + this.headers = new DataHeaders(); + } + + public static DataEntryBuilder newDataEntryBuilder(Meta keyMeta, Meta valueMeta){ + return new DataEntryBuilder(keyMeta, valueMeta); + } + + public static DataEntryBuilder newDataEntryBuilder(Meta valueMeta){ + return new DataEntryBuilder(valueMeta); + } + + public static DataEntryBuilder newDataEntryBuilder(){ + return new DataEntryBuilder(); } public DataEntryBuilder timestamp(Long timestamp) { @@ -62,6 +108,11 @@ public DataEntryBuilder timestamp(Long timestamp) { return this; } + public DataEntryBuilder queueId(Integer queueId){ + this.queueId = queueId; + return this; + } + public DataEntryBuilder entryType(EntryType entryType) { this.entryType = entryType; return this; @@ -72,27 +123,254 @@ public DataEntryBuilder queue(String queueName) { return this; } - public DataEntryBuilder putFiled(String fieldName, Object value) { + // headers + + public DataEntryBuilder header(Header header){ + this.headers.add(header); + return this; + } + + public DataEntryBuilder header(String key, Meta meta, Object value){ + this.headers.add(key, meta, value); + return this; + } + + public DataEntryBuilder header(String key, String value){ + this.headers.addString(key, value); + return this; + } + + public DataEntryBuilder header(String key, boolean value){ + this.headers.addBoolean(key, value); + return this; + } + + public DataEntryBuilder header(String key, byte value){ + this.headers.addByte(key, value); + return this; + } + + public DataEntryBuilder header(String key, short value){ + this.headers.addShort(key, value); + return this; + } + + public DataEntryBuilder header(String key, int value){ + this.headers.addInt(key, value); + return this; + } + + public DataEntryBuilder header(String key, long value){ + this.headers.addLong(key, value); + return this; + } + + public DataEntryBuilder header(String key, float value){ + this.headers.addFloat(key, value); + return this; + } + + public DataEntryBuilder header(String key, double value){ + this.headers.addDouble(key, value); + return this; + } + + public DataEntryBuilder header(String key, byte[] value){ + this.headers.addBytes(key, value); + return this; + } + + public DataEntryBuilder header(String key, List value, Meta meta){ + this.headers.addList(key, value, meta); + return this; + } + + public DataEntryBuilder header(String key, Map value, Meta meta){ + this.headers.addMap(key, value, meta); + return this; + } - Field field = lookupField(fieldName); - payload[field.getIndex()] = value; + public DataEntryBuilder header(String key, Struct value){ + this.headers.addStruct(key, value); return this; } + public DataEntryBuilder header(String key, BigDecimal value){ + this.headers.addDecimal(key, value); + return this; + } + + public DataEntryBuilder header(String key, java.util.Date value){ + this.headers.addDate(key, value); + return this; + } + + public DataEntryBuilder keyMeta(Meta keyMeta){ + MetaAndData tmpKey = new MetaAndData(keyMeta); + if(this.key != null && this.key.getData() != null){ + tmpKey.putData(this.key.getData()); + } + this.key = tmpKey; + return this; + } + + public DataEntryBuilder valueMeta(Meta valueMeta){ + MetaAndData tmpValue = new MetaAndData(valueMeta); + if(this.value != null && this.value.getData() != null){ + tmpValue.putData(this.value.getData()); + } + this.value = tmpValue; + return this; + } + + + // base + + public DataEntryBuilder keyData(Object data){ + switch (this.key.getMeta().getType()){ + case STRUCT: + List fields = null; + if(data instanceof Struct){ + fields = ((Struct)data).getFields(); + } else if(data instanceof MetaAndData){ + fields = ((MetaAndData)data).getMeta().getFields(); + } else{ + MetaAndData strData = MetaAndData.getMetaDataFromString(data.toString()); + fields = strData.getMeta().getFields(); + } + if(fields != null){ + for (int i = 0; i < fields.size(); i++) { + String fieldName = fields.get(i).name(); + keyData(fieldName, ((Struct)data).getObject(fieldName)); + } + } + break; + case MAP: + keyData((Map)data); + break; + case ARRAY: + keyData((List)data); + break; + default: + this.key.putData(data); + break; + } + return this; + } + + public DataEntryBuilder valueData(Object data){ + switch (this.value.getMeta().getType()){ + case STRUCT: + List fields = null; + if(data instanceof Struct){ + fields = ((Struct)data).getFields(); + } else if(data instanceof MetaAndData){ + fields = ((MetaAndData)data).getMeta().getFields(); + } else{ + MetaAndData strData = MetaAndData.getMetaDataFromString(data.toString()); + fields = strData.getMeta().getFields(); + } + if(fields != null){ + for (int i = 0; i < fields.size(); i++) { + String fieldName = fields.get(i).name(); + valueData(fieldName, ((Struct)data).getObject(fieldName)); + } + } + break; + case MAP: + valueData((Map)data); + break; + case ARRAY: + valueData((List)data); + break; + default: + this.value.putData(data); + break; + } + return this; + } + + + // map + + public DataEntryBuilder keyData(Object key, Object value){ + this.key.putData(key, value); + return this; + } + + public DataEntryBuilder keyData(Map map){ + this.key.putData(map); + return this; + } + + public DataEntryBuilder valueData(Object key, Object value){ + this.value.putData(key, value); + return this; + } + + public DataEntryBuilder valueData(Map map){ + this.value.putData(map); + return this; + } + + // array + + public DataEntryBuilder keyData(List elements){ + this.key.putData(elements); + return this; + } + + public DataEntryBuilder valueData(List elements){ + this.value.putData(elements); + return this; + } + + + // struct + + public DataEntryBuilder keyData(String fieldName, Object value) { + this.key.putData(fieldName, value); + return this; + } + + public DataEntryBuilder keyData(Field field, Object value) { + this.key.putData(field, value); + return this; + } + + public DataEntryBuilder key(MetaAndData metaAndData) { + this.key = metaAndData; + return this; + } + + + public DataEntryBuilder valueData(String fieldName, Object data) { + this.value.putData(fieldName, data); + return this; + } + + public DataEntryBuilder valueData(Field field, Object data) { + this.value.putData(field, data); + return this; + } + + public DataEntryBuilder value(MetaAndData metaAndData) { + this.value = metaAndData; + return this; + } + + + public SourceDataEntry buildSourceDataEntry(ByteBuffer sourcePartition, ByteBuffer sourcePosition) { - return new SourceDataEntry(sourcePartition, sourcePosition, timestamp, entryType, queueName, schema, payload); + return new SourceDataEntry(sourcePartition, sourcePosition, timestamp, queueName, queueId, entryType, key, + value, headers); } public SinkDataEntry buildSinkDataEntry(Long queueOffset) { - return new SinkDataEntry(queueOffset, timestamp, entryType, queueName, schema, payload); + return new SinkDataEntry(queueOffset, timestamp, queueName, queueId, entryType, key, value, headers); } - private Field lookupField(String fieldName) { - Field field = schema.getField(fieldName); - if (field == null) - throw new RuntimeException(fieldName + " is not a valid field name"); - return field; - } + } diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/Date.java b/connector/src/main/java/io/openmessaging/connector/api/data/Date.java new file mode 100644 index 0000000..72a43b6 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/Date.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.connector.api.data; + +import java.util.Calendar; +import java.util.TimeZone; + +/** + *

+ * A date representing a calendar day with no time of day or timezone. The corresponding Java type is a java.util.Date + * with hours, minutes, seconds, milliseconds set to 0. The underlying representation is an integer representing the + * number of standardized days (based on a number of milliseconds with 24 hours/day, 60 minutes/hour, 60 seconds/minute, + * 1000 milliseconds/second with n) since Unix epoch. + *

+ */ +public class Date { + public static final String LOGICAL_NAME = "io.openmessaging.connector.api.data.Date"; + + private static final long MILLIS_PER_DAY = 24 * 60 * 60 * 1000; + + private static final TimeZone UTC = TimeZone.getTimeZone("UTC"); + + /** + * Returns a MetaBuilder for a Date. By returning a MetaBuilder you can override additional meta settings such + * as required/optional, default value, and documentation. + * @return a MetaBuilder + */ + public static MetaBuilder builder() { + return MetaBuilder.int32() + .name(LOGICAL_NAME); + } + + public static final Meta META = builder().meta(); + + /** + * Convert a value from its logical format (Date) to it's encoded format. + * @param value the logical value + * @return the encoded value + */ + public static int fromLogical(Meta meta, java.util.Date value) { + if (!(LOGICAL_NAME.equals(meta.getName()))) { + throw new RuntimeException("Requested conversion of Date object but the meta does not match."); + } + Calendar calendar = Calendar.getInstance(UTC); + calendar.setTime(value); + if (calendar.get(Calendar.HOUR_OF_DAY) != 0 || calendar.get(Calendar.MINUTE) != 0 || + calendar.get(Calendar.SECOND) != 0 || calendar.get(Calendar.MILLISECOND) != 0) { + throw new RuntimeException("RocketMQ Connect Date type should not have any time fields set to non-zero values."); + } + long unixMillis = calendar.getTimeInMillis(); + return (int) (unixMillis / MILLIS_PER_DAY); + } + + public static java.util.Date toLogical(Meta meta, int value) { + if (!(LOGICAL_NAME.equals(meta.getName()))) { + throw new RuntimeException("Requested conversion of Date object but the meta does not match."); + } + return new java.util.Date(value * MILLIS_PER_DAY); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/Decimal.java b/connector/src/main/java/io/openmessaging/connector/api/data/Decimal.java new file mode 100644 index 0000000..704794d --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/Decimal.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.connector.api.data; + +import java.math.BigDecimal; +import java.math.BigInteger; + +/** + *

+ * An arbitrary-precision signed decimal number. The value is unscaled * 10 ^ -scale where: + *

    + *
  • unscaled is an integer
    • + *
  • scale is an integer representing how many digits the decimal point should be shifted on the unscaled value
    • + *
+ *

+ *

+ * Decimal does not provide a fixed meta because it is parameterized by the scale, which is fixed on the meta + * rather than being part of the value. + *

+ *

+ * The underlying representation of this type is bytes containing a two's complement integer + *

+ */ +public class Decimal { + public static final String LOGICAL_NAME = "io.openmessaging.connector.api.data.Decimal"; + public static final String SCALE_FIELD = "scale"; + + /** + * Returns a MetaBuilder for a Decimal with the given scale factor. By returning a MetaBuilder you can override + * additional meta settings such as required/optional, default value, and documentation. + * @param scale the scale factor to apply to unscaled values + * @return a MetaBuilder + */ + public static MetaBuilder builder(int scale) { + return MetaBuilder.bytes() + .name(LOGICAL_NAME) + .parameter(SCALE_FIELD, Integer.toString(scale)); + } + + public static Meta meta(int scale) { + return builder(scale).build(); + } + + /** + * Convert a value from its logical format (BigDecimal) to it's encoded format. + * @param value the logical value + * @return the encoded value + */ + public static byte[] fromLogical(Meta meta, BigDecimal value) { + if (value.scale() != scale(meta)) { + throw new RuntimeException("BigDecimal has mismatching scale value for given Decimal meta"); + } + return value.unscaledValue().toByteArray(); + } + + public static BigDecimal toLogical(Meta meta, byte[] value) { + return new BigDecimal(new BigInteger(value), scale(meta)); + } + + private static int scale(Meta meta) { + String scaleString = meta.getParameters().get(SCALE_FIELD); + if (scaleString == null) { + throw new RuntimeException("Invalid Decimal meta: scale parameter not found."); + } + try { + return Integer.parseInt(scaleString); + } catch (NumberFormatException e) { + throw new RuntimeException("Invalid scale parameter found in Decimal meta: ", e); + } + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/Field.java b/connector/src/main/java/io/openmessaging/connector/api/data/Field.java index a3df93d..3e6079e 100644 --- a/connector/src/main/java/io/openmessaging/connector/api/data/Field.java +++ b/connector/src/main/java/io/openmessaging/connector/api/data/Field.java @@ -17,8 +17,10 @@ package io.openmessaging.connector.api.data; +import java.util.Objects; + /** - * Filed of the schema. + * Filed of the meta. * * @version OMS 0.1.0 * @since OMS 0.1.0 @@ -26,48 +28,63 @@ public class Field { /** - * Index of a field. The index of fields in a schema should be unique and continuous。 + * Index of a field. The index of fields in a meta should be unique and continuous。 */ - private int index; + private final int index; /** - * The name of a file. Should be unique in a shcema. + * The name of a field. Should be unique in a meta. */ - private String name; + private final String name; /** - * The type of the file. + * The type of the field. */ - private FieldType type; - - public Field(int index, String name, FieldType type) { + private final Meta meta; + public Field(int index, String name, Meta meta) { this.index = index; this.name = name; - this.type = type; + this.meta = meta; } - public int getIndex() { + public int index() { return index; } - public void setIndex(int index) { - this.index = index; + public String name() { + return name; } - public String getName() { - return name; + public Meta meta() { + return meta; } - public void setName(String name) { - this.name = name; + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Field field = (Field) o; + return Objects.equals(index, field.index) && + Objects.equals(name, field.name) && + Objects.equals(meta, field.meta); } - public FieldType getType() { - return type; + @Override + public int hashCode() { + return Objects.hash(name, index, meta); } - public void setType(FieldType type) { - this.type = type; + @Override + public String toString() { + return "Field{" + + "name=" + name + + ", index=" + index + + ", meta=" + meta + + "}"; } } diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/FieldType.java b/connector/src/main/java/io/openmessaging/connector/api/data/FieldType.java deleted file mode 100644 index c53e07e..0000000 --- a/connector/src/main/java/io/openmessaging/connector/api/data/FieldType.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.openmessaging.connector.api.data; - -/** - * Define the field type. - * - * @version OMS 0.1.0 - * @since OMS 0.1.0 - */ -public enum FieldType { - - /** Integer */ - INT32, - - /** Long */ - INT64, - - /** BigInteger */ - BIG_INTEGER, - - /** Float */ - FLOAT32, - - /** Double */ - FLOAT64, - - /** Boolean */ - BOOLEAN, - - /** String */ - STRING, - - /** Byte */ - BYTES, - - /** List */ - ARRAY, - - /** Map */ - MAP, - - /** Date */ - DATETIME; -} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/Meta.java b/connector/src/main/java/io/openmessaging/connector/api/data/Meta.java new file mode 100644 index 0000000..e78d9ba --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/Meta.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.openmessaging.connector.api.data; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Meta + * + * @version OMS 0.1.0 + * @since OMS 0.1.0 + */ +public abstract class Meta { + /** + * Maps Types to a list of Java classes that can be used to represent them. + */ + private static final Map> META_TYPE_CLASSES = new EnumMap<>(Type.class); + /** + * Maps known logical types to a list of Java classes that can be used to represent them. + */ + private static final Map> LOGICAL_TYPE_CLASSES = new HashMap<>(); + /** + * Maps the Java classes to the corresponding Type. + */ + private static final Map, Type> JAVA_CLASS_META_TYPES = new HashMap<>(); + + static { + META_TYPE_CLASSES.put(Type.INT8, Collections.singletonList((Class)Byte.class)); + META_TYPE_CLASSES.put(Type.INT16, Collections.singletonList((Class)Short.class)); + META_TYPE_CLASSES.put(Type.INT32, Collections.singletonList((Class)Integer.class)); + META_TYPE_CLASSES.put(Type.INT64, Collections.singletonList((Class)Long.class)); + META_TYPE_CLASSES.put(Type.FLOAT32, Collections.singletonList((Class)Float.class)); + META_TYPE_CLASSES.put(Type.FLOAT64, Collections.singletonList((Class)Double.class)); + META_TYPE_CLASSES.put(Type.BOOLEAN, Collections.singletonList((Class)Boolean.class)); + META_TYPE_CLASSES.put(Type.STRING, Collections.singletonList((Class)String.class)); + // Bytes are special and have 2 representations. byte[] causes problems because it doesn't handle equals() and + // hashCode() like we want objects to, so we support both byte[] and ByteBuffer. Using plain byte[] can cause + // those methods to fail, so ByteBuffers are recommended + META_TYPE_CLASSES.put(Type.BYTES, Arrays.asList((Class)byte[].class, (Class)ByteBuffer.class)); + META_TYPE_CLASSES.put(Type.ARRAY, Collections.singletonList((Class)List.class)); + META_TYPE_CLASSES.put(Type.MAP, Collections.singletonList((Class)Map.class)); + META_TYPE_CLASSES.put(Type.STRUCT, Collections.singletonList((Class)Struct.class)); + + for (Map.Entry> metaClasses : META_TYPE_CLASSES.entrySet()) { + for (Class metaClass : metaClasses.getValue()) { + JAVA_CLASS_META_TYPES.put(metaClass, metaClasses.getKey()); + } + } + + LOGICAL_TYPE_CLASSES.put(Decimal.LOGICAL_NAME, Collections.singletonList((Class)BigDecimal.class)); + LOGICAL_TYPE_CLASSES.put(Date.LOGICAL_NAME, Collections.singletonList((Class)java.util.Date.class)); + LOGICAL_TYPE_CLASSES.put(Time.LOGICAL_NAME, Collections.singletonList((Class)java.util.Date.class)); + LOGICAL_TYPE_CLASSES.put(Timestamp.LOGICAL_NAME, Collections.singletonList((Class)java.util.Date.class)); + // We don't need to put these into JAVA_CLASS_META_TYPES since that's only used to determine metas for + // metaless data and logical types will have ambiguous metas (e.g. many of them use the same Java class) so + // they should not be used without metas. + } + + public static Meta INT8_META = MetaBuilder.int8().build(); + public static Meta INT16_META = MetaBuilder.int16().build(); + public static Meta INT32_META = MetaBuilder.int32().build(); + public static Meta INT64_META = MetaBuilder.int64().build(); + public static Meta FLOAT32_META = MetaBuilder.float32().build(); + public static Meta FLOAT64_META = MetaBuilder.float64().build(); + public static Meta BOOLEAN_META = MetaBuilder.bool().build(); + public static Meta STRING_META = MetaBuilder.string().build(); + public static Meta BYTES_META = MetaBuilder.bytes().build(); + + /** + * The type of fields{@link Type} + */ + protected final Type type; + /** + * Name of the meta. + * Optional name, dataSource and version provide a built-in way to indicate what type of data is included. + * Most useful for structs to indicate the semantics of the struct and map it to some existing underlying + * serializer-specific schema. However, can also be useful in specifying other logical types (e.g. a set + * is an array with additional constraints). + */ + protected String name; + /** + * Data source information. + */ + protected String dataSource; + /** + * Version of the meta. + */ + protected Integer version; + /** + * Possible parameters. + */ + protected Map parameters; + + protected Integer hash = null; + + /** + * Base construct. + * + * @param type {@link Type} + * @param name Name of the meta. + * @param version Version of the meta. + * @param dataSource Data source information. + * @param parameters Possible parameters. + */ + public Meta(Type type, String name, Integer version, String dataSource, Map parameters) { + if (null == type) { + throw new RuntimeException("type cannot be null"); + } + this.type = type; + this.name = name; + this.version = version; + this.dataSource = dataSource; + this.parameters = parameters; + } + + /** + * For map type only.Get {@link Meta} information about key. + * + * @return Returns meta information for map key. + */ + public abstract Meta getKeyMeta(); + + /** + * Get {@link Meta} information about value only for map and array types. + * + * @return If the type is map, return the map-value's type; if the type is array, return the list-value's type + */ + public abstract Meta getValueMeta(); + + /** + * For {@link Struct} type only. + * + * @return Returns the List<{@link Field}> corresponding to object[]. + */ + public abstract List getFields(); + + /** + * For {@link Struct} type only. + * + * @param fieldName The name of the field. + * @return Get the field by field name.{@link Field} + */ + public abstract Field getFieldByName(String fieldName); + + /** + * Get the meta's {@link Type}. + * + * @return {@link Type} + */ + public Type getType() { + return this.type; + } + + public String getDataSource() { + return dataSource; + } + + public void setDataSource(String dataSource) { + this.dataSource = dataSource; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map getParameters() { + return parameters; + } + + public void setParameters(Map parameters) { + this.parameters = parameters; + } + + public static void validateValue(Meta meta, Object value) { + validateValue(null, meta, value); + } + + public static void validateValue(String name, Meta meta, Object value) { + if (value == null) { + throw new RuntimeException("Invalid value: null used for required field: \"" + name + + "\", meta type: " + meta.getType()); + } + + List expectedClasses = expectedClassesFor(meta); + + if (expectedClasses == null) { + throw new RuntimeException("Invalid Java object for meta type " + meta.getType() + + ": " + value.getClass() + + " for field: \"" + name + "\""); + } + + boolean foundMatch = false; + for (Class expectedClass : expectedClasses) { + if (expectedClass.isInstance(value)) { + foundMatch = true; + break; + } + } + if (!foundMatch) { + throw new RuntimeException("Invalid Java object for meta type " + meta.getType() + + ": " + value.getClass() + + " for field: \"" + name + "\""); + } + + switch (meta.getType()) { + case STRUCT: + Struct struct = (Struct)value; + if (!struct.meta().equals(meta)) { + throw new RuntimeException("Struct metas do not match."); + } + struct.validate(); + break; + case ARRAY: + List array = (List)value; + for (Object entry : array) { + validateValue(meta.getValueMeta(), entry); + } + break; + case MAP: + Map map = (Map)value; + for (Map.Entry entry : map.entrySet()) { + validateValue(meta.getKeyMeta(), entry.getKey()); + validateValue(meta.getValueMeta(), entry.getValue()); + } + break; + } + } + + private static List expectedClassesFor(Meta meta) { + List expectedClasses = LOGICAL_TYPE_CLASSES.get(meta.getName()); + if (expectedClasses == null) { + expectedClasses = META_TYPE_CLASSES.get(meta.getType()); + } + return expectedClasses; + } + + /** + * Validate that the value can be used for this meta, i.e. that its type matches the scmetahema type and optional + * requirements. Throws a DataException if the value is invalid. + * + * @param value the value to validate + */ + public void validateValue(Object value) { + validateValue(this, value); + } + + /** + * Get the {@link Type} associated with the given class. + * + * @param klass the Class to + * @return the corresponding type, or null if there is no matching type + */ + public static Type getMetaType(Class klass) { + synchronized (JAVA_CLASS_META_TYPES) { + Type metaType = JAVA_CLASS_META_TYPES.get(klass); + if (metaType != null) { + return metaType; + } + + // Since the lookup only checks the class, we need to also try + for (Map.Entry, Type> entry : JAVA_CLASS_META_TYPES.entrySet()) { + try { + klass.asSubclass(entry.getKey()); + // Cache this for subsequent lookups + JAVA_CLASS_META_TYPES.put(klass, entry.getValue()); + return entry.getValue(); + } catch (ClassCastException e) { + // Expected, ignore + } + } + } + return null; + } + + public Meta meta() { + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Meta meta = (Meta)o; + return Objects.equals(dataSource, meta.dataSource) && + Objects.equals(name, meta.name) && + Objects.equals(type, meta.type) && + Objects.equals(parameters, meta.parameters); + } + + @Override + public int hashCode() { + if (this.hash == null) { + this.hash = Objects.hash(this.type, this.dataSource, this.name, this.parameters); + } + return this.hash; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Meta{"); + if (this.name != null) { + sb.append("name=").append(this.name).append(", "); + } + if (this.version != null) { + sb.append("version=").append(this.version).append(", "); + } + sb.append("type=").append(this.type).append(", "); + if (this.dataSource != null) { + sb.append("dataSource=").append(this.dataSource).append(", "); + } + sb.append("parameters=").append(this.parameters).append("}"); + return sb.toString(); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/MetaAndData.java b/connector/src/main/java/io/openmessaging/connector/api/data/MetaAndData.java new file mode 100644 index 0000000..05560a6 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/MetaAndData.java @@ -0,0 +1,1184 @@ +package io.openmessaging.connector.api.data; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.text.CharacterIterator; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.text.StringCharacterIterator; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Calendar; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Set; +import java.util.TimeZone; +import java.util.regex.Pattern; + +/** + * Data with meta information. + * Provide a variety of data formatting methods. Easy access to desired data types. + * Provide static methods for data type validation. + * Provide the function of decompiling data strings to MetaAndData. + * + * @author liuboyan + */ +public class MetaAndData { + + private final Meta meta; + private Object data; + + public MetaAndData(Meta meta) { + this.meta = meta; + initData(); + } + + public MetaAndData(Meta meta, Object data) { + this.meta = meta; + if (data != null) { + this.data = data; + } else { + initData(); + } + } + + public Meta getMeta() { + return meta; + } + + public MetaAndData setData(Object data) { + this.data = data; + return this; + } + + public Object getData() { + return data; + } + + private void initData() { + if (this.meta == null || this.meta.getType() == null) { + return; + } + if (this.data == null) { + switch (meta.getType()) { + case STRUCT: + this.data = new Struct(meta); + break; + case ARRAY: + this.data = new ArrayList<>(); + break; + case MAP: + this.data = new HashMap<>(); + break; + default: + break; + } + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MetaAndData that = (MetaAndData)o; + return Objects.equals(meta, that.meta) && + Objects.equals(data, that.data); + } + + @Override + public int hashCode() { + return Objects.hash(meta, data); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("MetaAndData{").append("meta="); + if (meta != null) { + sb.append(meta); + } + sb.append(", ").append("data="); + if (data != null) { + sb.append(data); + } + sb.append("}"); + return sb.toString(); + } + + /******************************CONSTRUCT********************************/ + + public static MetaAndData getMetaDataFromString(String value) { + return new MetaAndData(Meta.STRING_META).parseString(value); + } + + // base + + public MetaAndData putData(Object data) { + if (this.meta != null) { + if (this.meta.type == Type.ARRAY) { + this.meta.getValueMeta().validateValue(data); + ((List)this.data).add(data); + } else { + this.meta.validateValue(data); + this.data = data; + } + } else { + this.data = data; + } + return this; + } + + // map {"a":15} + + public MetaAndData putData(Object key, Object value) { + if (key == null) { + throw new RuntimeException("key should not be null."); + } + if (this.meta != null) { + this.meta.getKeyMeta().validateValue(key); + this.meta.getValueMeta().validateValue(value); + ((Map)this.data).put(key, value); + } else { + ((Map)this.data).put(key, value); + } + return this; + } + + public MetaAndData putData(Map map) { + if (map == null) { + return this; + } + if (this.meta != null) { + map.forEach((key, value) -> { + this.meta.getKeyMeta().validateValue(key); + this.meta.getValueMeta().validateValue(value); + ((Map)this.data).put(key, value); + }); + } else { + ((Map)this.data).putAll(map); + } + return this; + } + + // array + + public MetaAndData putData(List elements) { + if (elements == null) { + return this; + } + if (this.meta != null) { + elements.forEach(element -> { + this.meta.getValueMeta().validateValue(element); + ((List)this.data).add(element); + }); + } else { + ((List)this.data).addAll(elements); + } + return this; + } + + // struct + + public MetaAndData putData(String fieldName, Object value) { + ((Struct)this.data).put(fieldName, value); + return this; + } + + public MetaAndData putData(Field field, Object value) { + ((Struct)this.data).put(field, value); + return this; + } + + /******************************VALUES********************************/ + + private static final TimeZone UTC = TimeZone.getTimeZone("UTC"); + private static final MetaAndData TRUE_META_AND_VALUE = new MetaAndData(Meta.BOOLEAN_META, Boolean.TRUE); + private static final MetaAndData FALSE_META_AND_VALUE = new MetaAndData(Meta.BOOLEAN_META, Boolean.FALSE); + private static final Meta ARRAY_SELECTOR_META = MetaBuilder.array(Meta.STRING_META).build(); + private static final Meta MAP_SELECTOR_META = MetaBuilder.map(Meta.STRING_META, Meta.STRING_META).build(); + private static final Meta STRUCT_SELECTOR_META = MetaBuilder.struct().build(); + private static final String TRUE_LITERAL = Boolean.TRUE.toString(); + private static final String FALSE_LITERAL = Boolean.FALSE.toString(); + private static final long MILLIS_PER_DAY = 24 * 60 * 60 * 1000; + private static final String NULL_VALUE = "null"; + private static final String ISO_8601_DATE_FORMAT_PATTERN = "yyyy-MM-dd"; + private static final String ISO_8601_TIME_FORMAT_PATTERN = "HH:mm:ss.SSS'Z'"; + private static final String ISO_8601_TIMESTAMP_FORMAT_PATTERN = ISO_8601_DATE_FORMAT_PATTERN + "'T'" + + ISO_8601_TIME_FORMAT_PATTERN; + + private static final String QUOTE_DELIMITER = "\""; + private static final String COMMA_DELIMITER = ","; + private static final String ENTRY_DELIMITER = ":"; + private static final String ARRAY_BEGIN_DELIMITER = "["; + private static final String ARRAY_END_DELIMITER = "]"; + private static final String MAP_BEGIN_DELIMITER = "{"; + private static final String MAP_END_DELIMITER = "}"; + private static final int ISO_8601_DATE_LENGTH = ISO_8601_DATE_FORMAT_PATTERN.length(); + /** + * subtract single quotes + */ + private static final int ISO_8601_TIME_LENGTH = ISO_8601_TIME_FORMAT_PATTERN.length() - 2; + private static final int ISO_8601_TIMESTAMP_LENGTH = ISO_8601_TIMESTAMP_FORMAT_PATTERN.length() - 4; + + private static final Pattern TWO_BACKSLASHES = Pattern.compile("\\\\"); + + private static final Pattern DOUBLEQOUTE = Pattern.compile("\""); + + public Boolean convertToBoolean() throws RuntimeException { + return (Boolean)convertTo(this.meta, Meta.BOOLEAN_META, this.data); + } + + public Byte convertToByte() throws RuntimeException { + return (Byte)convertTo(this.meta, Meta.INT8_META, this.data); + } + + public Short convertToShort() throws RuntimeException { + return (Short)convertTo(this.meta, Meta.INT16_META, this.data); + } + + public Integer convertToInteger() throws RuntimeException { + return (Integer)convertTo(this.meta, Meta.INT32_META, this.data); + } + + public Long convertToLong() throws RuntimeException { + return (Long)convertTo(this.meta, Meta.INT64_META, this.data); + } + + public Float convertToFloat() throws RuntimeException { + return (Float)convertTo(this.meta, Meta.FLOAT32_META, this.data); + } + + public Double convertToDouble() throws RuntimeException { + return (Double)convertTo(this.meta, Meta.FLOAT64_META, this.data); + } + + public String convertToString() { + return (String)convertTo(this.meta, Meta.STRING_META, this.data); + } + + public List convertToList() { + return (List)convertTo(this.meta, ARRAY_SELECTOR_META, this.data); + } + + public Map convertToMap() { + return (Map)convertTo(this.meta, MAP_SELECTOR_META, this.data); + } + + public Struct convertToStruct() { + return (Struct)convertTo(this.meta, STRUCT_SELECTOR_META, this.data); + } + + public java.util.Date convertToTime() { + return (java.util.Date)convertTo(this.meta, Time.META, this.data); + } + + public java.util.Date convertToDate() { + return (java.util.Date)convertTo(this.meta, Date.META, this.data); + } + + public java.util.Date convertToTimestamp() { + return (java.util.Date)convertTo(this.meta, Timestamp.META, this.data); + } + + public BigDecimal convertToDecimal(int scale) { + return (BigDecimal)convertTo(this.meta, Decimal.meta(scale), this.data); + } + + public Meta inferMeta() { + if (this.data == null) { + return null; + } + if (this.data instanceof String) { + return Meta.STRING_META; + } + if (this.data instanceof Boolean) { + return Meta.BOOLEAN_META; + } + if (this.data instanceof Byte) { + return Meta.INT8_META; + } + if (this.data instanceof Short) { + return Meta.INT16_META; + } + if (this.data instanceof Integer) { + return Meta.INT32_META; + } + if (this.data instanceof Long) { + return Meta.INT64_META; + } + if (this.data instanceof Float) { + return Meta.FLOAT32_META; + } + if (this.data instanceof Double) { + return Meta.FLOAT64_META; + } + if (this.data instanceof byte[] || this.data instanceof ByteBuffer) { + return Meta.BYTES_META; + } + if (this.data instanceof List) { + List list = (List)this.data; + if (list.isEmpty()) { + return null; + } + MetaDetector detector = new MetaDetector(); + for (Object element : list) { + if (!detector.canDetect(element)) { + return null; + } + } + return MetaBuilder.array(detector.meta()).build(); + } + if (this.data instanceof Map) { + Map map = (Map)this.data; + if (map.isEmpty()) { + return null; + } + MetaDetector keyDetector = new MetaDetector(); + MetaDetector valueDetector = new MetaDetector(); + for (Map.Entry entry : map.entrySet()) { + if (!keyDetector.canDetect(entry.getKey()) || !valueDetector.canDetect(entry.getValue())) { + return null; + } + } + return MetaBuilder.map(keyDetector.meta(), valueDetector.meta()).build(); + } + if (this.data instanceof Struct) { + return ((Struct)this.data).meta(); + } + return null; + } + + /************************************VALUES_PROTECTED*************************************/ + + /** + * Convert the value to the desired type. + * + * @param fromMeta the meta for the desired type; may not be null + * @param toMeta the meta for the supplied value; may be null if not known + * @return the converted value; never null + * @throws RuntimeException if the value could not be converted to the desired type + */ + protected Object convertTo(Meta fromMeta, Meta toMeta, Object value) throws RuntimeException { + if (value == null) { + throw new RuntimeException("Unable to convert a null value to a meta that requires a value"); + } + if (toMeta == null) { + throw new RuntimeException("Unable to convert a value to a null meta"); + } + switch (toMeta.getType()) { + case BYTES: + if (Decimal.LOGICAL_NAME.equals(toMeta.getName())) { + if (value instanceof ByteBuffer) { + value = toArray((ByteBuffer)value); + } + if (value instanceof byte[]) { + return Decimal.toLogical(toMeta, (byte[])value); + } + if (value instanceof BigDecimal) { + return value; + } + if (value instanceof Number) { + // Not already a decimal, so treat it as a double ... + double converted = ((Number)value).doubleValue(); + return new BigDecimal(converted); + } + if (value instanceof String) { + return new BigDecimal(value.toString()).doubleValue(); + } + } + if (value instanceof ByteBuffer) { + return toArray((ByteBuffer)value); + } + if (value instanceof byte[]) { + return value; + } + if (value instanceof BigDecimal) { + return Decimal.fromLogical(toMeta, (BigDecimal)value); + } + break; + case STRING: + StringBuilder sb = new StringBuilder(); + append(sb, value, false); + return sb.toString(); + case BOOLEAN: + if (value instanceof Boolean) { + return value; + } + if (value instanceof String) { + MetaAndData parsed = parseString(value.toString()); + if (parsed.getData() instanceof Boolean) { + return parsed.getData(); + } + } + return asLong(value, fromMeta, null) == 0L ? Boolean.FALSE : Boolean.TRUE; + case INT8: + if (value instanceof Byte) { + return value; + } + return (byte)asLong(value, fromMeta, null); + case INT16: + if (value instanceof Short) { + return value; + } + return (short)asLong(value, fromMeta, null); + case INT32: + if (Date.LOGICAL_NAME.equals(toMeta.getName())) { + if (value instanceof String) { + MetaAndData parsed = parseString(value.toString()); + value = parsed.getData(); + } + if (value instanceof java.util.Date) { + if (fromMeta != null) { + String fromMetaName = fromMeta.getName(); + if (Date.LOGICAL_NAME.equals(fromMetaName)) { + return value; + } + if (Timestamp.LOGICAL_NAME.equals(fromMetaName)) { + // Just get the number of days from this timestamp + long millis = ((java.util.Date)value).getTime(); + int days = (int)(millis / MILLIS_PER_DAY); // truncates + return Date.toLogical(toMeta, days); + } + } + } + long numeric = asLong(value, fromMeta, null); + return Date.toLogical(toMeta, (int)numeric); + } + if (Time.LOGICAL_NAME.equals(toMeta.getName())) { + if (value instanceof String) { + MetaAndData parsed = parseString(value.toString()); + value = parsed.getData(); + } + if (value instanceof java.util.Date) { + if (fromMeta != null) { + String fromMetaName = fromMeta.getName(); + if (Time.LOGICAL_NAME.equals(fromMetaName)) { + return value; + } + if (Timestamp.LOGICAL_NAME.equals(fromMetaName)) { + // Just get the time portion of this timestamp + Calendar calendar = Calendar.getInstance(UTC); + calendar.setTime((java.util.Date)value); + calendar.set(Calendar.YEAR, 1970); + calendar.set(Calendar.MONTH, 0); // Months are zero-based + calendar.set(Calendar.DAY_OF_MONTH, 1); + return Time.toLogical(toMeta, (int)calendar.getTimeInMillis()); + } + } + } + long numeric = asLong(value, fromMeta, null); + return Time.toLogical(toMeta, (int)numeric); + } + if (value instanceof Integer) { + return value; + } + return (int)asLong(value, fromMeta, null); + case INT64: + if (Timestamp.LOGICAL_NAME.equals(toMeta.getName())) { + if (value instanceof String) { + MetaAndData parsed = parseString(value.toString()); + value = parsed.getData(); + } + if (value instanceof java.util.Date) { + java.util.Date date = (java.util.Date)value; + if (fromMeta != null) { + String fromMetaName = fromMeta.getName(); + if (Date.LOGICAL_NAME.equals(fromMetaName)) { + int days = Date.fromLogical(fromMeta, date); + long millis = days * MILLIS_PER_DAY; + return Timestamp.toLogical(toMeta, millis); + } + if (Time.LOGICAL_NAME.equals(fromMetaName)) { + long millis = Time.fromLogical(fromMeta, date); + return Timestamp.toLogical(toMeta, millis); + } + if (Timestamp.LOGICAL_NAME.equals(fromMetaName)) { + return value; + } + } + } + long numeric = asLong(value, fromMeta, null); + return Timestamp.toLogical(toMeta, numeric); + } + if (value instanceof Long) { + return value; + } + return asLong(value, fromMeta, null); + case FLOAT32: + if (value instanceof Float) { + return value; + } + return (float)asDouble(value, fromMeta, null); + case FLOAT64: + if (value instanceof Double) { + return value; + } + return asDouble(value, fromMeta, null); + case ARRAY: + if (value instanceof String) { + MetaAndData metaAndData = parseString(value.toString()); + value = metaAndData.getData(); + } + if (value instanceof List) { + return value; + } + break; + case MAP: + if (value instanceof String) { + MetaAndData metaAndData = parseString(value.toString()); + value = metaAndData.getData(); + } + if (value instanceof Map) { + return value; + } + break; + case STRUCT: + if (value instanceof Struct) { + Struct struct = (Struct)value; + return struct; + } + if (value instanceof Map) { + Map mapData = (Map)value; + Set entries = mapData.entrySet(); + List fieldMetas = new LinkedList<>(); + List fieldNames = new LinkedList<>(); + List fieldDatas = new LinkedList(); + for (Map.Entry entry : entries) { + String fieldName = entry.getKey().toString(); + MetaAndData valueMeta = parseString( + convertTo(null, Meta.STRING_META, entry.getValue()).toString()); + if (valueMeta.getMeta() == null && valueMeta.getData() instanceof Map) { + valueMeta = new MetaAndData(valueMeta.convertToStruct().meta(), + valueMeta.convertToStruct()); + } + fieldMetas.add(valueMeta.getMeta()); + fieldNames.add(fieldName); + fieldDatas.add(valueMeta.getData()); + } + MetaBuilder structMetaBuilder = MetaBuilder.struct(); + for (int i = 0; i < fieldMetas.size(); i++) { + structMetaBuilder.field(fieldNames.get(i), fieldMetas.get(i)); + } + Meta structMeta = structMetaBuilder.build(); + Struct result = new Struct(structMeta); + for (int i = 0; i < fieldMetas.size(); i++) { + result.put(fieldNames.get(i), fieldDatas.get(i)); + } + return result; + } + } + throw new RuntimeException("Unable to convert " + value + " (" + value.getClass() + ") to " + toMeta); + } + + public MetaAndData parseString(String value) { + if (value == null) { + return null; + } + if (value.isEmpty()) { + return new MetaAndData(Meta.STRING_META, value); + } + Parser parser = new Parser(value); + return parse(parser, false); + } + + protected MetaAndData parse(Parser parser, boolean embedded) throws NoSuchElementException { + if (!parser.hasNext()) { + return null; + } + if (embedded) { + if (parser.canConsume(NULL_VALUE)) { + return null; + } + if (parser.canConsume(QUOTE_DELIMITER)) { + StringBuilder sb = new StringBuilder(); + while (parser.hasNext()) { + if (parser.canConsume(QUOTE_DELIMITER)) { + break; + } + sb.append(parser.next()); + } + return new MetaAndData(Meta.STRING_META, sb.toString()); + } + } + if (parser.canConsume(TRUE_LITERAL)) { + return TRUE_META_AND_VALUE; + } + if (parser.canConsume(FALSE_LITERAL)) { + return FALSE_META_AND_VALUE; + } + int startPosition = parser.mark(); + try { + if (parser.canConsume(ARRAY_BEGIN_DELIMITER)) { + List result = new ArrayList<>(); + Meta elementMeta = null; + while (parser.hasNext()) { + if (parser.canConsume(ARRAY_END_DELIMITER)) { + Meta listMeta = null; + if (elementMeta != null) { + listMeta = MetaBuilder.array(elementMeta).meta(); + } + result = alignListEntriesWithMeta(listMeta, result); + return new MetaAndData(listMeta, result); + } + if (parser.canConsume(COMMA_DELIMITER)) { + throw new RuntimeException("Unable to parse an empty array element: " + parser.original()); + } + MetaAndData element = parse(parser, true); + elementMeta = commonMetaFor(elementMeta, element); + result.add(element.getData()); + parser.canConsume(COMMA_DELIMITER); + } + // Missing either a comma or an end delimiter + if (COMMA_DELIMITER.equals(parser.previous())) { + throw new RuntimeException("Array is missing element after ',': " + parser.original()); + } + throw new RuntimeException("Array is missing terminating ']': " + parser.original()); + } + + if (parser.canConsume(MAP_BEGIN_DELIMITER)) { + Map result = new LinkedHashMap<>(); + Meta keyMeta = null; + Meta valueMeta = null; + boolean objFlag = false; + while (parser.hasNext()) { + if (parser.canConsume(MAP_END_DELIMITER)) { + Meta mapMeta = null; + if (!objFlag) { + mapMeta = MetaBuilder.map(keyMeta, valueMeta).meta(); + } + result = alignMapKeysAndValuesWithMeta(mapMeta, result); + return new MetaAndData(mapMeta, result); + } + if (parser.canConsume(COMMA_DELIMITER)) { + throw new RuntimeException( + "Unable to parse a map entry has no key or value: " + parser.original()); + } + MetaAndData key = parse(parser, true); + if (key == null || key.getData() == null) { + throw new RuntimeException("Map entry may not have a null key: " + parser.original()); + } + if (!parser.canConsume(ENTRY_DELIMITER)) { + throw new RuntimeException("Map entry is missing '=': " + parser.original()); + } + MetaAndData value = parse(parser, true); + Object entryValue = value != null ? value.getData() : null; + result.put(key.getData(), entryValue); + parser.canConsume(COMMA_DELIMITER); + keyMeta = commonMetaFor(keyMeta, key); + valueMeta = commonMetaFor(valueMeta, value); + if (!objFlag && (keyMeta == null || valueMeta == null)) { + objFlag = true; + } + } + // Missing either a comma or an end delimiter + if (COMMA_DELIMITER.equals(parser.previous())) { + throw new RuntimeException("Map is missing element after ',': " + parser.original()); + } + throw new RuntimeException("Map is missing terminating ']': " + parser.original()); + } + } catch (RuntimeException e) { + e.printStackTrace(); + parser.rewindTo(startPosition); + } + String token = parser.next().trim(); + assert !token + .isEmpty(); // original can be empty string but is handled right away; no way for token to be empty here + char firstChar = token.charAt(0); + boolean firstCharIsDigit = Character.isDigit(firstChar); + if (firstCharIsDigit || firstChar == '+' || firstChar == '-') { + try { + // Try to parse as a number ... + BigDecimal decimal = new BigDecimal(token); + try { + return new MetaAndData(Meta.INT8_META, decimal.byteValueExact()); + } catch (ArithmeticException e) { + // continue + } + try { + return new MetaAndData(Meta.INT16_META, decimal.shortValueExact()); + } catch (ArithmeticException e) { + // continue + } + try { + return new MetaAndData(Meta.INT32_META, decimal.intValueExact()); + } catch (ArithmeticException e) { + // continue + } + try { + return new MetaAndData(Meta.INT64_META, decimal.longValueExact()); + } catch (ArithmeticException e) { + // continue + } + double dValue = decimal.doubleValue(); + if (dValue != Double.NEGATIVE_INFINITY && dValue != Double.POSITIVE_INFINITY) { + return new MetaAndData(Meta.FLOAT64_META, dValue); + } + Meta meta = Decimal.meta(decimal.scale()); + return new MetaAndData(meta, decimal); + } catch (NumberFormatException e) { + // can't parse as a number + } + } + if (firstCharIsDigit) { + // Check for a date, time, or timestamp ... + int tokenLength = token.length(); + if (tokenLength == ISO_8601_DATE_LENGTH) { + try { + return new MetaAndData(Date.META, new SimpleDateFormat(ISO_8601_DATE_FORMAT_PATTERN).parse(token)); + } catch (ParseException e) { + // not a valid date + } + } else if (tokenLength == ISO_8601_TIME_LENGTH) { + try { + return new MetaAndData(Time.META, new SimpleDateFormat(ISO_8601_TIME_FORMAT_PATTERN).parse(token)); + } catch (ParseException e) { + // not a valid date + } + } else if (tokenLength == ISO_8601_TIMESTAMP_LENGTH) { + try { + return new MetaAndData(Timestamp.META, + new SimpleDateFormat(ISO_8601_TIMESTAMP_FORMAT_PATTERN).parse(token)); + } catch (ParseException e) { + // not a valid date + } + } + } + // At this point, the only thing this can be is a string. Embedded strings were processed above, + // so this is not embedded and we can use the original string... + return new MetaAndData(Meta.STRING_META, parser.original()); + } + + protected List alignListEntriesWithMeta(Meta meta, List input) { + if (meta == null) { + return input; + } + Meta valueMeta = meta.getValueMeta(); + List result = new ArrayList<>(); + for (Object value : input) { + Object newValue = convertTo(null, valueMeta, value); + result.add(newValue); + } + return result; + } + + protected Map alignMapKeysAndValuesWithMeta(Meta mapMeta, Map input) { + if (mapMeta == null) { + return input; + } + Meta keyMeta = mapMeta.getKeyMeta(); + Meta valueMeta = mapMeta.getValueMeta(); + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : input.entrySet()) { + Object newKey = convertTo(keyMeta, null, entry.getKey()); + Object newValue = convertTo(valueMeta, null, entry.getValue()); + result.put(newKey, newValue); + } + return result; + } + + protected Meta commonMetaFor(Meta previous, MetaAndData latest) { + if (latest == null) { + return previous; + } + if (previous == null) { + return latest.getMeta(); + } + Meta newMeta = latest.getMeta(); + Type previousType = previous.getType(); + Type newType = newMeta.getType(); + if (previousType != newType) { + switch (previous.getType()) { + case INT8: + if (newType == Type.INT16 || newType == Type.INT32 || newType == Type.INT64 + || newType == Type.FLOAT32 || newType == + Type.FLOAT64) { + return newMeta; + } + break; + case INT16: + if (newType == Type.INT8) { + return previous; + } + if (newType == Type.INT32 || newType == Type.INT64 || newType == Type.FLOAT32 + || newType == Type.FLOAT64) { + return newMeta; + } + break; + case INT32: + if (newType == Type.INT8 || newType == Type.INT16) { + return previous; + } + if (newType == Type.INT64 || newType == Type.FLOAT32 || newType == Type.FLOAT64) { + return newMeta; + } + break; + case INT64: + if (newType == Type.INT8 || newType == Type.INT16 || newType == Type.INT32) { + return previous; + } + if (newType == Type.FLOAT32 || newType == Type.FLOAT64) { + return newMeta; + } + break; + case FLOAT32: + if (newType == Type.INT8 || newType == Type.INT16 || newType == Type.INT32 + || newType == Type.INT64) { + return previous; + } + if (newType == Type.FLOAT64) { + return newMeta; + } + break; + case FLOAT64: + if (newType == Type.INT8 || newType == Type.INT16 || newType == Type.INT32 || newType == Type.INT64 + || newType == + Type.FLOAT32) { + return previous; + } + break; + } + return null; + } + if (!previous.equals(newMeta)) { + return null; + } + return previous; + } + + protected void append(StringBuilder sb, Object value, boolean embedded) { + if (value == null) { + sb.append(NULL_VALUE); + } else if (value instanceof Number) { + sb.append(value); + } else if (value instanceof Boolean) { + sb.append(value); + } else if (value instanceof String) { + if (embedded) { + String escaped = escape((String)value); + sb.append('"').append(escaped).append('"'); + } else { + sb.append(value); + } + } else if (value instanceof byte[]) { + value = Base64.getEncoder().encodeToString((byte[])value); + if (embedded) { + sb.append('"').append(value).append('"'); + } else { + sb.append(value); + } + } else if (value instanceof ByteBuffer) { + byte[] bytes = readBytes((ByteBuffer)value); + append(sb, bytes, embedded); + } else if (value instanceof List) { + List list = (List)value; + sb.append('['); + appendIterable(sb, list.iterator()); + sb.append(']'); + } else if (value instanceof Map) { + Map map = (Map)value; + sb.append('{'); + appendIterable(sb, map.entrySet().iterator()); + sb.append('}'); + } else if (value instanceof Struct) { + Struct struct = (Struct)value; + Meta meta = struct.meta(); + boolean first = true; + sb.append('{'); + for (Field field : meta.getFields()) { + if (first) { + first = false; + } else { + sb.append(','); + } + append(sb, field.name(), true); + sb.append(':'); + append(sb, struct.get(field), true); + } + sb.append('}'); + } else if (value instanceof Map.Entry) { + Map.Entry entry = (Map.Entry)value; + append(sb, entry.getKey(), true); + sb.append(':'); + append(sb, entry.getValue(), true); + } else if (value instanceof java.util.Date) { + java.util.Date dateValue = (java.util.Date)value; + String formatted = dateFormatFor(dateValue).format(dateValue); + sb.append(formatted); + } else { + throw new RuntimeException( + "Failed to serialize unexpected value type " + value.getClass().getName() + ": " + value); + } + } + + protected String escape(String value) { + String replace1 = TWO_BACKSLASHES.matcher(value).replaceAll("\\\\\\\\"); + return DOUBLEQOUTE.matcher(replace1).replaceAll("\\\\\""); + } + + protected void appendIterable(StringBuilder sb, Iterator iter) { + if (iter.hasNext()) { + append(sb, iter.next(), true); + while (iter.hasNext()) { + sb.append(','); + append(sb, iter.next(), true); + } + } + } + + protected double asDouble(Object value, Meta meta, Throwable error) { + try { + if (value instanceof Number) { + Number number = (Number)value; + return number.doubleValue(); + } + if (value instanceof String) { + return new BigDecimal(value.toString()).doubleValue(); + } + } catch (NumberFormatException e) { + error = e; + // fall through + } + return asLong(value, meta, error); + } + + protected long asLong(Object value, Meta fromMeta, Throwable error) { + try { + if (value instanceof Number) { + Number number = (Number)value; + return number.longValue(); + } + if (value instanceof String) { + return new BigDecimal(value.toString()).longValue(); + } + } catch (NumberFormatException e) { + error = e; + // fall through + } + if (fromMeta != null) { + String metaName = fromMeta.getName(); + if (value instanceof java.util.Date) { + if (Date.LOGICAL_NAME.equals(metaName)) { + return Date.fromLogical(fromMeta, (java.util.Date)value); + } + if (Time.LOGICAL_NAME.equals(metaName)) { + return Time.fromLogical(fromMeta, (java.util.Date)value); + } + if (Timestamp.LOGICAL_NAME.equals(metaName)) { + return Timestamp.fromLogical(fromMeta, (java.util.Date)value); + } + } + throw new RuntimeException("Unable to convert " + value + " (" + value.getClass() + ") to " + fromMeta, + error); + } + throw new RuntimeException("Unable to convert " + value + " (" + value.getClass() + ") to a number", error); + } + + /**************************UTILS****************************/ + + private DateFormat dateFormatFor(java.util.Date value) { + if (value.getTime() < MILLIS_PER_DAY) { + return new SimpleDateFormat(ISO_8601_TIME_FORMAT_PATTERN); + } + if (value.getTime() % MILLIS_PER_DAY == 0) { + return new SimpleDateFormat(ISO_8601_DATE_FORMAT_PATTERN); + } + return new SimpleDateFormat(ISO_8601_TIMESTAMP_FORMAT_PATTERN); + } + + private byte[] toArray(ByteBuffer buffer) { + return toArray(buffer, 0, buffer.remaining()); + } + + private byte[] toArray(ByteBuffer buffer, int offset, int size) { + byte[] dest = new byte[size]; + if (buffer.hasArray()) { + System.arraycopy(buffer.array(), buffer.position() + buffer.arrayOffset() + offset, dest, 0, size); + } else { + int pos = buffer.position(); + buffer.position(pos + offset); + buffer.get(dest); + buffer.position(pos); + } + return dest; + } + + /** + * Read a buffer into a Byte array for the given offset and length + */ + private byte[] readBytes(ByteBuffer buffer, int offset, int length) { + byte[] dest = new byte[length]; + if (buffer.hasArray()) { + System.arraycopy(buffer.array(), buffer.arrayOffset() + offset, dest, 0, length); + } else { + buffer.mark(); + buffer.position(offset); + buffer.get(dest, 0, length); + buffer.reset(); + } + return dest; + } + + /** + * Read the given byte buffer into a Byte array + */ + private byte[] readBytes(ByteBuffer buffer) { + return readBytes(buffer, 0, buffer.limit()); + } + + protected class MetaDetector { + private Type knownType = null; + + public MetaDetector() { + } + + public boolean canDetect(Object value) { + if (value == null) { + return true; + } + Meta schema = inferMeta(); + if (schema == null) { + return false; + } + if (knownType == null) { + knownType = schema.getType(); + } else if (knownType != schema.getType()) { + return false; + } + return true; + } + + public Meta meta() { + return MetaBuilder.type(knownType).meta(); + } + } + + protected class Parser { + private final String original; + private final CharacterIterator iter; + private String nextToken = null; + private String previousToken = null; + + public Parser(String original) { + this.original = original; + this.iter = new StringCharacterIterator(this.original); + } + + public int position() { + return iter.getIndex(); + } + + public int mark() { + return iter.getIndex() - (nextToken != null ? nextToken.length() : 0); + } + + public void rewindTo(int position) { + iter.setIndex(position); + nextToken = null; + } + + public String original() { + return original; + } + + public boolean hasNext() { + return nextToken != null || canConsumeNextToken(); + } + + protected boolean canConsumeNextToken() { + return iter.getEndIndex() > iter.getIndex(); + } + + public String next() { + if (nextToken != null) { + previousToken = nextToken; + nextToken = null; + } else { + previousToken = consumeNextToken(); + } + return previousToken; + } + + private String consumeNextToken() throws NoSuchElementException { + boolean escaped = false; + int start = iter.getIndex(); + char c = iter.current(); + while (c != CharacterIterator.DONE) { + switch (c) { + case '\\': + escaped = !escaped; + break; + case ':': + case ',': + case '{': + case '}': + case '[': + case ']': + case '\"': + if (!escaped) { + if (start < iter.getIndex()) { + // Return the previous token + return original.substring(start, iter.getIndex()); + } + // Consume and return this delimiter as a token + iter.next(); + return original.substring(start, start + 1); + } + // escaped, so continue + escaped = false; + break; + default: + // If escaped, then we don't care what was escaped + escaped = false; + break; + } + c = iter.next(); + } + return original.substring(start, iter.getIndex()); + } + + public String previous() { + return previousToken; + } + + public boolean canConsume(String expected) { + return canConsume(expected, true); + } + + public boolean canConsume(String expected, boolean ignoreLeadingAndTrailingWhitespace) { + if (isNext(expected, ignoreLeadingAndTrailingWhitespace)) { + // consume this token ... + nextToken = null; + return true; + } + return false; + } + + protected boolean isNext(String expected, boolean ignoreLeadingAndTrailingWhitespace) { + if (nextToken == null) { + if (!hasNext()) { + return false; + } + // There's another token, so consume it + nextToken = consumeNextToken(); + } + if (ignoreLeadingAndTrailingWhitespace) { + nextToken = nextToken.trim(); + while (nextToken.isEmpty() && canConsumeNextToken()) { + nextToken = consumeNextToken().trim(); + } + } + return nextToken.equals(expected); + } + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/MetaArray.java b/connector/src/main/java/io/openmessaging/connector/api/data/MetaArray.java new file mode 100644 index 0000000..f051d29 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/MetaArray.java @@ -0,0 +1,95 @@ +package io.openmessaging.connector.api.data; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Meta implement for type of ARRAY. + * + * @author liuboyan + */ +public class MetaArray extends Meta { + private final Meta valueMeta; + + /** + * Base construct. + * + * @param type {@link Type} + * @param version Version of the meta. + * @param name Name of the meta. + * @param dataSource Data source information. + * @param parameters Possible parameters. + */ + public MetaArray(Type type, String name, Integer version, String dataSource, Map parameters, Meta valueMeta) { + super(type, name, version, dataSource, parameters); + if (type != Type.ARRAY) { + throw new RuntimeException("type should be ARRAY."); + } + this.valueMeta = valueMeta; + } + + @Override + public Meta getKeyMeta() { + throw new RuntimeException("Cannot look up key meta on non-map type"); + } + + @Override + public Meta getValueMeta() { + return this.valueMeta; + } + + @Override + public List getFields() { + throw new RuntimeException("Cannot list fields on non-struct type"); + } + + @Override + public Field getFieldByName(String fieldName) { + throw new RuntimeException("Cannot look up fields on non-struct type"); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + + MetaArray meta = (MetaArray)o; + return Objects.equals(this.valueMeta, meta.valueMeta); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + this.valueMeta.hashCode(); + return result; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("MetaArray{"); + if (this.name != null) { + sb.append("name=").append(this.name).append(", "); + } + if (this.version != null) { + sb.append("version=").append(this.version).append(", "); + } + sb.append("type=").append(this.type).append(", "); + if (this.dataSource != null) { + sb.append("dataSource=").append(this.dataSource).append(", "); + } + if (this.valueMeta != null) { + sb.append("valueMeta=").append(this.valueMeta).append(", "); + } + sb.append("parameters=").append(this.parameters).append("}"); + return sb.toString(); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/MetaArrayBuilder.java b/connector/src/main/java/io/openmessaging/connector/api/data/MetaArrayBuilder.java new file mode 100644 index 0000000..1099383 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/MetaArrayBuilder.java @@ -0,0 +1,55 @@ +package io.openmessaging.connector.api.data; + +import java.util.Collections; +import java.util.List; + +/** + * MetaBuilder implement for type of ARRAY. + * + * @author liuboyan + */ +public class MetaArrayBuilder extends MetaBuilder { + private Meta valueMeta; + + public MetaArrayBuilder(Type type, Meta valueMeta) { + super(type); + this.valueMeta = valueMeta; + } + + @Override + public MetaBuilder field(String fieldName, Meta fieldMeta) { + throw new RuntimeException("Cannot create fields on type " + this.type()); + } + + @Override + public MetaBuilder field(Field field) { + throw new RuntimeException("Cannot create fields on type " + this.type()); + } + + @Override + public Meta keyMeta() { + return null; + } + + @Override + public Meta valueMeta() { + return this.valueMeta; + } + + @Override + public List fields() { + throw new RuntimeException("Cannot list fields on non-struct type"); + } + + @Override + public Field getFieldByName(String fieldName) { + throw new RuntimeException("Cannot look up fields on non-struct type"); + } + + @Override + public Meta build() { + return new MetaArray(this.type(), this.name(), this.version(), + this.dataSource(), this.parameters() == null ? null : Collections.unmodifiableMap(this.parameters()), + this.valueMeta); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/MetaBase.java b/connector/src/main/java/io/openmessaging/connector/api/data/MetaBase.java new file mode 100644 index 0000000..49bdd39 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/MetaBase.java @@ -0,0 +1,75 @@ +package io.openmessaging.connector.api.data; + +import java.util.List; +import java.util.Map; + +/** + * Meta implement for base types. + * {@link Type}.INT8 {@link Type}.INT16 {@link Type}.INT32 {@link Type}.INT64 + * {@link Type}.BIG_INTEGER {@link Type}.FLOAT32 {@link Type}.FLOAT64 {@link Type}.BOOLEAN + * {@link Type}.STRING {@link Type}.BYTES {@link Type}.DATETIME + * + * @author liuboyan + */ +public class MetaBase extends Meta { + /** + * Base construct. + * + * @param type {@link Type} + * @param name Name of the meta. + * @param version Version of the meta. + * @param dataSource Data source information. + * @param parameters Possible parameters. + */ + public MetaBase(Type type, String name, Integer version, String dataSource, Map parameters) { + super(type, name, version, dataSource, parameters); + } + + @Override + public Meta getKeyMeta() { + throw new RuntimeException("Cannot look up key meta on non-map type"); + } + + @Override + public Meta getValueMeta() { + throw new RuntimeException("Cannot look up value meta on non-map type"); + } + + @Override + public List getFields() { + throw new RuntimeException("Cannot list fields on non-struct type"); + } + + @Override + public Field getFieldByName(String fieldName) { + throw new RuntimeException("Cannot look up fields on non-struct type"); + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("MetaBase{"); + if (this.name != null) { + sb.append("name=").append(this.name).append(", "); + } + if (this.version != null) { + sb.append("version=").append(this.version).append(", "); + } + sb.append("type=").append(this.type).append(", "); + if (this.dataSource != null) { + sb.append("dataSource=").append(this.dataSource).append(", "); + } + sb.append("parameters=").append(this.parameters).append("}"); + return sb.toString(); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/MetaBaseBuilder.java b/connector/src/main/java/io/openmessaging/connector/api/data/MetaBaseBuilder.java new file mode 100644 index 0000000..8d0adb7 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/MetaBaseBuilder.java @@ -0,0 +1,54 @@ +package io.openmessaging.connector.api.data; + +import java.util.Collections; +import java.util.List; + +/** + * MetaBuilder implement for base types. + * {@link Type}.INT8 {@link Type}.INT16 {@link Type}.INT32 {@link Type}.INT64 + * {@link Type}.BIG_INTEGER {@link Type}.FLOAT32 {@link Type}.FLOAT64 {@link Type}.BOOLEAN + * {@link Type}.STRING {@link Type}.BYTES {@link Type}.DATETIME + * + * @author liuboyan + */ +public class MetaBaseBuilder extends MetaBuilder { + public MetaBaseBuilder(Type type) { + super(type); + } + + @Override + public MetaBuilder field(String fieldName, Meta fieldMeta) { + return this; + } + + @Override + public MetaBuilder field(Field field) { + return this; + } + + @Override + public Meta keyMeta() { + return null; + } + + @Override + public Meta valueMeta() { + return null; + } + + @Override + public List fields() { + return null; + } + + @Override + public Field getFieldByName(String fieldName) { + return null; + } + + @Override + public Meta build() { + return new MetaBase(this.type(), this.name(), this.version(), + this.dataSource(), this.parameters() == null ? null : Collections.unmodifiableMap(this.parameters())); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/MetaBuilder.java b/connector/src/main/java/io/openmessaging/connector/api/data/MetaBuilder.java new file mode 100644 index 0000000..14be2cf --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/MetaBuilder.java @@ -0,0 +1,269 @@ +package io.openmessaging.connector.api.data; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + *

+ * MetaBuilder provides a fluent API for constructing {@link Meta} objects. It allows you to set each of the + * properties for the meta and each call returns the MetaBuilder so the calls can be chained. When nested + * types are required, use one of the predefined metas from {@link Meta} or use a second MetaBuilder inline. + *

+ *

+ * Here is an example of building a struct meta: + * 

+ *     Meta dateMeta = MetaBuilder.struct()
+ *         .name("com.example.CalendarDate").version(2)
+ *         .field("month", Meta.STRING_META)
+ *         .field("day", Meta.INT8_META)
+ *         .field("year", Meta.INT16_META)
+ *         .build();
+ *
+ *

+ *

+ * Here is an example of using a second MetaBuilder to construct complex, nested types: + * 

+ *     Meta userListMeta = MetaBuilder.array(
+ *         MetaBuilder.struct().name("com.example.User").field("username", Meta.STRING_META).field("id", Meta
+ *         .INT64_META).build()
+ *     ).build();
+ *
+ *

+ * + * @author liuboyan + */ +public abstract class MetaBuilder { + + private static final String TYPE_FIELD = "type"; + private static final String NAME_FIELD = "name"; + private static final String DATASOURCE = "dataSource"; + + private final Type type; + private String name; + private Integer version; + private String dataSource; + private Map parameters; + + public MetaBuilder(Type type) { + if (null == type) { + throw new RuntimeException("type cannot be null"); + } + this.type = type; + } + + public Integer version() { + return this.version; + } + + public MetaBuilder version(Integer version) { + this.version = version; + return this; + } + + public String name() { + return this.name; + } + + public MetaBuilder name(String name) { + checkCanSet(NAME_FIELD, this.name, name); + this.name = name; + return this; + } + + public String dataSource() { + return this.dataSource; + } + + public MetaBuilder dataSource(String dataSource) { + checkCanSet(DATASOURCE, this.dataSource, dataSource); + this.dataSource = dataSource; + return this; + } + + public Map parameters() { + return this.parameters == null ? null : Collections.unmodifiableMap(this.parameters); + } + + public MetaBuilder parameter(String propertyName, String propertyValue) { + // Preserve order of insertion with a LinkedHashMap. This isn't strictly necessary, but is nice if logical types + // can print their properties in a consistent order. + if (this.parameters == null) { + this.parameters = new LinkedHashMap<>(); + } + this.parameters.put(propertyName, propertyValue); + return this; + } + + public MetaBuilder parameters(Map props) { + // Avoid creating an empty set of properties so we never have an empty map + if (props.isEmpty()) { + return this; + } + if (this.parameters == null) { + this.parameters = new LinkedHashMap<>(); + } + this.parameters.putAll(props); + return this; + } + + public Type type() { + return this.type; + } + + public static MetaBuilder type(Type type) { + switch (type) { + case MAP: + return new MetaMapBuilder(type, null, null); + case ARRAY: + return new MetaArrayBuilder(type, null); + case STRUCT: + return new MetaStructBuilder(type); + default: + return new MetaBaseBuilder(type); + } + } + + public abstract MetaBuilder field(String fieldName, Meta fieldMeta); + + public abstract MetaBuilder field(Field field); + + public abstract Meta keyMeta(); + + public abstract Meta valueMeta(); + + public abstract List fields(); + + public abstract Field getFieldByName(String fieldName); + + public abstract Meta build(); + + /** + * Return a concrete instance of the {@link Meta} specified by this builder + * + * @return the {@link Meta} + */ + public Meta meta() { + return build(); + } + + private static void checkCanSet(String fieldName, Object fieldVal, Object val) { + if (fieldVal != null && fieldVal != val) { + throw new RuntimeException("Invalid MetaBuilder call: " + fieldName + " has already been set."); + } + } + + private static void checkNotNull(String fieldName, Object val, String fieldToSet) { + if (val == null) { + throw new RuntimeException( + "Invalid MetaBuilder call: " + fieldName + " must be specified to set " + fieldToSet); + } + } + + // Basic types + + /** + * @return a new {@link Type#INT8} MetaBuilder + */ + public static MetaBuilder int8() { + return new MetaBaseBuilder(Type.INT8); + } + + /** + * @return a new {@link Type#INT16} MetaBuilder + */ + public static MetaBuilder int16() { + return new MetaBaseBuilder(Type.INT16); + } + + /** + * @return a new {@link Type#INT32} MetaBuilder + */ + public static MetaBuilder int32() { + return new MetaBaseBuilder(Type.INT32); + } + + /** + * @return a new {@link Type#INT64} MetaBuilder + */ + public static MetaBuilder int64() { + return new MetaBaseBuilder(Type.INT64); + } + + /** + * @return a new {@link Type#FLOAT32} MetaBuilder + */ + public static MetaBuilder float32() { + return new MetaBaseBuilder(Type.FLOAT32); + } + + /** + * @return a new {@link Type#FLOAT64} MetaBuilder + */ + public static MetaBuilder float64() { + return new MetaBaseBuilder(Type.FLOAT64); + } + + /** + * @return a new {@link Type#BOOLEAN} MetaBuilder + */ + public static MetaBuilder bool() { + return new MetaBaseBuilder(Type.BOOLEAN); + } + + /** + * @return a new {@link Type#STRING} MetaBuilder + */ + public static MetaBuilder string() { + return new MetaBaseBuilder(Type.STRING); + } + + /** + * @return a new {@link Type#BYTES} MetaBuilder + */ + public static MetaBuilder bytes() { + return new MetaBaseBuilder(Type.BYTES); + } + + // Structs + + /** + * @return a new {@link Type#STRUCT} MetaBuilder + */ + public static MetaBuilder struct() { + return new MetaStructBuilder(Type.STRUCT); + } + + // Arrays + + /** + * @param valueMeta the meta for elements of the array + * @return a new {@link Type#ARRAY} MetaBuilder + */ + public static MetaBuilder array(Meta valueMeta) { + if (null == valueMeta) { + throw new RuntimeException("valueMeta cannot be null."); + } + MetaBuilder builder = new MetaArrayBuilder(Type.ARRAY, valueMeta); + return builder; + } + + // Maps + + /** + * @param keyMeta the meta for keys in the map + * @param valueMeta the meta for values in the map + * @return a new {@link Type#MAP} MetaBuilder + */ + public static MetaBuilder map(Meta keyMeta, Meta valueMeta) { + if (null == keyMeta) { + throw new RuntimeException("keyMeta cannot be null."); + } + if (null == valueMeta) { + throw new RuntimeException("valueMeta cannot be null."); + } + MetaBuilder builder = new MetaMapBuilder(Type.MAP, keyMeta, valueMeta); + return builder; + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/MetaMap.java b/connector/src/main/java/io/openmessaging/connector/api/data/MetaMap.java new file mode 100644 index 0000000..928e0f9 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/MetaMap.java @@ -0,0 +1,103 @@ +package io.openmessaging.connector.api.data; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Meta implement for type of MAP. + * + * @author liuboyan + */ +public class MetaMap extends Meta { + private final Meta keyMeta; + private final Meta valueMeta; + + /** + * Base construct. + * + * @param type {@link Type} + * @param name Name of the meta. + * @param version Version of the meta. + * @param dataSource Data source information. + * @param parameters Possible parameters. + */ + public MetaMap(Type type, String name, Integer version, String dataSource, Map parameters, Meta keyMeta, Meta valueMeta) { + super(type, name, version, dataSource, parameters); + if(type != Type.MAP){ + throw new RuntimeException("type should be MAP."); + } + this.keyMeta = keyMeta; + this.valueMeta = valueMeta; + } + + @Override + public Meta getKeyMeta() { + return this.keyMeta; + } + + @Override + public Meta getValueMeta() { + return this.valueMeta; + } + + @Override + public List getFields() { + throw new RuntimeException("Cannot list fields on non-struct type"); + } + + @Override + public Field getFieldByName(String fieldName) { + throw new RuntimeException("Cannot look up fields on non-struct type"); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + + MetaMap meta = (MetaMap) o; + return Objects.equals(this.keyMeta, meta.keyMeta) + && Objects.equals(this.valueMeta, meta.valueMeta); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + this.keyMeta.hashCode(); + result = 31 * result + this.valueMeta.hashCode(); + return result; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("MetaMap{"); + if(this.name != null){ + sb.append("name=").append(this.name).append(", "); + } + if (this.version != null) { + sb.append("version=").append(this.version).append(", "); + } + sb.append("type=").append(this.type).append(", "); + if(this.dataSource != null){ + sb.append("dataSource=").append(this.dataSource).append(", "); + } + if(this.keyMeta!=null){ + sb.append("keyMeta=").append(this.keyMeta).append(", "); + } + if(this.valueMeta!=null){ + sb.append("valueMeta=").append(this.valueMeta).append(", "); + } + sb.append("parameters=").append(this.parameters).append("}"); + return sb.toString(); + } + +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/MetaMapBuilder.java b/connector/src/main/java/io/openmessaging/connector/api/data/MetaMapBuilder.java new file mode 100644 index 0000000..9732f0b --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/MetaMapBuilder.java @@ -0,0 +1,57 @@ +package io.openmessaging.connector.api.data; + +import java.util.Collections; +import java.util.List; + +/** + * MetaBuilder implement for type of MAP. + * + * @author liuboyan + */ +public class MetaMapBuilder extends MetaBuilder { + private Meta keyMeta; + private Meta valueMeta; + + public MetaMapBuilder(Type type, Meta keyMeta, Meta valueMeta) { + super(type); + this.keyMeta = keyMeta; + this.valueMeta = valueMeta; + } + + @Override + public MetaBuilder field(String fieldName, Meta fieldMeta) { + throw new RuntimeException("Cannot create fields on type " + this.type()); + } + + @Override + public MetaBuilder field(Field field) { + throw new RuntimeException("Cannot create fields on type " + this.type()); + } + + @Override + public Meta keyMeta() { + return this.keyMeta; + } + + @Override + public Meta valueMeta() { + return this.valueMeta; + } + + @Override + public List fields() { + throw new RuntimeException("Cannot list fields on non-struct type"); + } + + @Override + public Field getFieldByName(String fieldName) { + throw new RuntimeException("Cannot look up fields on non-struct type"); + } + + @Override + public Meta build() { + return new MetaMap(this.type(), this.name(), this.version(), + this.dataSource(), this.parameters() == null ? null : Collections.unmodifiableMap(this.parameters()), + this.keyMeta, this.valueMeta); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/MetaStruct.java b/connector/src/main/java/io/openmessaging/connector/api/data/MetaStruct.java new file mode 100644 index 0000000..00ad2e7 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/MetaStruct.java @@ -0,0 +1,110 @@ +package io.openmessaging.connector.api.data; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Meta implement for type of STRUCT. + * + * @author liuboyan + */ +public class MetaStruct extends Meta { + + /** + * Structure of the meta, contains a list of {@link Field} + */ + private List fields; + /** + * field name 和 field的对应map，便于快速查找 + */ + private Map fieldsByName; + + /** + * Base construct. + * + * @param type {@link Type} + * @param name Name of the meta. + * @param version Version of the meta. + * @param dataSource Data source information. + * @param parameters Possible parameters. + */ + public MetaStruct(Type type, String name, Integer version, String dataSource, Map parameters, List fields) { + super(type, name, version, dataSource, parameters); + if(type != Type.STRUCT){ + throw new RuntimeException("type should be STRUCT."); + } + this.fields = fields == null ? Collections.emptyList() : fields; + this.fieldsByName = new HashMap<>(this.fields.size()); + for (Field field : this.fields) { + fieldsByName.put(field.name(), field); + } + } + + + @Override + public Meta getKeyMeta() { + throw new RuntimeException("Cannot look up key meta on non-map type"); + } + + @Override + public Meta getValueMeta() { + throw new RuntimeException("Cannot look up value meta on non-map type"); + } + + @Override + public List getFields() { + return this.fields; + } + + @Override + public Field getFieldByName(String fieldName) { + return this.fieldsByName.get(fieldName); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + + MetaStruct meta = (MetaStruct) o; + return Objects.equals(this.fields, meta.fields); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + this.fields.hashCode(); + return result; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("MetaStruct{"); + if(this.name != null){ + sb.append("name=").append(this.name).append(", "); + } + if (this.version != null) { + sb.append("version=").append(this.version).append(", "); + } + sb.append("type=").append(this.type).append(", "); + if(this.dataSource != null){ + sb.append("dataSource=").append(this.dataSource).append(", "); + } + if(this.fields!=null){ + sb.append("fields=").append(this.fields).append(", "); + } + sb.append("parameters=").append(this.parameters).append("}"); + return sb.toString(); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/MetaStructBuilder.java b/connector/src/main/java/io/openmessaging/connector/api/data/MetaStructBuilder.java new file mode 100644 index 0000000..4c1d865 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/MetaStructBuilder.java @@ -0,0 +1,72 @@ +package io.openmessaging.connector.api.data; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * MetaBuilder implement for type of STRUCT. + * + * @author liuboyan + */ +public class MetaStructBuilder extends MetaBuilder { + private Map fields; + + public MetaStructBuilder(Type type) { + super(type); + this.fields = new LinkedHashMap<>(); + } + + @Override + public MetaBuilder field(String fieldName, Meta fieldMeta) { + if (null == fieldName || fieldName.isEmpty()) { + throw new RuntimeException("fieldName cannot be null."); + } + if (null == fieldMeta) { + throw new RuntimeException("fieldMeta for field " + fieldName + " cannot be null."); + } + int fieldIndex = this.fields.size(); + if (this.fields.containsKey(fieldName)) { + throw new RuntimeException("Cannot create field because of field name duplication " + fieldName); + } + this.fields.put(fieldName, new Field(fieldIndex, fieldName, fieldMeta)); + return this; + } + + @Override + public MetaBuilder field(Field field) { + return this.field(field.name(), field.meta()); + } + + @Override + public Meta keyMeta() { + return null; + } + + @Override + public Meta valueMeta() { + return null; + } + + @Override + public List fields() { + return new ArrayList<>(this.fields.values()); + } + + @Override + public Field getFieldByName(String fieldName) { + return fields.get(fieldName); + } + + @Override + public Meta build() { + return new MetaStruct(this.type(), + this.name(), + this.version(), + this.dataSource(), + this.parameters() == null ? null : Collections.unmodifiableMap(this.parameters()), + fields == null ? null : Collections.unmodifiableList(new ArrayList<>(fields.values()))); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/Schema.java b/connector/src/main/java/io/openmessaging/connector/api/data/Schema.java deleted file mode 100644 index 5c7a684..0000000 --- a/connector/src/main/java/io/openmessaging/connector/api/data/Schema.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.openmessaging.connector.api.data; - -import java.util.List; - -/** - * Schema - * - * @version OMS 0.1.0 - * @since OMS 0.1.0 - */ -public class Schema { - - /** - * Data source information. - */ - private String dataSource; - /** - * Name of the schema. - */ - private String name; - /** - * Structure of the schema, contains a list of {@link Field} - */ - private List fields; - - public String getDataSource() { - return dataSource; - } - - public void setDataSource(String dataSource) { - this.dataSource = dataSource; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public List getFields() { - return fields; - } - - public void setFields(List fields) { - this.fields = fields; - } - - public Field getField(String fieldName) { - - for (Field field : fields) { - if (field.getName().equals(fieldName)) { - return field; - } - } - return null; - } -} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/SinkDataEntry.java b/connector/src/main/java/io/openmessaging/connector/api/data/SinkDataEntry.java index be37f2b..a462797 100644 --- a/connector/src/main/java/io/openmessaging/connector/api/data/SinkDataEntry.java +++ b/connector/src/main/java/io/openmessaging/connector/api/data/SinkDataEntry.java @@ -17,6 +17,8 @@ package io.openmessaging.connector.api.data; +import io.openmessaging.connector.api.header.Headers; + /** * SinkDataEntry is read from message queue and includes the queueOffset of the data in message queue. * @@ -24,21 +26,96 @@ * @since OMS 0.1.0 */ public class SinkDataEntry extends DataEntry { + /** + * Offset in the message queue. + */ + private Long queueOffset; + public SinkDataEntry(Long queueOffset, Long timestamp, - EntryType entryType, String queueName, - Schema schema, - Object[] payload) { - super(timestamp, entryType, queueName, schema, payload); + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value, + Headers headers) { + super(timestamp, queueName, queueId, entryType, key, value, headers); this.queueOffset = queueOffset; } - /** - * Offset in the message queue. - */ - private Long queueOffset; + public SinkDataEntry(Long queueOffset, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value, + Headers headers) { + this(queueOffset, null, queueName, queueId, entryType, key, value, headers); + } + + public SinkDataEntry(Long queueOffset, + Long timestamp, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value) { + this(queueOffset, timestamp, queueName, queueId, entryType, key, value, null); + } + + + public SinkDataEntry newRecord(Long queueOffset, + Long timestamp, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value) { + return new SinkDataEntry(queueOffset, timestamp, queueName, queueId, entryType, key, value, getHeaders().duplicate()); + } + + public SinkDataEntry newRecord(Long queueOffset, + Long timestamp, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value, + Headers headers) { + return new SinkDataEntry(queueOffset, timestamp, queueName, queueId, entryType, key, value, headers); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + + SinkDataEntry that = (SinkDataEntry) o; + + return queueOffset.equals(that.queueOffset); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + Long.hashCode(queueOffset); + return result; + } + + @Override + public String toString() { + return "SinkDataEntry{" + + "queueOffset=" + queueOffset + + "} " + super.toString(); + } public Long getQueueOffset() { return queueOffset; diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/SourceDataEntry.java b/connector/src/main/java/io/openmessaging/connector/api/data/SourceDataEntry.java index 5d8a769..16742df 100644 --- a/connector/src/main/java/io/openmessaging/connector/api/data/SourceDataEntry.java +++ b/connector/src/main/java/io/openmessaging/connector/api/data/SourceDataEntry.java @@ -18,6 +18,9 @@ package io.openmessaging.connector.api.data; import java.nio.ByteBuffer; +import java.util.Objects; + +import io.openmessaging.connector.api.header.Headers; /** * SourceDataEntries are generated by SourceTasks and passed to specific message queue to store. @@ -26,28 +29,132 @@ * @since OMS 0.1.0 */ public class SourceDataEntry extends DataEntry { + /** + * Partition of the data source. + */ + private ByteBuffer sourcePartition; + + /** + * Position of current data entry of {@link SourceDataEntry#sourcePartition}. + */ + private ByteBuffer sourcePosition; + public SourceDataEntry(ByteBuffer sourcePartition, ByteBuffer sourcePosition, Long timestamp, - EntryType entryType, String queueName, - Schema schema, - Object[] payload) { - super(timestamp, entryType, queueName, schema, payload); + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value, + Headers headers) { + super(timestamp, queueName, queueId, entryType, key, value, headers); this.sourcePartition = sourcePartition; this.sourcePosition = sourcePosition; } - /** - * Partition of the data source. - */ - private ByteBuffer sourcePartition; - /** - * Position of current data entry of {@link SourceDataEntry#sourcePartition}. - */ - private ByteBuffer sourcePosition; + + public SourceDataEntry(ByteBuffer sourcePartition, + ByteBuffer sourcePosition, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData value, + Headers headers) { + this(sourcePartition, sourcePosition, null, + queueName, queueId, entryType, + null, value, headers); + } + + + public SourceDataEntry(ByteBuffer sourcePartition, + ByteBuffer sourcePosition, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData value) { + this(sourcePartition, sourcePosition, null, + queueName, queueId, entryType, + null, value, null); + } + + public SourceDataEntry(ByteBuffer sourcePartition, + ByteBuffer sourcePosition, + String queueName, + EntryType entryType, + MetaAndData value) { + this(sourcePartition, sourcePosition, null, + queueName, null, entryType, + null, value, null); + } + + public SourceDataEntry(ByteBuffer sourcePartition, + ByteBuffer sourcePosition, + String queueName, + EntryType entryType, + MetaAndData key, + MetaAndData value) { + this(sourcePartition, sourcePosition, null, + queueName, null, entryType, + key, value, null); + } + + public SourceDataEntry(ByteBuffer sourcePartition, + ByteBuffer sourcePosition, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value) { + this(sourcePartition, sourcePosition, null, + queueName, queueId, entryType, + key, value, null); + } + + public SourceDataEntry(ByteBuffer sourcePartition, + ByteBuffer sourcePosition, + Long timestamp, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value) { + this(sourcePartition, sourcePosition, timestamp, + queueName, queueId, entryType, + key, value, null); + } + + + + + public SourceDataEntry newDataEntry( + Long timestamp, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value) { + return new SourceDataEntry(this.sourcePartition, this.sourcePosition, + timestamp, queueName, queueId, entryType, key, value, getHeaders().duplicate()); + } + + public SourceDataEntry newDataEntry( + Long timestamp, + String queueName, + Integer queueId, + EntryType entryType, + MetaAndData key, + MetaAndData value, + Headers headers) { + return new SourceDataEntry(this.sourcePartition, this.sourcePosition, + timestamp, queueName, queueId, entryType, key, value, headers); + } + + + + public ByteBuffer getSourcePartition() { return sourcePartition; @@ -64,4 +171,39 @@ public ByteBuffer getSourcePosition() { public void setSourcePosition(ByteBuffer sourcePosition) { this.sourcePosition = sourcePosition; } + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + + SourceDataEntry that = (SourceDataEntry) o; + + return Objects.equals(this.sourcePartition, that.sourcePartition) && + Objects.equals(this.sourcePosition, that.sourcePosition); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (this.sourcePartition != null ? this.sourcePartition.hashCode() : 0); + result = 31 * result + (this.sourcePosition != null ? this.sourcePosition.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "SourceDataEntry{" + + "sourcePartition=" + this.sourcePartition + + ", sourcePosition=" + this.sourcePosition + + "} " + super.toString(); + } } diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/Struct.java b/connector/src/main/java/io/openmessaging/connector/api/data/Struct.java new file mode 100644 index 0000000..6afb3c0 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/Struct.java @@ -0,0 +1,247 @@ +package io.openmessaging.connector.api.data; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * A structured record containing a set of named fields with values, each field using an independent {@link Meta}. + * Struct objects must specify a complete {@link Meta} up front, and only fields specified in the Meta may be set. + * + * @author liuboyan + */ +public class Struct { + + private final Meta meta; + private final Object[] values; + + public Struct(Meta meta) { + if (meta.getType() != Type.STRUCT) { + throw new RuntimeException("meta type should be STRUCT."); + } + this.meta = meta; + this.values = new Object[meta.getFields().size()]; + } + + public Meta meta() { + return meta; + } + + public Object get(String fieldName) { + Field field = lookupField(fieldName); + return get(field); + } + + public Object get(Field field) { + return values[field.index()]; + } + + public List getFields() { + return this.meta.getFields(); + } + + // Note that all getters have to have boxed return types since the fields might be optional + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a Byte. + */ + public Byte getInt8(String fieldName) { + return (Byte)getCheckType(fieldName, Type.INT8); + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a Short. + */ + public Short getInt16(String fieldName) { + return (Short)getCheckType(fieldName, Type.INT16); + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a Integer. + */ + public Integer getInt32(String fieldName) { + return (Integer)getCheckType(fieldName, Type.INT32); + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a Long. + */ + public Long getInt64(String fieldName) { + return (Long)getCheckType(fieldName, Type.INT64); + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a Float. + */ + public Float getFloat32(String fieldName) { + return (Float)getCheckType(fieldName, Type.FLOAT32); + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a Double. + */ + public Double getFloat64(String fieldName) { + return (Double)getCheckType(fieldName, Type.FLOAT64); + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a Boolean. + */ + public Boolean getBoolean(String fieldName) { + return (Boolean)getCheckType(fieldName, Type.BOOLEAN); + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a String. + */ + public String getString(String fieldName) { + return (String)getCheckType(fieldName, Type.STRING); + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a byte[]. + */ + public byte[] getBytes(String fieldName) { + Object bytes = getCheckType(fieldName, Type.BYTES); + if (bytes instanceof ByteBuffer) { + return ((ByteBuffer)bytes).array(); + } + return (byte[])bytes; + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a List. + */ + @SuppressWarnings("unchecked") + public List getArray(String fieldName) { + return (List)getCheckType(fieldName, Type.ARRAY); + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a Map. + */ + @SuppressWarnings("unchecked") + public Map getMap(String fieldName) { + return (Map)getCheckType(fieldName, Type.MAP); + } + + /** + * Equivalent to calling {@link #get(String)} and casting the result to a Struct. + */ + public Struct getStruct(String fieldName) { + return (Struct)getCheckType(fieldName, Type.STRUCT); + } + + public Object getObject(String fieldName) { + Field field = lookupField(fieldName); + return values[field.index()]; + } + + /** + * Set the value of a field. Validates the value, throwing a {@link RuntimeException} if it does not match the + * field's + * {@link Meta}. + * + * @param fieldName the name of the field to set + * @param value the value of the field + * @return the Struct, to allow chaining of {@link #put(String, Object)} calls + */ + public Struct put(String fieldName, Object value) { + Field field = lookupField(fieldName); + return put(field, value); + } + + /** + * Set the value of a field. Validates the value, throwing a {@link RuntimeException} if it does not match the + * field's + * {@link Meta}. + * + * @param field the field to set + * @param value the value of the field + * @return the Struct, to allow chaining of {@link #put(String, Object)} calls + */ + public Struct put(Field field, Object value) { + if (null == field) { + throw new RuntimeException("field cannot be null."); + } + Meta.validateValue(field.name(), field.meta(), value); + values[field.index()] = value; + return this; + } + + public MetaAndData toMetaData() { + return new MetaAndData(this.meta, this); + } + + /** + * Validates that this struct has filled in all the necessary data with valid values. For required fields + * without defaults, this validates that a value has been set and has matching types/metas. If any validation + * fails, throws a DataException. + */ + public void validate() { + for (Field field : meta.getFields()) { + Meta fieldMeta = field.meta(); + Object value = values[field.index()]; + Meta.validateValue(field.name(), fieldMeta, value); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Struct struct = (Struct)o; + return Objects.equals(meta, struct.meta) && + Arrays.deepEquals(values, struct.values); + } + + @Override + public int hashCode() { + return Objects.hash(meta, Arrays.deepHashCode(values)); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Struct{["); + boolean first = true; + for (int i = 0; i < values.length; i++) { + final Object value = values[i]; + if (value != null) { + final Field field = meta.getFields().get(i); + if (first) { + first = false; + } else { + sb.append(","); + } + sb.append(field.toString()).append("=").append(value); + } + } + return sb.append("]}").toString(); + } + + private Field lookupField(String fieldName) { + Field field = meta.getFieldByName(fieldName); + if (field == null) { + throw new RuntimeException(fieldName + " is not a valid field name"); + } + return field; + } + + // Get the field's value, but also check that the field matches the specified type, throwing an exception if it + // doesn't. + // Used to implement the get*() methods that return typed data instead of Object + private Object getCheckType(String fieldName, Type type) { + Field field = lookupField(fieldName); + if (field.meta().getType() != type) { + throw new RuntimeException("Field '" + fieldName + "' is not of type " + type); + } + return values[field.index()]; + } + +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/Time.java b/connector/src/main/java/io/openmessaging/connector/api/data/Time.java new file mode 100644 index 0000000..23541a1 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/Time.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.connector.api.data; + + +import java.util.Calendar; +import java.util.TimeZone; + +/** + *

+ * A time representing a specific point in a day, not tied to any specific date. The corresponding Java type is a + * java.util.Date where only hours, minutes, seconds, and milliseconds can be non-zero. This effectively makes it a + * point in time during the first day after the Unix epoch. The underlying representation is an integer + * representing the number of milliseconds after midnight. + *

+ */ +public class Time { + public static final String LOGICAL_NAME = "io.openmessaging.connector.api.data.Time"; + + private static final long MILLIS_PER_DAY = 24 * 60 * 60 * 1000; + + private static final TimeZone UTC = TimeZone.getTimeZone("UTC"); + + /** + * Returns a MetaBuilder for a Time. By returning a MetaBuilder you can override additional meta settings such + * as required/optional, default value, and documentation. + * @return a MetaBuilder + */ + public static MetaBuilder builder() { + return MetaBuilder.int32() + .name(LOGICAL_NAME); + } + + public static final Meta META = builder().meta(); + + /** + * Convert a value from its logical format (Time) to it's encoded format. + * @param value the logical value + * @return the encoded value + */ + public static int fromLogical(Meta meta, java.util.Date value) { + if (!(LOGICAL_NAME.equals(meta.getName()))) { + throw new RuntimeException("Requested conversion of Time object but the meta does not match."); + } + Calendar calendar = Calendar.getInstance(UTC); + calendar.setTime(value); + long unixMillis = calendar.getTimeInMillis(); + if (unixMillis < 0 || unixMillis > MILLIS_PER_DAY) { + throw new RuntimeException("RocketMQ Connect Time type should not have any date fields set to non-zero values."); + } + return (int) unixMillis; + } + + public static java.util.Date toLogical(Meta meta, int value) { + if (!(LOGICAL_NAME.equals(meta.getName()))) { + throw new RuntimeException("Requested conversion of Date object but the meta does not match."); + } + if (value < 0 || value > MILLIS_PER_DAY) { + throw new RuntimeException("Time values must use number of milliseconds greater than 0 and less than 86400000"); + } + return new java.util.Date(value); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/Timestamp.java b/connector/src/main/java/io/openmessaging/connector/api/data/Timestamp.java new file mode 100644 index 0000000..71ac0eb --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/Timestamp.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.connector.api.data; + + +/** + *

+ * A timestamp representing an absolute time, without timezone information. The corresponding Java type is a + * java.util.Date. The underlying representation is a long representing the number of milliseconds since Unix epoch. + *

+ */ +public class Timestamp { + public static final String LOGICAL_NAME = "io.openmessaging.connector.api.data.Timestamp"; + + /** + * Returns a MetaBuilder for a Timestamp. By returning a MetaBuilder you can override additional meta settings such + * as required/optional, default value, and documentation. + * @return a MetaBuilder + */ + public static MetaBuilder builder() { + return MetaBuilder.int64() + .name(LOGICAL_NAME); + } + + public static final Meta META = builder().meta(); + + /** + * Convert a value from its logical format (Date) to it's encoded format. + * @param value the logical value + * @return the encoded value + */ + public static long fromLogical(Meta meta, java.util.Date value) { + if (!(LOGICAL_NAME.equals(meta.getName()))) { + throw new RuntimeException("Requested conversion of Timestamp object but the meta does not match."); + } + return value.getTime(); + } + + public static java.util.Date toLogical(Meta meta, long value) { + if (!(LOGICAL_NAME.equals(meta.getName()))) { + throw new RuntimeException("Requested conversion of Timestamp object but the meta does not match."); + } + return new java.util.Date(value); + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/data/Type.java b/connector/src/main/java/io/openmessaging/connector/api/data/Type.java new file mode 100644 index 0000000..b36de47 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/data/Type.java @@ -0,0 +1,96 @@ +package io.openmessaging.connector.api.data; + +import java.util.Locale; + +/** + * The type of a meta. These only include the core types; + * logical types must be determined by checking the meta name. + */ +public enum Type { + /** + * 8-bit signed integer + */ + INT8, + /** + * 16-bit signed integer + */ + INT16, + /** + * 32-bit signed integer + */ + INT32, + /** + * 64-bit signed integer + */ + INT64, + /** + * BigInteger + */ + BIG_INTEGER, + /** + * 32-bit IEEE 754 floating point number + */ + FLOAT32, + /** + * 64-bit IEEE 754 floating point number + */ + FLOAT64, + /** + * Boolean value (true or false) + */ + BOOLEAN, + /** + * Character string that supports all Unicode characters. + * + * Note that this does not imply any specific encoding (e.g. UTF-8) as this is an in-memory representation. + */ + STRING, + /** + * Sequence of unsigned 8-bit bytes + */ + BYTES, + /** + * Date + */ + DATETIME, + + /** + * An ordered sequence of elements, each of which shares the same type. + */ + ARRAY, + /** + * A mapping from keys to values. Both keys and values can be arbitrarily complex types, including complex types + */ + MAP, + /** + * A structured record containing a set of named fields, each field using a fixed, independent. + */ + STRUCT; + + private String name; + + Type() { + this.name = this.name().toLowerCase(Locale.ROOT); + } + + public String getName() { + return name; + } + + public boolean isPrimitive() { + switch (this) { + case INT8: + case INT16: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BOOLEAN: + case STRING: + case BYTES: + return true; + } + return false; + } + +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/header/DataHeader.java b/connector/src/main/java/io/openmessaging/connector/api/header/DataHeader.java new file mode 100644 index 0000000..20b935a --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/header/DataHeader.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.connector.api.header; + + +import java.util.Objects; + +import io.openmessaging.connector.api.data.Meta; +import io.openmessaging.connector.api.data.MetaAndData; + +/** + * A {@link Header} implementation. + */ +public class DataHeader implements Header { + + private final String key; + private final MetaAndData metaAndData; + + public DataHeader(String key, MetaAndData metaAndData) { + Objects.requireNonNull(key, "Null header keys are not permitted"); + this.key = key; + this.metaAndData = metaAndData; + } + + public DataHeader(String key, Meta meta, Object value) { + this(key,new MetaAndData(meta, value)); + } + + @Override + public MetaAndData data() { + return this.metaAndData; + } + + @Override + public String key() { + return this.key; + } + + @Override + public Header rename(String key) { + Objects.requireNonNull(key, "Null header keys are not permitted"); + if (this.key.equals(key)) { + return this; + } + return new DataHeader(key, this.metaAndData); + } + + @Override + public Header with(Meta meta, Object value) { + return new DataHeader(key, this.metaAndData); + } + + @Override + public int hashCode() { + return Objects.hash(this.key, this.metaAndData); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj instanceof Header) { + Header that = (Header) obj; + return Objects.equals(this.key, that.key()) + && Objects.equals(this.data(), that.data()); + } + return false; + } + + @Override + public String toString() { + return "DataHeader(key=" + key + ", value=" + data().getData() + ", meta=" + data().getMeta() + ")"; + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/header/DataHeaders.java b/connector/src/main/java/io/openmessaging/connector/api/header/DataHeaders.java new file mode 100644 index 0000000..b5bd2b7 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/header/DataHeaders.java @@ -0,0 +1,502 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.connector.api.header; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; + +import io.openmessaging.connector.api.data.Date; +import io.openmessaging.connector.api.data.Decimal; +import io.openmessaging.connector.api.data.Meta; +import io.openmessaging.connector.api.data.Struct; +import io.openmessaging.connector.api.data.Time; +import io.openmessaging.connector.api.data.Timestamp; +import io.openmessaging.connector.api.data.Type; + +/** + * A basic {@link Headers} implementation. + */ +public class DataHeaders implements Headers { + + private static final int EMPTY_HASH = Objects.hash(new LinkedList<>()); + + /** + * An immutable and therefore sharable empty iterator. + */ + private static final Iterator
EMPTY_ITERATOR = new Iterator
() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public Header next() { + throw new NoSuchElementException(); + } + + @Override + public void remove() { + throw new IllegalStateException(); + } + }; + + /** + * This field is set lazily, but once set to a list it is never set back to null + */ + private Map headers; + + public DataHeaders() { + } + + public DataHeaders(Iterable
original) { + if (original == null) { + return; + } + if (original instanceof DataHeaders) { + DataHeaders originalHeaders = (DataHeaders) original; + if (!originalHeaders.isEmpty()) { + headers = new LinkedHashMap<>(originalHeaders.headers); + } + } else { + headers = new LinkedHashMap<>(); + for (Header header : original) { + headers.put(header.key(), header); + } + } + } + + @Override + public int size() { + return headers == null ? 0 : headers.size(); + } + + @Override + public boolean isEmpty() { + return headers == null ? true : headers.isEmpty(); + } + + @Override + public Headers clear() { + if (headers != null) { + headers.clear(); + } + return this; + } + + @Override + public Headers add(Header header) { + Objects.requireNonNull(header, "Unable to add a null header."); + if (headers == null) { + headers = new LinkedHashMap<>(); + } + headers.put(header.key(), header); + return this; + } + + protected Headers addWithoutValidating(String key, Object value, Meta meta) { + return add(new DataHeader(key, meta, value)); + } + + @Override + public Headers add(String key, Meta meta, Object value) { + checkMetaMatches(meta, value); + return add(new DataHeader(key, meta, value)); + } + + @Override + public Headers addString(String key, String value) { + return addWithoutValidating(key, value, Meta.STRING_META); + } + + @Override + public Headers addBytes(String key, byte[] value) { + return addWithoutValidating(key, value, Meta.BYTES_META); + } + + @Override + public Headers addBoolean(String key, boolean value) { + return addWithoutValidating(key, value, Meta.BOOLEAN_META); + } + + @Override + public Headers addByte(String key, byte value) { + return addWithoutValidating(key, value, Meta.INT8_META); + } + + @Override + public Headers addShort(String key, short value) { + return addWithoutValidating(key, value, Meta.INT16_META); + } + + @Override + public Headers addInt(String key, int value) { + return addWithoutValidating(key, value, Meta.INT32_META); + } + + @Override + public Headers addLong(String key, long value) { + return addWithoutValidating(key, value, Meta.INT64_META); + } + + @Override + public Headers addFloat(String key, float value) { + return addWithoutValidating(key, value, Meta.FLOAT32_META); + } + + @Override + public Headers addDouble(String key, double value) { + return addWithoutValidating(key, value, Meta.FLOAT64_META); + } + + @Override + public Headers addList(String key, List value, Meta meta) { + if (value == null) { + return add(key, null, null); + } + checkMetaType(meta, Type.ARRAY); + return addWithoutValidating(key, value, meta); + } + + @Override + public Headers addMap(String key, Map value, Meta meta) { + if (value == null) { + return add(key, null, null); + } + checkMetaType(meta, Type.MAP); + return addWithoutValidating(key, value, meta); + } + + @Override + public Headers addStruct(String key, Struct value) { + if (value == null) { + return add(key, null, null); + } + checkMetaType(value.meta(), Type.STRUCT); + return addWithoutValidating(key, value, value.meta()); + } + + @Override + public Headers addDecimal(String key, BigDecimal value) { + if (value == null) { + return add(key, null, null); + } + // Check that this is a decimal ... + Meta meta = Decimal.meta(value.scale()); + Decimal.fromLogical(meta, value); + return addWithoutValidating(key, value, meta); + } + + @Override + public Headers addDate(String key, java.util.Date value) { + if (value != null) { + // Check that this is a date ... + Date.fromLogical(Date.META, value); + } + return addWithoutValidating(key, value, Date.META); + } + + @Override + public Headers addTime(String key, java.util.Date value) { + if (value != null) { + // Check that this is a time ... + Time.fromLogical(Time.META, value); + } + return addWithoutValidating(key, value, Time.META); + } + + @Override + public Headers addTimestamp(String key, java.util.Date value) { + if (value != null) { + // Check that this is a timestamp ... + Timestamp.fromLogical(Timestamp.META, value); + } + return addWithoutValidating(key, value, Timestamp.META); + } + + @Override + public Header findHeader(String key) { + return new FilterByKeyIterator(iterator(), key).makeNext(); + } + + @Override + public Map toMap(){ + return new HashMap<>(this.headers); + } + + @Override + public Iterator
iterator() { + if (headers != null) { + return headers.values().iterator(); + } + return EMPTY_ITERATOR; + } + + @Override + public Headers remove(String key) { + checkKey(key); + if (!isEmpty()) { + Iterator
iterator = iterator(); + while (iterator.hasNext()) { + if (iterator.next().key().equals(key)) { + iterator.remove(); + } + } + } + return this; + } + @Override + public int hashCode() { + return isEmpty() ? EMPTY_HASH : Objects.hash(headers); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj instanceof Headers) { + Headers that = (Headers) obj; + Iterator
thisIter = this.iterator(); + Iterator
thatIter = that.iterator(); + while (thisIter.hasNext() && thatIter.hasNext()) { + if (!Objects.equals(thisIter.next(), thatIter.next())) { + return false; + } + } + return !thisIter.hasNext() && !thatIter.hasNext(); + } + return false; + } + + @Override + public String toString() { + return "DataHeaders(headers=" + (headers != null ? headers : "") + ")"; + } + + @Override + public DataHeaders duplicate() { + return new DataHeaders(this); + } + + /** + * Check that the key is not null + * + * @param key the key; may not be null + * @throws NullPointerException if the supplied key is null + */ + private void checkKey(String key) { + Objects.requireNonNull(key, "Header key cannot be null"); + } + + /** + * Check the {@link Type() meta's type} matches the specified type. + * + * @param meta the meta; never null + * @param type the expected type + * @throws RuntimeException if the meta's type does not match the expected type + */ + private void checkMetaType(Meta meta, Type type) { + if (meta.getType() != type) { + throw new RuntimeException("Expecting " + type + " but instead found " + meta.getType()); + } + } + + /** + * Check that the value and its meta are compatible. + * + * @param meta + * @param value the meta and value pair + * @throws RuntimeException if the meta is not compatible with the value + */ + private void checkMetaMatches(Meta meta, Object value) { + if (meta != null && value != null) { + switch (meta.getType()) { + case BYTES: + if (value instanceof ByteBuffer) { + return; + } + if (value instanceof byte[]) { + return; + } + if (value instanceof BigDecimal && Decimal.LOGICAL_NAME.equals(meta.getName())) { + return; + } + break; + case STRING: + if (value instanceof String) { + return; + } + break; + case BOOLEAN: + if (value instanceof Boolean) { + return; + } + break; + case INT8: + if (value instanceof Byte) { + return; + } + break; + case INT16: + if (value instanceof Short) { + return; + } + break; + case INT32: + if (value instanceof Integer) { + return; + } + if (value instanceof java.util.Date && Date.LOGICAL_NAME.equals(meta.getName())) { + return; + } + if (value instanceof java.util.Date && Time.LOGICAL_NAME.equals(meta.getName())) { + return; + } + break; + case INT64: + if (value instanceof Long) { + return; + } + if (value instanceof java.util.Date && Timestamp.LOGICAL_NAME.equals(meta.getName())) { + return; + } + break; + case FLOAT32: + if (value instanceof Float) { + return; + } + break; + case FLOAT64: + if (value instanceof Double) { + return; + } + break; + case ARRAY: + if (value instanceof List) { + return; + } + break; + case MAP: + if (value instanceof Map) { + return; + } + break; + case STRUCT: + if (value instanceof Struct) { + return; + } + break; + } + throw new RuntimeException("The value " + value + " is not compatible with the meta " + meta); + } + } + + private static final class FilterByKeyIterator implements Iterator
{ + private enum State { + READY, NOT_READY, DONE, FAILED + } + + private State state = State.NOT_READY; + private Header next; + + + private final Iterator
original; + private final String key; + + private FilterByKeyIterator(Iterator
original, String key) { + this.original = original; + this.key = key; + } + + protected Header makeNext() { + while (original.hasNext()) { + Header header = original.next(); + if (!header.key().equals(key)) { + continue; + } + return header; + } + return this.allDone(); + } + + @Override + public boolean hasNext() { + switch (state) { + case FAILED: + throw new IllegalStateException("Iterator is in failed state"); + case DONE: + return false; + case READY: + return true; + default: + return maybeComputeNext(); + } + } + + @Override + public Header next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + state = State.NOT_READY; + if (next == null) { + throw new IllegalStateException("Expected item but none found."); + } + return next; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Removal not supported"); + } + + + + + public Header peek() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return next; + } + + protected Header allDone() { + state = State.DONE; + return null; + } + + + private Boolean maybeComputeNext() { + state = State.FAILED; + next = makeNext(); + if (state == State.DONE) { + return false; + } else { + state = State.READY; + return true; + } + } + } +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/header/Header.java b/connector/src/main/java/io/openmessaging/connector/api/header/Header.java new file mode 100644 index 0000000..647d439 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/header/Header.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.connector.api.header; + +import io.openmessaging.connector.api.data.Meta; +import io.openmessaging.connector.api.data.MetaAndData; + +/** + * A {@link Header} is a key-value pair, and multiple headers can be included with the key, value, and timestamp in each RocketMQ message. + * The data contains both the meta information and the value object. + *

+ * This is an immutable interface. + */ +public interface Header { + + /** + * Get the header's value as deserialized by Connect's header converter. + * + * @return + */ + MetaAndData data(); + + /** + * The header's key, which is not necessarily unique within the set of headers on a RocketMQ message. + * + * @return the header's key; never null + */ + String key(); + + /** + * Return a new {@link Header} object that has the same key but with the supplied value. + * + * @param meta the meta for the new value; may be null + * @param value the new value + * @return the new {@link Header}; never null + */ + Header with(Meta meta, Object value); + + /** + * Return a new {@link Header} object that has the same meta and value but with the supplied key. + * + * @param key the key for the new header; may not be null + * @return the new {@link Header}; never null + */ + Header rename(String key); +} diff --git a/connector/src/main/java/io/openmessaging/connector/api/header/Headers.java b/connector/src/main/java/io/openmessaging/connector/api/header/Headers.java new file mode 100644 index 0000000..de72637 --- /dev/null +++ b/connector/src/main/java/io/openmessaging/connector/api/header/Headers.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.openmessaging.connector.api.header; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +import io.openmessaging.connector.api.data.Meta; +import io.openmessaging.connector.api.data.Struct; + +/** + * A mutable ordered collection of {@link Header} objects. + * Note that multiple headers shouldn't have the same {@link Header#key() key}. + */ +public interface Headers extends Iterable

{ + + /** + * Get the number of headers in this object. + * + * @return the number of headers; never negative + */ + int size(); + + /** + * Determine whether this object has no headers. + * + * @return true if there are no headers, or false if there is at least one header + */ + boolean isEmpty(); + + /** + * Get the collection of {@link Header} objects whose {@link Header#key() keys} all match the specified key. + * + * @param key the key; may not be null + * @return the iterator over headers with the specified key; may be null if there are no headers with the + * specified key + */ + Header findHeader(String key); + + /** + * Get the map of {@link Header} objects. + * + * @return the map of headers + */ + Map toMap(); + + /** + * Add the given {@link Header} to this collection. + * + * @param header the header; may not be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers add(Header header); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @param meta the meta for the header's value; may not be null if the value is not null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers add(String key, Meta meta, Object value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addString(String key, String value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addBoolean(String key, boolean value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addByte(String key, byte value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addShort(String key, short value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addInt(String key, int value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addLong(String key, long value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addFloat(String key, float value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addDouble(String key, double value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addBytes(String key, byte[] value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @param meta the meta describing the list value; may not be null + * @return this object to facilitate chaining multiple methods; never null + * @throws Exception if the header's value is invalid + */ + Headers addList(String key, List value, Meta meta); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @param meta the meta describing the map value; may not be null + * @return this object to facilitate chaining multiple methods; never null + * @throws Exception if the header's value is invalid + */ + Headers addMap(String key, Map value, Meta meta); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + * @throws Exception if the header's value is invalid + */ + Headers addStruct(String key, Struct value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addDecimal(String key, BigDecimal value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addDate(String key, java.util.Date value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addTime(String key, java.util.Date value); + + /** + * Add to this collection a {@link Header} with the given key and value. + * + * @param key the header's key; may not be null + * @param value the header's value; may be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers addTimestamp(String key, java.util.Date value); + + /** + * Removes all {@link Header} objects whose {@link Header#key() key} matches the specified key. + * + * @param key the key; may not be null + * @return this object to facilitate chaining multiple methods; never null + */ + Headers remove(String key); + + /** + * Removes all headers from this object. + * + * @return this object to facilitate chaining multiple methods; never null + */ + Headers clear(); + + /** + * Create a copy of this {@link Headers} object. The new copy will contain all of the same {@link Header} objects as + * this object. + * + * @return the copy; never null + */ + Headers duplicate(); + +} diff --git a/connector/src/main/test/io/openmessaging/connector/api/data/HeadersTest.java b/connector/src/main/test/io/openmessaging/connector/api/data/HeadersTest.java new file mode 100644 index 0000000..3ac4c4a --- /dev/null +++ b/connector/src/main/test/io/openmessaging/connector/api/data/HeadersTest.java @@ -0,0 +1,49 @@ +package io.openmessaging.connector.api.data; + +import java.math.BigDecimal; +import java.util.Map; + +import io.openmessaging.connector.api.header.DataHeader; +import io.openmessaging.connector.api.header.DataHeaders; +import io.openmessaging.connector.api.header.Header; +import io.openmessaging.connector.api.header.Headers; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class HeadersTest { + + + @Test + public void commonTest(){ + Meta strMeta = MetaBuilder.string().name("myStrMeta").build(); + Header header = new DataHeader("header1", strMeta, "headerValue1"); + //System.out.println("construct header: " + header); + assertNotNull(header); + + Headers headers = new DataHeaders(); + headers.add(header) + .add("header1", strMeta, "headerValue2") + .addBoolean("header2", true) + .addDecimal("header3 ", BigDecimal.ONE) + ; + //System.out.println("construct headers: " + headers); + assertNotNull(headers); + + Header findHeader1 = headers.findHeader("header1"); + //System.out.println("except header named header1, real is: " + findHeader1); + assertEquals(findHeader1.key(), "header1"); + findHeader1.rename("header2"); + + Header findHeader2 = headers.findHeader("header2"); + //System.out.println("except header valued TRUE, real is: " + findHeader2); + assertEquals(findHeader2.key(), "header2"); + + Map headerMap = headers.toMap(); + //System.out.println("show header Map: " + headerMap); + assertNotNull(headerMap); + headerMap.put("header3", header); + } + +} diff --git a/connector/src/main/test/io/openmessaging/connector/api/data/MetaAndDataTest.java b/connector/src/main/test/io/openmessaging/connector/api/data/MetaAndDataTest.java new file mode 100644 index 0000000..1de0448 --- /dev/null +++ b/connector/src/main/test/io/openmessaging/connector/api/data/MetaAndDataTest.java @@ -0,0 +1,105 @@ +package io.openmessaging.connector.api.data; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class MetaAndDataTest { + + + @Test + public void structParserTest(){ + Meta structMeta = MetaBuilder.struct() + .name("structMeta") + .field("f1", Meta.STRING_META) + .field("f2", Meta.INT32_META) + .field("f3", MetaBuilder.array(Meta.STRING_META).build()) + .build(); + MetaAndData structMetaAndData = new MetaAndData( + structMeta + ); + + structMetaAndData.putData("f1", "asdf"); + structMetaAndData.putData("f2", 32); + structMetaAndData.putData("f3", new ArrayList(){{ + add("a"); + add("b"); + }}); + + String jsonStr = structMetaAndData.convertToString(); + assertNotNull(jsonStr); + //System.out.println(jsonStr); + + // MAP MetaAndData + MetaAndData newMetaAndData = MetaAndData.getMetaDataFromString(jsonStr); + assertNotNull(newMetaAndData); + //System.out.println(newMetaAndData); + + // MAP TO STRUCT + Struct struct = newMetaAndData.convertToStruct(); + assertNotNull(struct); + //System.out.println(struct); + } + + @Test + public void listParserTest(){ + String str = "[1, 2, 3, \"four\"]"; + MetaAndData result = MetaAndData.getMetaDataFromString(str); + + List list = (List) result.getData(); + //System.out.println(list); + assertEquals(4, list.size()); + assertEquals(1, ((Number) list.get(0)).intValue()); + assertEquals(2, ((Number) list.get(1)).intValue()); + assertEquals(3, ((Number) list.get(2)).intValue()); + assertEquals("four", list.get(3)); + } + + @Test + public void commonTest(){ + MetaAndData metaAndData = new MetaAndData(Decimal.builder(10).build(), BigDecimal.ONE); + assertNotNull(metaAndData); + + BigDecimal de = metaAndData.convertToDecimal(10); + assertEquals(de, BigDecimal.ONE); + assertNotEquals(de, 1); + + BigDecimal de2 = metaAndData.convertToDecimal(1); + assertEquals(de2, BigDecimal.ONE); + + metaAndData.putData(BigDecimal.TEN); + assertEquals(metaAndData.convertToDecimal(10), BigDecimal.TEN); + + assertEquals((BigDecimal)metaAndData.getData(), BigDecimal.TEN); + + assertNull(metaAndData.inferMeta()); + } + + @Test + public void stringTest(){ + MetaAndData metaAndData = new MetaAndData(Meta.STRING_META, "message value 中文信息"); + assertNotNull(metaAndData); + + String msg = metaAndData.convertToString(); + assertEquals(msg, "message value 中文信息"); + + Exception ex = null; + try{ + metaAndData.convertToDouble(); + }catch (Exception e){ + ex = e; + } + assertNotNull(ex); + } + + + + +} diff --git a/connector/src/main/test/io/openmessaging/connector/api/data/MetaTest.java b/connector/src/main/test/io/openmessaging/connector/api/data/MetaTest.java new file mode 100644 index 0000000..9420e63 --- /dev/null +++ b/connector/src/main/test/io/openmessaging/connector/api/data/MetaTest.java @@ -0,0 +1,508 @@ +package io.openmessaging.connector.api.data; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class MetaTest { + + @Test + public void staticMethod(){ + Type type = Meta.getMetaType(String.class); + assertEquals(type, Type.STRING); + type = Meta.getMetaType(Struct.class); + assertEquals(type, Type.STRUCT); + type = Meta.getMetaType(Map.class); + assertEquals(type, Type.MAP); + type = Meta.getMetaType(ArrayList.class); + assertEquals(type, Type.ARRAY); + + + + Meta.validateValue(Meta.BYTES_META, "kfc".getBytes()); + Exception error = null; + try{ + // Throw exception for the value of STRING do not match the meta of BYTES . + Meta.validateValue(Meta.BYTES_META, "kfc"); + }catch (Exception e){ + error = e; + } + assertNotNull(error); + + Meta.validateValue("name", Meta.STRING_META, "fasd"); + + List list = new ArrayList<>(); + list.add("a"); + list.add("b"); + list.add("c"); + Object obj = list; + Meta.validateValue(MetaBuilder.array(Meta.STRING_META).build(), obj); + + error = null; + try{ + // Throw exception for the value of ARRAY do not match the meta of BYTES . + Meta.validateValue(MetaBuilder.array(Meta.BYTES_META).build(), obj); + }catch (Exception e){ + error = e; + } + assertNotNull(error); + } + + @Test + public void int8(){ + System.out.println("================INT8================"); + Meta meta1 = MetaBuilder.int8() + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.int8() + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.int8() + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.int8() + .build(); + System.out.println(meta4); + + Meta meta = new MetaBase(Type.INT8, "abc", 1,"dataSource", null); + System.out.println(meta); + + System.out.println(Meta.INT8_META); + } + + @Test + public void int16(){ + System.out.println("================INT16================"); + Meta meta1 = MetaBuilder.int16() + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.int16() + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.int16() + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.int16() + .build(); + System.out.println(meta4); + + Meta meta = new MetaBase(Type.INT16, "abc", 1, "dataSource", null); + System.out.println(meta); + + System.out.println(Meta.INT16_META); + } + + + @Test + public void int32(){ + System.out.println("================INT32================"); + Meta meta1 = MetaBuilder.int32() + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.int32() + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.int32() + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.int32() + .build(); + System.out.println(meta4); + + Meta meta = new MetaBase(Type.INT32, "abc", 1, "dataSource", null); + System.out.println(meta); + + System.out.println(Meta.INT32_META); + } + + + + @Test + public void int64(){ + System.out.println("================INT64================"); + Meta meta1 = MetaBuilder.int64() + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.int64() + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.int64() + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.int64() + .build(); + System.out.println(meta4); + + Meta meta = new MetaBase(Type.INT64, "abc", 1, "dataSource", null); + System.out.println(meta); + + System.out.println(Meta.INT64_META); + } + + + + + @Test + public void float32(){ + System.out.println("================FLOAT32================"); + Meta meta1 = MetaBuilder.float32() + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.float32() + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.float32() + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.float32() + .build(); + System.out.println(meta4); + + Meta meta = new MetaBase(Type.FLOAT32, "abc", 1, "dataSource", null); + System.out.println(meta); + + System.out.println(Meta.FLOAT32_META); + } + + + @Test + public void float64(){ + System.out.println("================FLOAT64================"); + Meta meta1 = MetaBuilder.float64() + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.float64() + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.float64() + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.float64() + .build(); + System.out.println(meta4); + + Meta meta = new MetaBase(Type.FLOAT64, "abc", 1, "dataSource", null); + System.out.println(meta); + + System.out.println(Meta.FLOAT64_META); + } + + + @Test + public void bool(){ + System.out.println("================BOOLEAN================"); + Meta meta1 = MetaBuilder.bool() + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.bool() + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.bool() + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.bool() + .build(); + System.out.println(meta4); + + Meta meta = new MetaBase(Type.BOOLEAN, "abc", 1, "dataSource", null); + System.out.println(meta); + + System.out.println(Meta.BOOLEAN_META); + } + + + @Test + public void str(){ + System.out.println("================STRING================"); + Meta meta1 = MetaBuilder.string() + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.string() + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.string() + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.string() + .build(); + System.out.println(meta4); + + Meta meta = new MetaBase(Type.STRING, "abc", 1, "dataSource", null); + System.out.println(meta); + + System.out.println(Meta.STRING_META); + } + + + @Test + public void bytes(){ + System.out.println("================BYTES================"); + Meta meta1 = MetaBuilder.bytes() + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.bytes() + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.bytes() + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.bytes() + .build(); + System.out.println(meta4); + + Meta meta = new MetaBase(Type.BYTES, "abc", 1, "dataSource", null); + System.out.println(meta); + + System.out.println(Meta.BYTES_META); + } + + + @Test + public void array(){ + System.out.println("================ARRAY================"); + Meta meta1 = MetaBuilder.array(Meta.STRING_META) + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.array(Meta.STRING_META) + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.array(Meta.STRING_META) + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.array(Meta.STRING_META) + .build(); + System.out.println(meta4); + + Meta meta = new MetaArray(Type.ARRAY, "abc", 1, "dataSource", + null, Meta.STRING_META); + System.out.println(meta); + + try{ + Meta metaError = new MetaMap(Type.MAP, "abc", 1, "dataSource", + null, Meta.STRING_META, Meta.BYTES_META); + }catch (Exception e){ + System.out.println("error for new MataMap but Type is not ARRAY."); + } + } + + + @Test + public void map(){ + System.out.println("================MAP================"); + Meta meta1 = MetaBuilder.map(Meta.STRING_META, Meta.BYTES_META) + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.map(Meta.STRING_META, Meta.BYTES_META) + .name("build2") + .dataSource("datasource1") + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.map(Meta.STRING_META, Meta.BYTES_META) + .dataSource("datasource1") + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.map(Meta.STRING_META, Meta.BYTES_META) + .build(); + System.out.println(meta4); + + Meta meta = new MetaMap(Type.MAP, "abc", 1, "dataSource", + null, Meta.STRING_META, Meta.BYTES_META); + System.out.println(meta); + + try{ + Meta metaError = new MetaMap(Type.ARRAY, "abc", 1, "dataSource", + null, Meta.STRING_META, Meta.BYTES_META); + }catch (Exception e){ + System.out.println("error for new MataMap but Type is not MAP."); + } + + } + + + @Test + public void struct(){ + System.out.println("================STRUCT================"); + Meta meta1 = MetaBuilder.struct() + .name("build1") + .version(100) + .dataSource("datasource1") + .parameter("p1","v1") + .parameter("p2", "v2") + .field("f1", Meta.STRING_META) + .field("f2", Meta.BOOLEAN_META) + .field("f3", Meta.FLOAT32_META) + .build(); + System.out.println(meta1); + + Meta meta2 = MetaBuilder.struct() + .name("build2") + .dataSource("datasource1") + .field("f1", Meta.STRING_META) + .field("f2", Meta.BOOLEAN_META) + .field("f3", Meta.FLOAT32_META) + .build(); + System.out.println(meta2); + + Meta meta3 = MetaBuilder.struct() + .dataSource("datasource1") + .field("f1", Meta.STRING_META) + .field("f2", Meta.BOOLEAN_META) + .field("f3", Meta.FLOAT32_META) + .build(); + System.out.println(meta3); + + Meta meta4 = MetaBuilder.struct() + .field("f1", Meta.STRING_META) + .field("f2", Meta.BOOLEAN_META) + .field("f3", Meta.FLOAT32_META) + .build(); + System.out.println(meta4); + + Meta meta5 = MetaBuilder.struct() + .build(); + System.out.println(meta5); + + int size = 10; + List fields = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + Field field = new Field(i, "f"+i, Meta.STRING_META); + fields.add(field); + } + Meta meta = new MetaStruct(Type.STRUCT, "abc", 1, "dataSource", + null, fields); + System.out.println(meta); + + try{ + Meta metaError = new MetaStruct(Type.MAP, "abc", 1, "dataSource", + null, fields); + }catch (Exception e){ + System.out.println("error for new MetaStruct but Type is not STRUCT."); + } + + } + +} diff --git a/connector/src/main/test/io/openmessaging/connector/api/data/SinkDataEntryTest.java b/connector/src/main/test/io/openmessaging/connector/api/data/SinkDataEntryTest.java new file mode 100644 index 0000000..628e4be --- /dev/null +++ b/connector/src/main/test/io/openmessaging/connector/api/data/SinkDataEntryTest.java @@ -0,0 +1,117 @@ +package io.openmessaging.connector.api.data; + + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class SinkDataEntryTest { + + /** + * simple test for kv data + * SET myset asldfjsaldjglas PX 360000 + */ + @Test + public void testKV() { + String command = "SET"; + String key = "myset"; + String value = "asldfjsaldjglas"; + long px = 360000; + + DataEntryBuilder builder = + DataEntryBuilder.newDataEntryBuilder() + .keyMeta(Meta.STRING_META) + .keyData(key) + .valueMeta(Meta.STRING_META) + .valueData(value) + .header("REDIS_COMMAND", command) + .header("PX", px); + + SinkDataEntry sinkDataEntry = builder.buildSinkDataEntry( + 1500L + ); + + assertNotNull(sinkDataEntry); + assertEquals(sinkDataEntry.getKey().convertToString(), "myset"); + assertEquals(sinkDataEntry.getValue().convertToString(), "asldfjsaldjglas"); + assertEquals(sinkDataEntry.getHeaders().findHeader("REDIS_COMMAND").data().convertToString(), "SET"); + assertTrue(sinkDataEntry.getHeaders().toMap().get("PX").data().convertToLong() == 360000); + } + + /** + * simple test for table data + */ + @Test + public void testTable() { + // construct table meta + Meta tableMeta = MetaBuilder.struct() + .field("id", Meta.INT64_META) + .field("name", Meta.STRING_META) + .field("age", Meta.INT16_META) + .field("score", Meta.INT32_META) + .field("isNB", Meta.BOOLEAN_META) + .build(); + + // construct sourceDataEntry + DataEntryBuilder builder = + DataEntryBuilder.newDataEntryBuilder(tableMeta) + .valueData("id", 1L) + .valueData("name", "小红") + .valueData("age", (short)17) + .valueData("score", 99) + .valueData("isNB", true); + + SinkDataEntry sinkDataEntry = builder.buildSinkDataEntry(1500L); + + //System.out.println(sinkDataEntry); + assertNotNull(sinkDataEntry); + assertTrue(sinkDataEntry.getValue().convertToStruct().getInt64("id") == 1L); + assertEquals(sinkDataEntry.getValue().convertToStruct().getString("name"), "小红"); + assertTrue(sinkDataEntry.getValue().convertToStruct().getInt16("age") == (short)17); + assertTrue(sinkDataEntry.getValue().convertToStruct().getInt32("score") == 99); + assertTrue(sinkDataEntry.getValue().convertToStruct().getBoolean("isNB")); + } + + /** + * test data for type of struct + * + * @return + */ + @Test + public void testStruct() { + // 1. construct meta + + DataEntryBuilder structDataEntry = new DataEntryBuilder( + Meta.STRING_META, + MetaBuilder.struct() + .name("myStruct") + .field("field_string", Meta.STRING_META) + .field("field_int32", Meta.INT32_META) + .parameter("parameter", "sadfsadf") + .build() + ); + + // 2. construct data + + structDataEntry + .queue("last_queue") + .queueId(1) + .timestamp(System.currentTimeMillis()) + .entryType(EntryType.UPDATE) + .keyData("schema_data_lalala") + .valueData("field_string", "nihao") + .valueData("field_int32", 321) + .header("int_header", 1) + ; + + // 3. construct dataEntry + + SinkDataEntry sinkDataEntry = structDataEntry.buildSinkDataEntry(1500L); + + //System.out.println(sinkDataEntry); + assertNotNull(sinkDataEntry); + } + +} diff --git a/connector/src/main/test/io/openmessaging/connector/api/data/SourceDataEntryTest.java b/connector/src/main/test/io/openmessaging/connector/api/data/SourceDataEntryTest.java new file mode 100644 index 0000000..5f3eb7b --- /dev/null +++ b/connector/src/main/test/io/openmessaging/connector/api/data/SourceDataEntryTest.java @@ -0,0 +1,128 @@ +package io.openmessaging.connector.api.data; + + +import java.nio.ByteBuffer; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class SourceDataEntryTest { + + /** + * simple test for kv data + * SET myset asldfjsaldjglas PX 360000 + */ + @Test + public void testKV(){ + String command = "SET"; + String key = "myset"; + String value = "asldfjsaldjglas"; + long px = 360000; + + + DataEntryBuilder builder = + DataEntryBuilder.newDataEntryBuilder() + .keyMeta(Meta.STRING_META) + .keyData(key) + .valueMeta(Meta.STRING_META) + .valueData(value) + .header("REDIS_COMMAND", command) + .header("PX", px); + + SourceDataEntry sourceDataEntry = builder.buildSourceDataEntry( + ByteBuffer.wrap("partition1".getBytes()), + ByteBuffer.wrap("1098".getBytes()) + ); + + assertNotNull(sourceDataEntry); + assertEquals(sourceDataEntry.getKey().convertToString(), "myset"); + assertEquals(sourceDataEntry.getValue().convertToString(), "asldfjsaldjglas"); + assertEquals(sourceDataEntry.getHeaders().findHeader("REDIS_COMMAND").data().convertToString(), "SET"); + assertTrue(sourceDataEntry.getHeaders().toMap().get("PX").data().convertToLong() == 360000); + //System.out.println(sourceDataEntry); + } + + /** + * simple test for table data + */ + @Test + public void testTable(){ + // construct table meta + Meta tableMeta = MetaBuilder.struct() + .field("id", Meta.INT64_META) + .field("name", Meta.STRING_META) + .field("age", Meta.INT16_META) + .field("score", Meta.INT32_META) + .field("isNB", Meta.BOOLEAN_META) + .build(); + + // construct sourceDataEntry + DataEntryBuilder builder = + DataEntryBuilder.newDataEntryBuilder(tableMeta) + .valueData("id", 1L) + .valueData("name", "小红") + .valueData("age", (short)17) + .valueData("score", 99) + .valueData("isNB", true); + + SourceDataEntry sourceDataEntry = builder.buildSourceDataEntry( + ByteBuffer.wrap("partition1".getBytes()), + ByteBuffer.wrap("1098".getBytes()) + ); + + assertNotNull(sourceDataEntry); + assertNotNull(sourceDataEntry); + assertTrue(sourceDataEntry.getValue().convertToStruct().getInt64("id") == 1L); + assertEquals(sourceDataEntry.getValue().convertToStruct().getString("name"), "小红"); + assertTrue(sourceDataEntry.getValue().convertToStruct().getInt16("age") == (short)17); + assertTrue(sourceDataEntry.getValue().convertToStruct().getInt32("score") == 99); + assertTrue(sourceDataEntry.getValue().convertToStruct().getBoolean("isNB")); + //System.out.println(sourceDataEntry); + } + + /** + * test data for type of struct + * + * @return + */ + @Test + public void testStruct(){ + // 1. construct meta + + DataEntryBuilder structDataEntry = new DataEntryBuilder( + Meta.STRING_META, + MetaBuilder.struct() + .name("myStruct") + .field("field_string", Meta.STRING_META) + .field("field_int32", Meta.INT32_META) + .parameter("parameter", "sadfsadf") + .build() + ); + + // 2. construct data + + structDataEntry + .queue("last_queue") + .queueId(1) + .timestamp(System.currentTimeMillis()) + .entryType(EntryType.UPDATE) + .keyData("schema_data_lalala") + .valueData("field_string", "nihao") + .valueData("field_int32", 321) + .header("int_header", 1) + ; + + // 3. construct dataEntry + + SourceDataEntry sourceDataEntry = structDataEntry.buildSourceDataEntry(null, null); + + assertNotNull(sourceDataEntry); + //System.out.println(sourceDataEntry); + } + + + +} diff --git a/pom.xml b/pom.xml index 9559e01..b6bd1b8 100644 --- a/pom.xml +++ b/pom.xml @@ -138,6 +138,12 @@ openmessaging-api 0.3.1-alpha + + junit + junit + 4.12 + test + \ No newline at end of file