From 8ecf3a2e1636ad806a07f6dee8b43fd0ad9f658e Mon Sep 17 00:00:00 2001 From: rusirijayodaillesinghe Date: Fri, 23 Jun 2023 09:29:43 +0530 Subject: [PATCH 1/4] Implement the PoC for CDC inbound endpoint. Modify the payload removing unwanted information Add Securevault support for password encryption --- .../pom.xml | 20 ++ .../InboundRequestProcessorFactoryImpl.java | 5 +- .../protocol/cdc/CDCEndpointManager.java | 261 ++++++++++++++++++ .../protocol/cdc/CDCEventExecutorManager.java | 50 ++++ .../endpoint/protocol/cdc/CDCEventOutput.java | 142 ++++++++++ .../endpoint/protocol/cdc/CDCListener.java | 50 ++++ .../protocol/cdc/CDCSourceHandler.java | 165 +++++++++++ .../protocol/cdc/InboundCDCConstants.java | 51 ++++ .../protocol/cdc/InboundCDCEventExecutor.java | 49 ++++ .../src/main/resources/log4j.properties | 8 + .../CDCInbound/cdc-inbound-endpoint.xml | 19 ++ .../resources/CDCInbound/cdc_process_seq.xml | 16 ++ pom.xml | 107 ++++++- 13 files changed, 940 insertions(+), 3 deletions(-) create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml index c8f9d0d93d..bb7dd45d96 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml @@ -180,6 +180,26 @@ log4j-api test + + org.wso2.carbon.mediation + org.wso2.carbon.inbound.endpoint.persistence + + + io.debezium + debezium-embedded + + + io.debezium + debezium-api + + + io.debezium + debezium-connector-mysql + + + org.slf4j + slf4j-api + diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java index 389117f872..c8b4fb23db 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java @@ -21,6 +21,7 @@ import org.apache.synapse.inbound.InboundProcessorParams; import org.apache.synapse.inbound.InboundRequestProcessor; import org.apache.synapse.inbound.InboundRequestProcessorFactory; +import org.wso2.carbon.inbound.endpoint.protocol.cdc.CDCListener; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericEventBasedConsumer; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericEventBasedListener; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericInboundListener; @@ -44,7 +45,7 @@ */ public class InboundRequestProcessorFactoryImpl implements InboundRequestProcessorFactory { - public static enum Protocols {jms, file, http, https, hl7, kafka, mqtt, rabbitmq, ws, wss, grpc, httpws, httpswss} + public static enum Protocols {jms, file, http, https, hl7, kafka, mqtt, rabbitmq, ws, wss, grpc, httpws, httpswss, cdc} /** * return underlying Request Processor Implementation according to protocol @@ -97,6 +98,8 @@ public InboundRequestProcessor createInboundProcessor(InboundProcessorParams par case grpc: inboundRequestProcessor = new InboundGRPCListener(params); break; + case cdc: + inboundRequestProcessor = new CDCListener(params); } } else if (params.getClassImpl() != null) { if (GenericInboundListener.isListeningInboundEndpoint(params)) { diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java new file mode 100644 index 0000000000..a2320804ad --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java @@ -0,0 +1,261 @@ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import io.debezium.engine.DebeziumEngine; +import io.debezium.engine.format.Json; +import org.apache.synapse.mediators.Value; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.MessageContext; +import org.apache.synapse.SynapseException; +import org.apache.synapse.core.SynapseEnvironment; +import org.apache.synapse.inbound.InboundProcessorParams; +import org.apache.synapse.util.xpath.SynapseXPath; +import org.jaxen.JaxenException; +import org.wso2.carbon.inbound.endpoint.common.AbstractInboundEndpointManager; + +import java.io.File; +import java.io.IOException; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.wso2.carbon.inbound.endpoint.common.Constants.SUPER_TENANT_DOMAIN_NAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_CONNECTOR_CLASS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_DBNAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_HOSTNAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PASSWORD; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PORT; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_SERVER_ID; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_USER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OPERATIONS_EXCLUDE_LIST; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_MAX_THREADS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_NAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SNAPSHOT_MODE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TABLES_INCLUDE_LIST; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TOPIC_PREFIX; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TRUE; + +public class CDCEndpointManager extends AbstractInboundEndpointManager { + + private static final Log log = LogFactory.getLog(CDCEndpointManager.class); + + private static CDCEndpointManager instance = null; + private InboundCDCEventExecutor eventExecutor; + + private static final String SECURE_VAULT_REGEX = "(wso2:vault-lookup\\('(.*?)'\\))"; + private static Pattern vaultLookupPattern = Pattern.compile(SECURE_VAULT_REGEX); + + private CDCEndpointManager() { + super(); + } + + public static CDCEndpointManager getInstance() { + if (instance == null) { + instance = new CDCEndpointManager(); + } + return instance; + } + + @Override + public boolean startListener(int port, String name, InboundProcessorParams inboundParameters) { + + if (CDCEventExecutorManager.getInstance().isRegisteredExecutor(port)) { + log.info("CDC Listener already started on port " + port); + return true; + } + + log.info("Starting CDC Listener on port " + port); + + eventExecutor = new InboundCDCEventExecutor(); + CDCEventExecutorManager.getInstance().registerEventExecutor(port, eventExecutor); + Properties props = setProperties(inboundParameters); + + CDCSourceHandler sourceHandler = new CDCSourceHandler(port, inboundParameters); + DebeziumEngine> engine = DebeziumEngine.create(Json.class) + .using(props) + .notifying(record -> { + sourceHandler.requestReceived(record); + System.out.println(record); + }).build(); + + eventExecutor.getExecutorService().execute(engine); + log.info("Debezium engine started"); + + return true; + } + + private Properties setProperties (InboundProcessorParams params) { + Properties inboundProperties = params.getProperties(); + log.info("Initializing the properties"); + final Properties props = new Properties(); + try { + props.setProperty(DEBEZIUM_NAME, inboundProperties.getProperty(DEBEZIUM_NAME)); + if (inboundProperties.getProperty(DEBEZIUM_SNAPSHOT_MODE) != null) { + props.setProperty(DEBEZIUM_SNAPSHOT_MODE, inboundProperties.getProperty(DEBEZIUM_SNAPSHOT_MODE)); + } + + if (inboundProperties.getProperty(DEBEZIUM_MAX_THREADS) != null) { + props.setProperty(DEBEZIUM_MAX_THREADS, inboundProperties.getProperty(DEBEZIUM_MAX_THREADS)); + } + + if (inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE) != null) { + props.setProperty(DEBEZIUM_OFFSET_STORAGE, inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE)); + } else { + props.setProperty(DEBEZIUM_OFFSET_STORAGE, "org.apache.kafka.connect.storage.FileOffsetBackingStore"); + } + + if (inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME) != null) { + props.setProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME, inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME)); + } else { + String filePath = "cdc/offsetStorage/" + params.getName() + "_.dat"; + createFile(filePath); + props.setProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME, filePath); + } + + if (inboundProperties.getProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS) != null) { + props.setProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS, inboundProperties.getProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS)); + } else { + props.setProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS, "1000"); + } + + /* begin connector properties */ + props.setProperty(DEBEZIUM_CONNECTOR_CLASS, inboundProperties.getProperty(DEBEZIUM_CONNECTOR_CLASS)); + props.setProperty(DEBEZIUM_DATABASE_HOSTNAME, inboundProperties.getProperty(DEBEZIUM_DATABASE_HOSTNAME)); + props.setProperty(DEBEZIUM_DATABASE_PORT, inboundProperties.getProperty(DEBEZIUM_DATABASE_PORT)); + props.setProperty(DEBEZIUM_DATABASE_USER, inboundProperties.getProperty(DEBEZIUM_DATABASE_USER)); + + String passwordString = inboundProperties.getProperty(DEBEZIUM_DATABASE_PASSWORD); + SynapseEnvironment synapseEnvironment = params.getSynapseEnvironment(); + MessageContext messageContext = synapseEnvironment.createMessageContext(); + + props.setProperty(DEBEZIUM_DATABASE_PASSWORD, resolveSecureVault(messageContext, passwordString)); + + props.setProperty(DEBEZIUM_DATABASE_DBNAME, inboundProperties.getProperty(DEBEZIUM_DATABASE_DBNAME)); + props.setProperty(DEBEZIUM_DATABASE_SERVER_ID, inboundProperties.getProperty(DEBEZIUM_DATABASE_SERVER_ID)); + + if (inboundProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL) != null) { + props.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, inboundProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL)); + } else { + props.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, TRUE); + } + + if (inboundProperties.getProperty(DEBEZIUM_TOPIC_PREFIX) != null) { + props.setProperty(DEBEZIUM_TOPIC_PREFIX, inboundProperties.getProperty(DEBEZIUM_TOPIC_PREFIX)); + } else { + props.setProperty(DEBEZIUM_TOPIC_PREFIX, params.getName() +"_topic"); + } + + props.setProperty(DEBEZIUM_VALUE_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + props.setProperty(DEBEZIUM_KEY_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + props.setProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE, TRUE); + props.setProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE, TRUE); + + if (inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL) != null) { + props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL)); + } else { + props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, "io.debezium.storage.file.history.FileSchemaHistory"); + } + + if (inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME) != null) { + props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME, inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME)); + } else { + String filePath = "cdc/schemaHistory/" + params.getName() + "_.dat"; + createFile(filePath); + props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME, filePath); + } + + if (inboundProperties.getProperty(DEBEZIUM_TABLES_INCLUDE_LIST) != null) { + props.setProperty(DEBEZIUM_TABLES_INCLUDE_LIST, inboundProperties.getProperty(DEBEZIUM_TABLES_INCLUDE_LIST)); + } + + if (inboundProperties.getProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST) != null) { + props.setProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST, inboundProperties.getProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST)); + } + } catch (IOException e) { + throw new RuntimeException(e); + } catch (NullPointerException e) { + log.error("A required property value is not defined", e); + throw new RuntimeException(e); + } + return props; + } + + private static synchronized String resolveSecureVault(MessageContext messageContext, String passwordString) { + if (passwordString == null) { + return null; + } + Matcher lookupMatcher = vaultLookupPattern.matcher(passwordString); + String resolvedValue = ""; + if (lookupMatcher.find()) { + Value expression; + String expressionStr = lookupMatcher.group(1); + try { + expression = new Value(new SynapseXPath(expressionStr)); + + } catch (JaxenException e) { + throw new SynapseException("Error while building the expression : " + expressionStr, e); + } + resolvedValue = expression.evaluateValue(messageContext); + if (StringUtils.isEmpty(resolvedValue)) { + log.warn("Found Empty value for expression : " + expression.getExpression()); + resolvedValue = ""; + } + } else { + resolvedValue = passwordString; + } + return resolvedValue; + } + + @Override + public boolean startEndpoint(int port, String name, InboundProcessorParams inboundParameters) { + log.info("Starting CDC Endpoint on port " + port); + dataStore.registerListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME, InboundCDCConstants.CDC, name, + inboundParameters); + + boolean start = startListener(port, name, inboundParameters); + + if (start) { + //do nothing + } else { + dataStore.unregisterListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME); + return false; + } + return true; + } + + private void createFile (String filePath) throws IOException { + File file = new File(filePath); + file.getParentFile().mkdirs(); + if(!file.exists()) { + file.createNewFile(); + } + } + + @Override + public void closeEndpoint(int port) { + log.info("Closing CDC Endpoint on port " + port); + eventExecutor.getExecutorService().shutdown(); + dataStore.unregisterListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME); + + if (!CDCEventExecutorManager.getInstance().isRegisteredExecutor(port)) { + log.info("Listener Endpoint is not started"); + return; + } else if (dataStore.isEndpointRegistryEmpty(port)) { + CDCEventExecutorManager.getInstance().shutdownExecutor(port); + } + + } + +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java new file mode 100644 index 0000000000..80040b285f --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + + +import java.util.concurrent.ConcurrentHashMap; + +public class CDCEventExecutorManager { + + private ConcurrentHashMap executorPoolMap = new ConcurrentHashMap(); + + private static CDCEventExecutorManager instance = null; + + public static CDCEventExecutorManager getInstance() { + if (instance == null) { + instance = new CDCEventExecutorManager(); + } + return instance; + } + + public void shutdownExecutor(int port) { + executorPoolMap.get(port).shutdownEventExecutor(); + executorPoolMap.remove(port); + } + + public void registerEventExecutor(int port, InboundCDCEventExecutor eventExecutor) { + executorPoolMap.put(port, eventExecutor); + } + + public boolean isRegisteredExecutor(int port) { + return executorPoolMap.containsKey(port); + } + +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java new file mode 100644 index 0000000000..df9439c101 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java @@ -0,0 +1,142 @@ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import org.json.JSONObject; + +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.AFTER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.BEFORE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DB; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.OP; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.PAYLOAD; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.SOURCE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TS_MS; + + +public class CDCEventOutput { + + private Object beforeEvent; + private Object afterEvent; + private Long ts_ms; + + private String database; + private Object table; + private String op; + private JSONObject payload; + + private enum operations {c, r, u, d}; + + CDCEventOutput (ChangeEvent event) { + String valueString = event.value().toString(); + JSONObject value = new JSONObject(valueString); + this.payload = value.getJSONObject(PAYLOAD); + } + + public Object getJsonPayloadBeforeEvent() { + Object beforeObject = null; + if (payload.has(BEFORE)) { + beforeObject = payload.get(BEFORE); + } + return beforeObject; + + } + + public void setJsonPayloadBeforeEvent(Object beforeEvent) { + this.beforeEvent = beforeEvent; + } + + public Object getJsonPayloadAfterEvent() { + Object afterObject = null; + if (payload.has(AFTER)) { + afterObject = payload.get(AFTER); + } + return afterObject; + } + + public void setJsonPayloadAfterEvent(Object afterEvent) { + this.afterEvent = afterEvent; + } + + public Long getTs_ms() { + if (payload.has(TS_MS)) { + return payload.getLong(TS_MS); + } + return null; + } + + public void setTs_ms(Long ts_ms) { + this.ts_ms = ts_ms; + } + + public String getDatabase() { + if (getSource() != null) { + if (getSource().has(DB)) { + return getSource().getString(DB); + } + return null; + } + return null; + } + + public void setDatabase(String database) { + this.database = database; + } + + public Object getTable() { + Object tableObject = null; + if (getSource() != null) { + if (getSource().has(TABLE)) { + tableObject = getSource().get(TABLE); + } + } + return tableObject; + } + + public void setTable(Object table) { + this.table = table; + } + + private JSONObject getSource () { + if (payload.has(SOURCE)) { + return payload.getJSONObject(SOURCE); + } + return null; + } + + public String getOp() { + if (payload.has(OP)) { + return getOpString(payload.getString(OP)); + } + return null; + } + private String getOpString(String op) { + if (op != null) { + switch (operations.valueOf(op)) { + case c: + return "CREATE"; + case r: + return "READ"; + case u: + return "UPDATE"; + case d: + return "DELETE"; + } + } + return null; + } + + public void setOp(String op) { + this.op = op; + } + + public JSONObject getOutputJsonPayload () { + if (payload == null) { + return null; + } + JSONObject jsonPayload = new JSONObject(); + jsonPayload.put(OP, getOp()); + jsonPayload.put(BEFORE, getJsonPayloadBeforeEvent()); + jsonPayload.put(AFTER, getJsonPayloadAfterEvent()); + return jsonPayload; + } +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java new file mode 100644 index 0000000000..d249a37677 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java @@ -0,0 +1,50 @@ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.SynapseException; +import org.apache.synapse.inbound.InboundProcessorParams; +import org.apache.synapse.inbound.InboundRequestProcessor; +import org.wso2.carbon.inbound.endpoint.persistence.PersistenceUtils; + +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.INBOUND_ENDPOINT_PARAMETER_CDC_PORT; + +public class CDCListener implements InboundRequestProcessor { + + private InboundProcessorParams processorParams; + private int port; + private String name; + private static final Log LOGGER = LogFactory.getLog(CDCListener.class); + + public CDCListener(InboundProcessorParams params) { + + processorParams = params; + String portParam = params.getProperties() + .getProperty(INBOUND_ENDPOINT_PARAMETER_CDC_PORT); + try { + port = Integer.parseInt(portParam); + } catch (NumberFormatException e) { + handleException("Validation failed for the port parameter " + portParam, e); + } + name = params.getName(); + } + + protected void handleException(String msg, Exception e) { + LOGGER.error(msg, e); + throw new SynapseException(msg, e); + } + + @Override + public void init() { + System.out.println("Init called"); + int offsetPort = port + PersistenceUtils.getPortOffset(processorParams.getProperties()); + CDCEndpointManager.getInstance().startEndpoint(offsetPort, name, processorParams); + } + + @Override + public void destroy() { + System.out.println("Destroy called"); + int offsetPort = port + PersistenceUtils.getPortOffset(processorParams.getProperties()); + CDCEndpointManager.getInstance().closeEndpoint(offsetPort); + } +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java new file mode 100644 index 0000000000..7df87f60ff --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java @@ -0,0 +1,165 @@ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import org.apache.axiom.om.OMAbstractFactory; +import org.apache.axiom.om.OMElement; +import org.apache.axiom.soap.SOAPEnvelope; +import org.apache.axiom.soap.SOAPFactory; +import org.apache.axiom.util.UIDGenerator; +import org.apache.axis2.AxisFault; +import org.apache.axis2.builder.Builder; +import org.apache.axis2.builder.BuilderUtil; +import org.apache.axis2.context.OperationContext; +import org.apache.axis2.context.ServiceContext; +import org.apache.axis2.description.InOutAxisOperation; +import org.apache.axis2.transport.TransportUtils; +import org.apache.commons.io.input.AutoCloseInputStream; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.MessageContext; +import org.apache.synapse.SynapseConstants; +import org.apache.synapse.api.ApiConstants; +import org.apache.synapse.core.axis2.Axis2MessageContext; +import org.apache.synapse.core.axis2.MessageContextCreatorForAxis2; +import org.apache.synapse.inbound.InboundProcessorParams; +import org.apache.synapse.mediators.MediatorFaultHandler; +import org.apache.synapse.mediators.base.SequenceMediator; +import org.wso2.carbon.inbound.endpoint.osgi.service.ServiceReferenceHolder; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.IOException; + + +import static org.wso2.carbon.inbound.endpoint.common.Constants.SUPER_TENANT_DOMAIN_NAME; +import static org.wso2.carbon.inbound.endpoint.common.Constants.TENANT_DOMAIN; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.*; + + +public class CDCSourceHandler { + + private int port; + private InboundProcessorParams params; + private static final String tenantDomain = SUPER_TENANT_DOMAIN_NAME; + private static final Log log = LogFactory.getLog(CDCSourceHandler.class); + private static final String contentType = "application/json"; + + public CDCSourceHandler(int port, InboundProcessorParams params) { + this.port = port; + this.params = params; + } + + public void requestReceived(ChangeEvent eventRecord) { + if (eventRecord == null || eventRecord.value() == null) { + log.debug("CDC Source Handler received empty event record"); + } else { + log.debug("CDC Source Handler request received"); + + MessageContext synCtx = null; + try { + + synCtx = getSynapseMessageContext(tenantDomain); + + CDCEventOutput cdcEventOutput = new CDCEventOutput(eventRecord); + synCtx.setProperty(DATABASE_NAME, cdcEventOutput.getDatabase()); + synCtx.setProperty(TABLES, cdcEventOutput.getTable().toString()); + synCtx.setProperty(OPERATIONS, cdcEventOutput.getOp()); + synCtx.setProperty(TS_MS, cdcEventOutput.getTs_ms().toString()); + + org.apache.axis2.context.MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) synCtx) + .getAxis2MessageContext(); + Builder builder = BuilderUtil.getBuilderFromSelector(contentType, axis2MsgCtx); + + if (builder != null) { + String serializedChangeEvent = cdcEventOutput.getOutputJsonPayload().toString(); + InputStream in = new AutoCloseInputStream( + new ByteArrayInputStream(serializedChangeEvent.getBytes())); + + OMElement documentElement = builder.processDocument(in, contentType, axis2MsgCtx); + synCtx.setEnvelope(TransportUtils.createSOAPEnvelope(documentElement)); + + if (log.isDebugEnabled()) { + log.debug("CDCEvent being injected to Sequence"); + } + injectForMediation(synCtx); + return; + } + + } catch (AxisFault e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + private static org.apache.axis2.context.MessageContext createAxis2MessageContext() { + org.apache.axis2.context.MessageContext axis2MsgCtx = new org.apache.axis2.context.MessageContext(); + axis2MsgCtx.setMessageID(UIDGenerator.generateURNString()); + axis2MsgCtx.setConfigurationContext( + ServiceReferenceHolder.getInstance().getConfigurationContextService().getServerConfigContext()); + axis2MsgCtx.setProperty(org.apache.axis2.context.MessageContext.CLIENT_API_NON_BLOCKING, Boolean.TRUE); + axis2MsgCtx.setServerSide(true); + + return axis2MsgCtx; + } + + private static org.apache.synapse.MessageContext createSynapseMessageContext(String tenantDomain) throws AxisFault { + org.apache.axis2.context.MessageContext axis2MsgCtx = createAxis2MessageContext(); + ServiceContext svcCtx = new ServiceContext(); + OperationContext opCtx = new OperationContext(new InOutAxisOperation(), svcCtx); + axis2MsgCtx.setServiceContext(svcCtx); + axis2MsgCtx.setOperationContext(opCtx); + + axis2MsgCtx.setProperty(TENANT_DOMAIN, tenantDomain); + + SOAPFactory fac = OMAbstractFactory.getSOAP11Factory(); + SOAPEnvelope envelope = fac.getDefaultEnvelope(); + axis2MsgCtx.setEnvelope(envelope); + return MessageContextCreatorForAxis2.getSynapseMessageContext(axis2MsgCtx); + } + + public org.apache.synapse.MessageContext getSynapseMessageContext(String tenantDomain) throws AxisFault { + MessageContext synCtx = createSynapseMessageContext(tenantDomain); + synCtx.setProperty(SynapseConstants.IS_INBOUND, true); + ((Axis2MessageContext) synCtx).getAxis2MessageContext().setProperty(SynapseConstants.IS_INBOUND, true); + + return synCtx; + } + + + private void injectForMediation(org.apache.synapse.MessageContext synCtx) { + SequenceMediator faultSequence = getFaultSequence(synCtx); + + MediatorFaultHandler mediatorFaultHandler = new MediatorFaultHandler(faultSequence); + synCtx.pushFaultHandler(mediatorFaultHandler); + if (log.isDebugEnabled()) { + log.debug("injecting message to sequence : " + params.getInjectingSeq()); + } + synCtx.setProperty("inbound.endpoint.name", params.getName()); + synCtx.setProperty(ApiConstants.API_CALLER, params.getName()); + + SequenceMediator injectingSequence = null; + if (params.getInjectingSeq() != null) { + injectingSequence = (SequenceMediator) synCtx.getSequence(params.getInjectingSeq()); + } + if (injectingSequence == null) { + injectingSequence = (SequenceMediator) synCtx.getMainSequence(); + } + + synCtx.getEnvironment().injectMessage(synCtx, injectingSequence); + + } + + private SequenceMediator getFaultSequence(org.apache.synapse.MessageContext synCtx) { + SequenceMediator faultSequence = null; + if (params.getOnErrorSeq() != null) { + faultSequence = (SequenceMediator) synCtx.getSequence(params.getOnErrorSeq()); + } + if (faultSequence == null) { + faultSequence = (SequenceMediator) synCtx.getFaultSequence(); + } + return faultSequence; + } + +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java new file mode 100644 index 0000000000..156a414e74 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java @@ -0,0 +1,51 @@ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +public class InboundCDCConstants { + + /** Inbound Endpoint Parameters **/ + public static final String CDC = "cdc"; + public static final String INBOUND_ENDPOINT_PARAMETER_CDC_PORT = "inbound.cdc.port"; + + //debezium + public static final String DEBEZIUM_NAME = "name"; + public static final String DEBEZIUM_SNAPSHOT_MODE = "snapshot.mode"; + public static final String DEBEZIUM_MAX_THREADS = "snapshot.max.threads"; + public static final String DEBEZIUM_OFFSET_STORAGE = "offset.storage"; + public static final String DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME = "offset.storage.file.filename"; + public static final String DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS = "offset.flush.interval.ms"; + public static final String DEBEZIUM_CONNECTOR_CLASS = "connector.class"; + public static final String DEBEZIUM_DATABASE_HOSTNAME = "database.hostname"; + public static final String DEBEZIUM_DATABASE_PORT = "database.port"; + public static final String DEBEZIUM_DATABASE_USER = "database.user"; + public static final String DEBEZIUM_DATABASE_PASSWORD = "database.password"; + public static final String DEBEZIUM_DATABASE_DBNAME = "database.dbname"; + public static final String DEBEZIUM_TABLES_INCLUDE_LIST = "table.include.list"; + public static final String DEBEZIUM_OPERATIONS_EXCLUDE_LIST = "skipped.operations"; + public static final String DEBEZIUM_DATABASE_SERVER_ID = "database.server.id"; + public static final String DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL = "database.allowPublicKeyRetrieval"; + public static final String DEBEZIUM_TOPIC_PREFIX = "topic.prefix"; + + public static final String DEBEZIUM_VALUE_CONVERTER = "value.converter"; + public static final String DEBEZIUM_KEY_CONVERTER = "key.converter"; + public static final String DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE = "value.converter.schemas.enable"; + public static final String DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE = "key.converter.schemas.enable"; + + public static final String DEBEZIUM_SCHEMA_HISTORY_INTERNAL = "schema.history.internal"; + public static final String DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME = "schema.history.internal.file.filename"; + + /** Output Properties **/ + public static final String DATABASE_NAME = "database"; + public static final String TABLES ="tables"; + public static final String OPERATIONS ="operations"; + public static final String TS_MS = "ts_ms"; + + public static final String BEFORE = "before"; + public static final String AFTER = "after"; + public static final String SOURCE = "source"; + public static final String OP = "op"; + public static final String PAYLOAD = "payload"; + public static final String DB = "db"; + public static final String TABLE = "table"; + + public static final String TRUE = "true"; +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java new file mode 100644 index 0000000000..37e3fbd70d --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 Inc. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +public class InboundCDCEventExecutor { + + private ExecutorService executorService; + + public InboundCDCEventExecutor () { + executorService = Executors.newSingleThreadExecutor(); + } + + public ExecutorService getExecutorService() { + return this.executorService; + } + + public void shutdownEventExecutor() { + executorService.shutdown(); + try { + if (!executorService.awaitTermination(800, TimeUnit.MILLISECONDS)) { + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + } + } + + +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties new file mode 100644 index 0000000000..1da8230168 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties @@ -0,0 +1,8 @@ +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml new file mode 100644 index 0000000000..e3caf7bc88 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml @@ -0,0 +1,19 @@ + + + 9091 + engine + org.apache.kafka.connect.storage.FileOffsetBackingStore + + io.debezium.connector.mysql.MySqlConnector + localhost + 3306 + root + rusiri@wso2 + students + 85744 + io.debezium.storage.file.history.FileSchemaHistory + + students.marks + c + + \ No newline at end of file diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml new file mode 100644 index 0000000000..16bab87ac4 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 6d56f97fcb..d184d4e4cf 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.wso2.ei wso2-micro-integrator-parent pom - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT WSO2 Micro Integrator http://wso2.com/products/enterprise-integrator/ WSO2 Micro Integrator @@ -1450,6 +1450,103 @@ opencensus ${opencensus.orbit.version} + + io.debezium + debezium-embedded + ${debezium.version} + + + io.debezium + debezium-api + ${debezium.version} + + + io.debezium + debezium-connector-mysql + ${debezium.version} + + + io.debezium + debezium-core + ${debezium.version} + + + org.slf4j + slf4j-api + + + + + io.debezium + debezium-storage-kafka + ${debezium.version} + + + org.slf4j + slf4j-api + + + + + io.debezium + debezium-storage-file + ${debezium.version} + + + org.slf4j + slf4j-api + + + + + io.debezium + debezium-ddl-parser + ${debezium.version} + + + org.slf4j + slf4j-api + + + + + com.github.shyiko + mysql-binlog-connector-java + 0.27.2 + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + org.slf4j + slf4j-api + + + + + org.apache.kafka + connect-runtime + ${kafka.version} + + + org.slf4j + slf4j-api + + + + + org.apache.kafka + connect-json + ${kafka.version} + + + org.slf4j + slf4j-api + + + @@ -1545,7 +1642,13 @@ 2.3.0 2.3.1 - 4.0.0-wso2v47 + + 2.4.1.Final + + + 3.3.1 + + 4.0.0-wso2v38 [4.0.0, 4.0.1) 4.7.175 1.1.3 From 8b6fb96dc31d0e951ef5442f39bf25a429f27ac0 Mon Sep 17 00:00:00 2001 From: rusirijayodaillesinghe Date: Mon, 10 Jul 2023 15:58:32 +0530 Subject: [PATCH 2/4] Extend poc with Polling mode and clustering support Add dependencies for MSSQL, Oracle and Postgres --- .../InboundRequestProcessorFactoryImpl.java | 4 +- .../protocol/cdc/CDCEndpointManager.java | 261 ----------------- .../protocol/cdc/CDCEventExecutorManager.java | 50 ---- .../endpoint/protocol/cdc/CDCEventOutput.java | 53 ++-- .../protocol/cdc/CDCInjectHandler.java | 184 ++++++++++++ .../endpoint/protocol/cdc/CDCListener.java | 50 ---- .../protocol/cdc/CDCPollingConsumer.java | 118 ++++++++ .../endpoint/protocol/cdc/CDCProcessor.java | 265 ++++++++++++++++++ .../protocol/cdc/CDCSourceHandler.java | 165 ----------- .../endpoint/protocol/cdc/CDCTask.java | 67 +++++ .../protocol/cdc/InboundCDCConstants.java | 35 ++- .../protocol/cdc/InboundCDCEventExecutor.java | 49 ---- .../src/main/resources/log4j.properties | 8 - pom.xml | 24 +- 14 files changed, 699 insertions(+), 634 deletions(-) delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java create mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java delete mode 100644 components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java index c8b4fb23db..77c4d16811 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/inboundfactory/InboundRequestProcessorFactoryImpl.java @@ -21,7 +21,7 @@ import org.apache.synapse.inbound.InboundProcessorParams; import org.apache.synapse.inbound.InboundRequestProcessor; import org.apache.synapse.inbound.InboundRequestProcessorFactory; -import org.wso2.carbon.inbound.endpoint.protocol.cdc.CDCListener; +import org.wso2.carbon.inbound.endpoint.protocol.cdc.CDCProcessor; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericEventBasedConsumer; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericEventBasedListener; import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericInboundListener; @@ -99,7 +99,7 @@ public InboundRequestProcessor createInboundProcessor(InboundProcessorParams par inboundRequestProcessor = new InboundGRPCListener(params); break; case cdc: - inboundRequestProcessor = new CDCListener(params); + inboundRequestProcessor = new CDCProcessor(params); } } else if (params.getClassImpl() != null) { if (GenericInboundListener.isListeningInboundEndpoint(params)) { diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java deleted file mode 100644 index a2320804ad..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEndpointManager.java +++ /dev/null @@ -1,261 +0,0 @@ -package org.wso2.carbon.inbound.endpoint.protocol.cdc; - -import io.debezium.engine.ChangeEvent; -import io.debezium.engine.DebeziumEngine; -import io.debezium.engine.format.Json; -import org.apache.synapse.mediators.Value; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.synapse.MessageContext; -import org.apache.synapse.SynapseException; -import org.apache.synapse.core.SynapseEnvironment; -import org.apache.synapse.inbound.InboundProcessorParams; -import org.apache.synapse.util.xpath.SynapseXPath; -import org.jaxen.JaxenException; -import org.wso2.carbon.inbound.endpoint.common.AbstractInboundEndpointManager; - -import java.io.File; -import java.io.IOException; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.wso2.carbon.inbound.endpoint.common.Constants.SUPER_TENANT_DOMAIN_NAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_CONNECTOR_CLASS; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_DBNAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_HOSTNAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PASSWORD; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PORT; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_SERVER_ID; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_USER; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OPERATIONS_EXCLUDE_LIST; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_MAX_THREADS; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_NAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SNAPSHOT_MODE; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TABLES_INCLUDE_LIST; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TOPIC_PREFIX; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TRUE; - -public class CDCEndpointManager extends AbstractInboundEndpointManager { - - private static final Log log = LogFactory.getLog(CDCEndpointManager.class); - - private static CDCEndpointManager instance = null; - private InboundCDCEventExecutor eventExecutor; - - private static final String SECURE_VAULT_REGEX = "(wso2:vault-lookup\\('(.*?)'\\))"; - private static Pattern vaultLookupPattern = Pattern.compile(SECURE_VAULT_REGEX); - - private CDCEndpointManager() { - super(); - } - - public static CDCEndpointManager getInstance() { - if (instance == null) { - instance = new CDCEndpointManager(); - } - return instance; - } - - @Override - public boolean startListener(int port, String name, InboundProcessorParams inboundParameters) { - - if (CDCEventExecutorManager.getInstance().isRegisteredExecutor(port)) { - log.info("CDC Listener already started on port " + port); - return true; - } - - log.info("Starting CDC Listener on port " + port); - - eventExecutor = new InboundCDCEventExecutor(); - CDCEventExecutorManager.getInstance().registerEventExecutor(port, eventExecutor); - Properties props = setProperties(inboundParameters); - - CDCSourceHandler sourceHandler = new CDCSourceHandler(port, inboundParameters); - DebeziumEngine> engine = DebeziumEngine.create(Json.class) - .using(props) - .notifying(record -> { - sourceHandler.requestReceived(record); - System.out.println(record); - }).build(); - - eventExecutor.getExecutorService().execute(engine); - log.info("Debezium engine started"); - - return true; - } - - private Properties setProperties (InboundProcessorParams params) { - Properties inboundProperties = params.getProperties(); - log.info("Initializing the properties"); - final Properties props = new Properties(); - try { - props.setProperty(DEBEZIUM_NAME, inboundProperties.getProperty(DEBEZIUM_NAME)); - if (inboundProperties.getProperty(DEBEZIUM_SNAPSHOT_MODE) != null) { - props.setProperty(DEBEZIUM_SNAPSHOT_MODE, inboundProperties.getProperty(DEBEZIUM_SNAPSHOT_MODE)); - } - - if (inboundProperties.getProperty(DEBEZIUM_MAX_THREADS) != null) { - props.setProperty(DEBEZIUM_MAX_THREADS, inboundProperties.getProperty(DEBEZIUM_MAX_THREADS)); - } - - if (inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE) != null) { - props.setProperty(DEBEZIUM_OFFSET_STORAGE, inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE)); - } else { - props.setProperty(DEBEZIUM_OFFSET_STORAGE, "org.apache.kafka.connect.storage.FileOffsetBackingStore"); - } - - if (inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME) != null) { - props.setProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME, inboundProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME)); - } else { - String filePath = "cdc/offsetStorage/" + params.getName() + "_.dat"; - createFile(filePath); - props.setProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME, filePath); - } - - if (inboundProperties.getProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS) != null) { - props.setProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS, inboundProperties.getProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS)); - } else { - props.setProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS, "1000"); - } - - /* begin connector properties */ - props.setProperty(DEBEZIUM_CONNECTOR_CLASS, inboundProperties.getProperty(DEBEZIUM_CONNECTOR_CLASS)); - props.setProperty(DEBEZIUM_DATABASE_HOSTNAME, inboundProperties.getProperty(DEBEZIUM_DATABASE_HOSTNAME)); - props.setProperty(DEBEZIUM_DATABASE_PORT, inboundProperties.getProperty(DEBEZIUM_DATABASE_PORT)); - props.setProperty(DEBEZIUM_DATABASE_USER, inboundProperties.getProperty(DEBEZIUM_DATABASE_USER)); - - String passwordString = inboundProperties.getProperty(DEBEZIUM_DATABASE_PASSWORD); - SynapseEnvironment synapseEnvironment = params.getSynapseEnvironment(); - MessageContext messageContext = synapseEnvironment.createMessageContext(); - - props.setProperty(DEBEZIUM_DATABASE_PASSWORD, resolveSecureVault(messageContext, passwordString)); - - props.setProperty(DEBEZIUM_DATABASE_DBNAME, inboundProperties.getProperty(DEBEZIUM_DATABASE_DBNAME)); - props.setProperty(DEBEZIUM_DATABASE_SERVER_ID, inboundProperties.getProperty(DEBEZIUM_DATABASE_SERVER_ID)); - - if (inboundProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL) != null) { - props.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, inboundProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL)); - } else { - props.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, TRUE); - } - - if (inboundProperties.getProperty(DEBEZIUM_TOPIC_PREFIX) != null) { - props.setProperty(DEBEZIUM_TOPIC_PREFIX, inboundProperties.getProperty(DEBEZIUM_TOPIC_PREFIX)); - } else { - props.setProperty(DEBEZIUM_TOPIC_PREFIX, params.getName() +"_topic"); - } - - props.setProperty(DEBEZIUM_VALUE_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); - props.setProperty(DEBEZIUM_KEY_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); - props.setProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE, TRUE); - props.setProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE, TRUE); - - if (inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL) != null) { - props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL)); - } else { - props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, "io.debezium.storage.file.history.FileSchemaHistory"); - } - - if (inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME) != null) { - props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME, inboundProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME)); - } else { - String filePath = "cdc/schemaHistory/" + params.getName() + "_.dat"; - createFile(filePath); - props.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME, filePath); - } - - if (inboundProperties.getProperty(DEBEZIUM_TABLES_INCLUDE_LIST) != null) { - props.setProperty(DEBEZIUM_TABLES_INCLUDE_LIST, inboundProperties.getProperty(DEBEZIUM_TABLES_INCLUDE_LIST)); - } - - if (inboundProperties.getProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST) != null) { - props.setProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST, inboundProperties.getProperty(DEBEZIUM_OPERATIONS_EXCLUDE_LIST)); - } - } catch (IOException e) { - throw new RuntimeException(e); - } catch (NullPointerException e) { - log.error("A required property value is not defined", e); - throw new RuntimeException(e); - } - return props; - } - - private static synchronized String resolveSecureVault(MessageContext messageContext, String passwordString) { - if (passwordString == null) { - return null; - } - Matcher lookupMatcher = vaultLookupPattern.matcher(passwordString); - String resolvedValue = ""; - if (lookupMatcher.find()) { - Value expression; - String expressionStr = lookupMatcher.group(1); - try { - expression = new Value(new SynapseXPath(expressionStr)); - - } catch (JaxenException e) { - throw new SynapseException("Error while building the expression : " + expressionStr, e); - } - resolvedValue = expression.evaluateValue(messageContext); - if (StringUtils.isEmpty(resolvedValue)) { - log.warn("Found Empty value for expression : " + expression.getExpression()); - resolvedValue = ""; - } - } else { - resolvedValue = passwordString; - } - return resolvedValue; - } - - @Override - public boolean startEndpoint(int port, String name, InboundProcessorParams inboundParameters) { - log.info("Starting CDC Endpoint on port " + port); - dataStore.registerListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME, InboundCDCConstants.CDC, name, - inboundParameters); - - boolean start = startListener(port, name, inboundParameters); - - if (start) { - //do nothing - } else { - dataStore.unregisterListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME); - return false; - } - return true; - } - - private void createFile (String filePath) throws IOException { - File file = new File(filePath); - file.getParentFile().mkdirs(); - if(!file.exists()) { - file.createNewFile(); - } - } - - @Override - public void closeEndpoint(int port) { - log.info("Closing CDC Endpoint on port " + port); - eventExecutor.getExecutorService().shutdown(); - dataStore.unregisterListeningEndpoint(port, SUPER_TENANT_DOMAIN_NAME); - - if (!CDCEventExecutorManager.getInstance().isRegisteredExecutor(port)) { - log.info("Listener Endpoint is not started"); - return; - } else if (dataStore.isEndpointRegistryEmpty(port)) { - CDCEventExecutorManager.getInstance().shutdownExecutor(port); - } - - } - -} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java deleted file mode 100644 index 80040b285f..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventExecutorManager.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.wso2.carbon.inbound.endpoint.protocol.cdc; - - -import java.util.concurrent.ConcurrentHashMap; - -public class CDCEventExecutorManager { - - private ConcurrentHashMap executorPoolMap = new ConcurrentHashMap(); - - private static CDCEventExecutorManager instance = null; - - public static CDCEventExecutorManager getInstance() { - if (instance == null) { - instance = new CDCEventExecutorManager(); - } - return instance; - } - - public void shutdownExecutor(int port) { - executorPoolMap.get(port).shutdownEventExecutor(); - executorPoolMap.remove(port); - } - - public void registerEventExecutor(int port, InboundCDCEventExecutor eventExecutor) { - executorPoolMap.put(port, eventExecutor); - } - - public boolean isRegisteredExecutor(int port) { - return executorPoolMap.containsKey(port); - } - -} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java index df9439c101..ec3af80f6a 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.wso2.carbon.inbound.endpoint.protocol.cdc; import io.debezium.engine.ChangeEvent; @@ -12,21 +30,13 @@ import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TABLE; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TS_MS; - public class CDCEventOutput { - private Object beforeEvent; - private Object afterEvent; - private Long ts_ms; - - private String database; - private Object table; - private String op; private JSONObject payload; private enum operations {c, r, u, d}; - CDCEventOutput (ChangeEvent event) { + CDCEventOutput(ChangeEvent event) { String valueString = event.value().toString(); JSONObject value = new JSONObject(valueString); this.payload = value.getJSONObject(PAYLOAD); @@ -41,10 +51,6 @@ public Object getJsonPayloadBeforeEvent() { } - public void setJsonPayloadBeforeEvent(Object beforeEvent) { - this.beforeEvent = beforeEvent; - } - public Object getJsonPayloadAfterEvent() { Object afterObject = null; if (payload.has(AFTER)) { @@ -53,10 +59,6 @@ public Object getJsonPayloadAfterEvent() { return afterObject; } - public void setJsonPayloadAfterEvent(Object afterEvent) { - this.afterEvent = afterEvent; - } - public Long getTs_ms() { if (payload.has(TS_MS)) { return payload.getLong(TS_MS); @@ -64,10 +66,6 @@ public Long getTs_ms() { return null; } - public void setTs_ms(Long ts_ms) { - this.ts_ms = ts_ms; - } - public String getDatabase() { if (getSource() != null) { if (getSource().has(DB)) { @@ -78,10 +76,6 @@ public String getDatabase() { return null; } - public void setDatabase(String database) { - this.database = database; - } - public Object getTable() { Object tableObject = null; if (getSource() != null) { @@ -92,10 +86,6 @@ public Object getTable() { return tableObject; } - public void setTable(Object table) { - this.table = table; - } - private JSONObject getSource () { if (payload.has(SOURCE)) { return payload.getJSONObject(SOURCE); @@ -109,6 +99,7 @@ public String getOp() { } return null; } + private String getOpString(String op) { if (op != null) { switch (operations.valueOf(op)) { @@ -125,10 +116,6 @@ private String getOpString(String op) { return null; } - public void setOp(String op) { - this.op = op; - } - public JSONObject getOutputJsonPayload () { if (payload == null) { return null; diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java new file mode 100644 index 0000000000..00e5fe2127 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import org.apache.axiom.om.OMElement; +import org.apache.axiom.om.util.UUIDGenerator; +import org.apache.axis2.AxisFault; +import org.apache.axis2.context.MessageContext; +import org.apache.axis2.transport.TransportUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.SynapseConstants; +import org.apache.synapse.SynapseException; +import org.apache.synapse.commons.json.JsonUtil; +import org.apache.synapse.core.SynapseEnvironment; +import org.apache.synapse.inbound.InboundEndpoint; +import org.apache.synapse.mediators.MediatorFaultHandler; +import org.apache.synapse.mediators.base.SequenceMediator; +import org.apache.synapse.transport.customlogsetter.CustomLogSetter; +import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericConstants; + +import java.io.InputStream; +import java.util.Map; +import java.util.Properties; + +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DATABASE_NAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.OPERATIONS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TABLES; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TS_MS; + +public class CDCInjectHandler { + + private static final Log logger = LogFactory.getLog(CDCInjectHandler.class); + + private String injectingSeq; + private String onErrorSeq; + private boolean sequential; + private Properties cdcProperties; + private SynapseEnvironment synapseEnvironment; + private Map transportHeaders; + private static final String contentType = "application/json"; + + public CDCInjectHandler(String injectingSeq, String onErrorSeq, boolean sequential, + SynapseEnvironment synapseEnvironment, Properties cdcProperties) { + this.injectingSeq = injectingSeq; + this.onErrorSeq = onErrorSeq; + this.sequential = sequential; + this.synapseEnvironment = synapseEnvironment; + this.cdcProperties = cdcProperties; + } + + /** + * Inject the message to the sequence + */ + public boolean invoke(Object object, String inboundEndpointName) throws SynapseException { + + ChangeEvent eventRecord = (ChangeEvent) object; + if (eventRecord == null || eventRecord.value() == null) { + if (logger.isDebugEnabled()) { + logger.debug("CDC Source Handler received empty event record"); + } + } else { + InputStream in = null; + try { + org.apache.synapse.MessageContext msgCtx = createMessageContext(); + msgCtx.setProperty(SynapseConstants.INBOUND_ENDPOINT_NAME, inboundEndpointName); + msgCtx.setProperty(SynapseConstants.ARTIFACT_NAME, SynapseConstants.FAIL_SAFE_MODE_INBOUND_ENDPOINT + inboundEndpointName); + msgCtx.setProperty(SynapseConstants.IS_INBOUND, true); + InboundEndpoint inboundEndpoint = msgCtx.getConfiguration().getInboundEndpoint(inboundEndpointName); + CustomLogSetter.getInstance().setLogAppender(inboundEndpoint.getArtifactContainerName()); + + CDCEventOutput cdcEventOutput = new CDCEventOutput(eventRecord); + msgCtx.setProperty(DATABASE_NAME, cdcEventOutput.getDatabase()); + msgCtx.setProperty(TABLES, cdcEventOutput.getTable().toString()); + msgCtx.setProperty(OPERATIONS, cdcEventOutput.getOp()); + msgCtx.setProperty(TS_MS, cdcEventOutput.getTs_ms().toString()); + + if (logger.isDebugEnabled()) { + logger.debug("Processed event : " + eventRecord); + } + MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) msgCtx) + .getAxis2MessageContext(); + + //Builder builder = null; + OMElement documentElement = null; + try { + documentElement = JsonUtil.getNewJsonPayload(axis2MsgCtx, + cdcEventOutput.getOutputJsonPayload().toString(), true, true); + + } catch (AxisFault ex) { + logger.error("Error while creating the OMElement", ex); + msgCtx.setProperty(SynapseConstants.ERROR_CODE, GenericConstants.INBOUND_BUILD_ERROR); + msgCtx.setProperty(SynapseConstants.ERROR_MESSAGE, ex.getMessage()); + SequenceMediator faultSequence = getFaultSequence(msgCtx); + faultSequence.mediate(msgCtx); + return true; + } + + // Inject the message to the sequence. + msgCtx.setEnvelope(TransportUtils.createSOAPEnvelope(documentElement)); + if (injectingSeq == null || injectingSeq.equals("")) { + logger.error("Sequence name not specified. Sequence : " + injectingSeq); + return false; + } + SequenceMediator seq = (SequenceMediator) synapseEnvironment.getSynapseConfiguration() + .getSequence(injectingSeq); + if (seq != null) { + if (logger.isDebugEnabled()) { + logger.debug("injecting message to sequence : " + injectingSeq); + } + if (!seq.isInitialized()) { + seq.init(synapseEnvironment); + } + SequenceMediator faultSequence = getFaultSequence(msgCtx); + MediatorFaultHandler mediatorFaultHandler = new MediatorFaultHandler(faultSequence); + msgCtx.pushFaultHandler(mediatorFaultHandler); + + if (!synapseEnvironment.injectInbound(msgCtx, seq, sequential)) { + return false; + } + } else { + logger.error("Sequence: " + injectingSeq + " not found"); + } + + } catch (AxisFault e) { + throw new RuntimeException(e); + } + } + return true; + } + + private SequenceMediator getFaultSequence(org.apache.synapse.MessageContext synCtx) { + SequenceMediator faultSequence = null; + if (this.onErrorSeq != null) { + faultSequence = (SequenceMediator) synCtx.getSequence(this.onErrorSeq); + } + + if (faultSequence == null) { + faultSequence = (SequenceMediator) synCtx.getFaultSequence(); + } + + return faultSequence; + } + + + /** + * @param transportHeaders the transportHeaders to set + */ + public void setTransportHeaders(Map transportHeaders) { + this.transportHeaders = transportHeaders; + } + + /** + * Create the initial message context for the file + */ + private org.apache.synapse.MessageContext createMessageContext() { + + org.apache.synapse.MessageContext msgCtx = synapseEnvironment.createMessageContext(); + MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) msgCtx) + .getAxis2MessageContext(); + axis2MsgCtx.setServerSide(true); + axis2MsgCtx.setMessageID(UUIDGenerator.getUUID()); + axis2MsgCtx.setProperty(MessageContext.TRANSPORT_HEADERS, transportHeaders); + msgCtx.setProperty(MessageContext.CLIENT_API_NON_BLOCKING, true); + return msgCtx; + } +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java deleted file mode 100644 index d249a37677..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCListener.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.wso2.carbon.inbound.endpoint.protocol.cdc; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.synapse.SynapseException; -import org.apache.synapse.inbound.InboundProcessorParams; -import org.apache.synapse.inbound.InboundRequestProcessor; -import org.wso2.carbon.inbound.endpoint.persistence.PersistenceUtils; - -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.INBOUND_ENDPOINT_PARAMETER_CDC_PORT; - -public class CDCListener implements InboundRequestProcessor { - - private InboundProcessorParams processorParams; - private int port; - private String name; - private static final Log LOGGER = LogFactory.getLog(CDCListener.class); - - public CDCListener(InboundProcessorParams params) { - - processorParams = params; - String portParam = params.getProperties() - .getProperty(INBOUND_ENDPOINT_PARAMETER_CDC_PORT); - try { - port = Integer.parseInt(portParam); - } catch (NumberFormatException e) { - handleException("Validation failed for the port parameter " + portParam, e); - } - name = params.getName(); - } - - protected void handleException(String msg, Exception e) { - LOGGER.error(msg, e); - throw new SynapseException(msg, e); - } - - @Override - public void init() { - System.out.println("Init called"); - int offsetPort = port + PersistenceUtils.getPortOffset(processorParams.getProperties()); - CDCEndpointManager.getInstance().startEndpoint(offsetPort, name, processorParams); - } - - @Override - public void destroy() { - System.out.println("Destroy called"); - int offsetPort = port + PersistenceUtils.getPortOffset(processorParams.getProperties()); - CDCEndpointManager.getInstance().closeEndpoint(offsetPort); - } -} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java new file mode 100644 index 0000000000..dbd56d18f2 --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.core.SynapseEnvironment; + +import java.util.Date; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; + +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.CDCProcessor.inboundEpEventQueueMap; + +/** + * This class implement the processing logic related to inbound CDC protocol. + * Common functionalities are include in synapse + * util that is found in synapse commons + */ +public class CDCPollingConsumer { + + private static final Log logger = LogFactory.getLog(CDCPollingConsumer.class); + private Properties cdcProperties; + private String inboundEndpointName; + private SynapseEnvironment synapseEnvironment; + private long scanInterval; + private Long lastRanTime; + private CDCInjectHandler injectHandler; + + public CDCPollingConsumer(Properties cdcProperties, String inboundEndpointName, SynapseEnvironment synapseEnvironment, + long scanInterval) { + this.cdcProperties = cdcProperties; + this.inboundEndpointName = inboundEndpointName; + this.synapseEnvironment = synapseEnvironment; + this.scanInterval = scanInterval; + this.lastRanTime = null; + } + + /** + * Register a handler to process the file stream after reading from the + * source + * + * @param injectHandler + */ + public void registerHandler(CDCInjectHandler injectHandler) { + this.injectHandler = injectHandler; + } + + /** + * This will be called by the task scheduler. If a cycle execution takes + * more than the schedule interval, tasks will call this method ignoring the + * interval. Timestamp based check is done to avoid that. + */ + public void execute() { + try { + if (logger.isDebugEnabled()) { + logger.debug("Start : CDC Inbound EP : " + inboundEndpointName); + } + // Check if the cycles are running in correct interval and start + // scan + long currentTime = (new Date()).getTime(); + if (lastRanTime == null || ((lastRanTime + (scanInterval)) <= currentTime)) { + lastRanTime = currentTime; + poll(); + } else if (logger.isDebugEnabled()) { + logger.debug( + "Skip cycle since cuncurrent rate is higher than the scan interval : CDC Inbound EP : " + inboundEndpointName); + } + if (logger.isDebugEnabled()) { + logger.debug("End : CDC Inbound EP : " + inboundEndpointName); + } + } catch (Exception e) { + logger.error("Error while getting events. " + e.getMessage(), e); + } + } + + /** + * Do the CDC processing operation for the given set of properties. Then inject + * according to the registered handler + */ + public ChangeEvent poll() { + + if (logger.isDebugEnabled()) { + logger.debug("Start : listening to DB events : "); + } + + BlockingQueue> eventQueue = inboundEpEventQueueMap.get(inboundEndpointName); + while (!eventQueue.isEmpty()) { + injectHandler.invoke(eventQueue.poll(), inboundEndpointName); + } + + if (logger.isDebugEnabled()) { + logger.debug("End : Listening to DB events : "); + } + return null; + } + + protected Properties getInboundProperties() { + return cdcProperties; + } + +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java new file mode 100644 index 0000000000..f5353e2acd --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import io.debezium.engine.ChangeEvent; +import io.debezium.engine.DebeziumEngine; +import io.debezium.engine.format.Json; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.MessageContext; +import org.apache.synapse.SynapseException; +import org.apache.synapse.core.SynapseEnvironment; +import org.apache.synapse.inbound.InboundProcessorParams; +import org.apache.synapse.inbound.InboundTaskProcessor; +import org.apache.synapse.mediators.Value; +import org.apache.synapse.task.TaskStartupObserver; +import org.apache.synapse.util.xpath.SynapseXPath; +import org.jaxen.JaxenException; +import org.wso2.carbon.inbound.endpoint.common.InboundRequestProcessorImpl; +import org.wso2.carbon.inbound.endpoint.common.InboundTask; +import org.wso2.carbon.inbound.endpoint.protocol.PollingConstants; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PASSWORD; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TOPIC_PREFIX; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TRUE; + +public class CDCProcessor extends InboundRequestProcessorImpl implements TaskStartupObserver, InboundTaskProcessor { + + private CDCPollingConsumer pollingConsumer; + private Properties cdcProperties; + private String injectingSeq; + private String onErrorSeq; + private boolean sequential; + + private static final String SECURE_VAULT_REGEX = "(wso2:vault-lookup\\('(.*?)'\\))"; + private static Pattern vaultLookupPattern = Pattern.compile(SECURE_VAULT_REGEX); + private static final String ENDPOINT_POSTFIX = "CDC" + COMMON_ENDPOINT_POSTFIX; + private static final String FILE_OFFSET_STORAGE_CLASS = "org.apache.kafka.connect.storage.FileOffsetBackingStore"; + private static final String FILE_SCHEMA_HISTORY_STORAGE_CLASS = "io.debezium.storage.file.history.FileSchemaHistory"; + private static final Log LOGGER = LogFactory.getLog(CDCProcessor.class); + protected static Map inboundEpEventQueueMap = new HashMap(); + private ExecutorService executorService = null; + + public CDCProcessor(InboundProcessorParams params) { + this.name = params.getName(); + this.injectingSeq = params.getInjectingSeq(); + this.onErrorSeq = params.getOnErrorSeq(); + this.synapseEnvironment = params.getSynapseEnvironment(); + this.cdcProperties = params.getProperties(); + setProperties(); + try { + this.interval = Long.parseLong(cdcProperties.getProperty(PollingConstants.INBOUND_ENDPOINT_INTERVAL)); + } catch (NumberFormatException nfe) { + throw new SynapseException("Invalid numeric value for interval.", nfe); + } catch (Exception e) { + throw new SynapseException("Invalid value for interval.", e); + } + this.sequential = true; + if (cdcProperties.getProperty(PollingConstants.INBOUND_ENDPOINT_SEQUENTIAL) != null) { + this.sequential = Boolean + .parseBoolean(cdcProperties.getProperty(PollingConstants.INBOUND_ENDPOINT_SEQUENTIAL)); + } + this.coordination = true; + if (cdcProperties.getProperty(PollingConstants.INBOUND_COORDINATION) != null) { + this.coordination = Boolean.parseBoolean(cdcProperties.getProperty(PollingConstants.INBOUND_COORDINATION)); + } + if (!inboundEpEventQueueMap.containsKey(this.name)) { + BlockingQueue> eventQueue = new LinkedBlockingQueue<>(); + inboundEpEventQueueMap.put(this.name, eventQueue); + } + } + + private void setProperties () { + LOGGER.info("Initializing the properties"); + try { + if (this.cdcProperties.getProperty(DEBEZIUM_OFFSET_STORAGE) == null) { + this.cdcProperties.setProperty(DEBEZIUM_OFFSET_STORAGE, FILE_OFFSET_STORAGE_CLASS); + } + if (this.cdcProperties.getProperty(DEBEZIUM_OFFSET_STORAGE).equals(FILE_OFFSET_STORAGE_CLASS)) { + String filePath; + if (this.cdcProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME) == null) { + filePath = "cdc/offsetStorage/" + this.name + "_.dat"; + } else { + filePath = this.cdcProperties.getProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME); + } + createFile(filePath); + this.cdcProperties.setProperty(DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME, filePath); + } + + if (this.cdcProperties.getProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS) == null) { + this.cdcProperties.setProperty(DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS, "1000"); + } + + String passwordString = this.cdcProperties.getProperty(DEBEZIUM_DATABASE_PASSWORD); + SynapseEnvironment synapseEnvironment = this.synapseEnvironment; + MessageContext messageContext = synapseEnvironment.createMessageContext(); + + this.cdcProperties.setProperty(DEBEZIUM_DATABASE_PASSWORD, resolveSecureVault(messageContext, passwordString)); + + if (this.cdcProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL) == null) { + this.cdcProperties.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, TRUE); + } + + if (this.cdcProperties.getProperty(DEBEZIUM_TOPIC_PREFIX) == null) { + this.cdcProperties.setProperty(DEBEZIUM_TOPIC_PREFIX, this.name +"_topic"); + } + + if (this.cdcProperties.getProperty(DEBEZIUM_VALUE_CONVERTER) == null) { + this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + } + if (this.cdcProperties.getProperty(DEBEZIUM_KEY_CONVERTER) == null) { + this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + } + if (this.cdcProperties.getProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE) == null) { + this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE, TRUE); + } + if (this.cdcProperties.getProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE) == null) { + this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE, TRUE); + } + + if (this.cdcProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL) == null) { + this.cdcProperties.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, FILE_SCHEMA_HISTORY_STORAGE_CLASS); + } + + if (this.cdcProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL).equals(FILE_SCHEMA_HISTORY_STORAGE_CLASS)) { + String filePath; + if (this.cdcProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME) == null) { + filePath = "cdc/schemaHistory/" + this.name + "_.dat"; + } else { + filePath = this.cdcProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME); + } + createFile(filePath); + this.cdcProperties.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME, filePath); + } + + } catch (IOException e) { + throw new RuntimeException(e); + } catch (NullPointerException e) { + LOGGER.error("A required property value is not defined", e); + throw new RuntimeException(e); + } + } + + private void createFile (String filePath) throws IOException { + File file = new File(filePath); + file.getParentFile().mkdirs(); + if(!file.exists()) { + file.createNewFile(); + } + } + + private static synchronized String resolveSecureVault(MessageContext messageContext, String passwordString) { + if (passwordString == null) { + return null; + } + Matcher lookupMatcher = vaultLookupPattern.matcher(passwordString); + String resolvedValue = ""; + if (lookupMatcher.find()) { + Value expression; + String expressionStr = lookupMatcher.group(1); + try { + expression = new Value(new SynapseXPath(expressionStr)); + + } catch (JaxenException e) { + throw new SynapseException("Error while building the expression : " + expressionStr, e); + } + resolvedValue = expression.evaluateValue(messageContext); + if (StringUtils.isEmpty(resolvedValue)) { + LOGGER.warn("Found Empty value for expression : " + expression.getExpression()); + resolvedValue = ""; + } + } else { + resolvedValue = passwordString; + } + return resolvedValue; + } + + + /** + * This will be called at the time of synapse artifact deployment. + */ + public void init() { + LOGGER.info("Inbound CDC listener " + name + " starting ..."); + pollingConsumer = new CDCPollingConsumer(cdcProperties, name, synapseEnvironment, interval); + pollingConsumer.registerHandler(new CDCInjectHandler(injectingSeq, onErrorSeq, sequential, + synapseEnvironment, cdcProperties)); + + DebeziumEngine> engine = DebeziumEngine.create(Json.class) + .using(this.cdcProperties) + .notifying(record -> { + try { + inboundEpEventQueueMap.get(this.name).offer(record, interval, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }).build(); + + executorService = Executors.newSingleThreadExecutor(); + executorService.execute(engine); + start(); + } + + + /** + * Register/start the schedule service + */ + public void start() { + InboundTask task = new CDCTask(pollingConsumer, interval); + start(task, ENDPOINT_POSTFIX); + } + + /** + * Remove inbound endpoints. + * + * @param removeTask Whether to remove scheduled task from the registry or not. + */ + @Override + public void destroy(boolean removeTask) { + if (removeTask) { + destroy(); + executorService.shutdown(); + } + } + + @Override + public void update() { + // This will not be called for inbound endpoints + } +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java deleted file mode 100644 index 7df87f60ff..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCSourceHandler.java +++ /dev/null @@ -1,165 +0,0 @@ -package org.wso2.carbon.inbound.endpoint.protocol.cdc; - -import io.debezium.engine.ChangeEvent; -import org.apache.axiom.om.OMAbstractFactory; -import org.apache.axiom.om.OMElement; -import org.apache.axiom.soap.SOAPEnvelope; -import org.apache.axiom.soap.SOAPFactory; -import org.apache.axiom.util.UIDGenerator; -import org.apache.axis2.AxisFault; -import org.apache.axis2.builder.Builder; -import org.apache.axis2.builder.BuilderUtil; -import org.apache.axis2.context.OperationContext; -import org.apache.axis2.context.ServiceContext; -import org.apache.axis2.description.InOutAxisOperation; -import org.apache.axis2.transport.TransportUtils; -import org.apache.commons.io.input.AutoCloseInputStream; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.synapse.MessageContext; -import org.apache.synapse.SynapseConstants; -import org.apache.synapse.api.ApiConstants; -import org.apache.synapse.core.axis2.Axis2MessageContext; -import org.apache.synapse.core.axis2.MessageContextCreatorForAxis2; -import org.apache.synapse.inbound.InboundProcessorParams; -import org.apache.synapse.mediators.MediatorFaultHandler; -import org.apache.synapse.mediators.base.SequenceMediator; -import org.wso2.carbon.inbound.endpoint.osgi.service.ServiceReferenceHolder; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.IOException; - - -import static org.wso2.carbon.inbound.endpoint.common.Constants.SUPER_TENANT_DOMAIN_NAME; -import static org.wso2.carbon.inbound.endpoint.common.Constants.TENANT_DOMAIN; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.*; - - -public class CDCSourceHandler { - - private int port; - private InboundProcessorParams params; - private static final String tenantDomain = SUPER_TENANT_DOMAIN_NAME; - private static final Log log = LogFactory.getLog(CDCSourceHandler.class); - private static final String contentType = "application/json"; - - public CDCSourceHandler(int port, InboundProcessorParams params) { - this.port = port; - this.params = params; - } - - public void requestReceived(ChangeEvent eventRecord) { - if (eventRecord == null || eventRecord.value() == null) { - log.debug("CDC Source Handler received empty event record"); - } else { - log.debug("CDC Source Handler request received"); - - MessageContext synCtx = null; - try { - - synCtx = getSynapseMessageContext(tenantDomain); - - CDCEventOutput cdcEventOutput = new CDCEventOutput(eventRecord); - synCtx.setProperty(DATABASE_NAME, cdcEventOutput.getDatabase()); - synCtx.setProperty(TABLES, cdcEventOutput.getTable().toString()); - synCtx.setProperty(OPERATIONS, cdcEventOutput.getOp()); - synCtx.setProperty(TS_MS, cdcEventOutput.getTs_ms().toString()); - - org.apache.axis2.context.MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) synCtx) - .getAxis2MessageContext(); - Builder builder = BuilderUtil.getBuilderFromSelector(contentType, axis2MsgCtx); - - if (builder != null) { - String serializedChangeEvent = cdcEventOutput.getOutputJsonPayload().toString(); - InputStream in = new AutoCloseInputStream( - new ByteArrayInputStream(serializedChangeEvent.getBytes())); - - OMElement documentElement = builder.processDocument(in, contentType, axis2MsgCtx); - synCtx.setEnvelope(TransportUtils.createSOAPEnvelope(documentElement)); - - if (log.isDebugEnabled()) { - log.debug("CDCEvent being injected to Sequence"); - } - injectForMediation(synCtx); - return; - } - - } catch (AxisFault e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - private static org.apache.axis2.context.MessageContext createAxis2MessageContext() { - org.apache.axis2.context.MessageContext axis2MsgCtx = new org.apache.axis2.context.MessageContext(); - axis2MsgCtx.setMessageID(UIDGenerator.generateURNString()); - axis2MsgCtx.setConfigurationContext( - ServiceReferenceHolder.getInstance().getConfigurationContextService().getServerConfigContext()); - axis2MsgCtx.setProperty(org.apache.axis2.context.MessageContext.CLIENT_API_NON_BLOCKING, Boolean.TRUE); - axis2MsgCtx.setServerSide(true); - - return axis2MsgCtx; - } - - private static org.apache.synapse.MessageContext createSynapseMessageContext(String tenantDomain) throws AxisFault { - org.apache.axis2.context.MessageContext axis2MsgCtx = createAxis2MessageContext(); - ServiceContext svcCtx = new ServiceContext(); - OperationContext opCtx = new OperationContext(new InOutAxisOperation(), svcCtx); - axis2MsgCtx.setServiceContext(svcCtx); - axis2MsgCtx.setOperationContext(opCtx); - - axis2MsgCtx.setProperty(TENANT_DOMAIN, tenantDomain); - - SOAPFactory fac = OMAbstractFactory.getSOAP11Factory(); - SOAPEnvelope envelope = fac.getDefaultEnvelope(); - axis2MsgCtx.setEnvelope(envelope); - return MessageContextCreatorForAxis2.getSynapseMessageContext(axis2MsgCtx); - } - - public org.apache.synapse.MessageContext getSynapseMessageContext(String tenantDomain) throws AxisFault { - MessageContext synCtx = createSynapseMessageContext(tenantDomain); - synCtx.setProperty(SynapseConstants.IS_INBOUND, true); - ((Axis2MessageContext) synCtx).getAxis2MessageContext().setProperty(SynapseConstants.IS_INBOUND, true); - - return synCtx; - } - - - private void injectForMediation(org.apache.synapse.MessageContext synCtx) { - SequenceMediator faultSequence = getFaultSequence(synCtx); - - MediatorFaultHandler mediatorFaultHandler = new MediatorFaultHandler(faultSequence); - synCtx.pushFaultHandler(mediatorFaultHandler); - if (log.isDebugEnabled()) { - log.debug("injecting message to sequence : " + params.getInjectingSeq()); - } - synCtx.setProperty("inbound.endpoint.name", params.getName()); - synCtx.setProperty(ApiConstants.API_CALLER, params.getName()); - - SequenceMediator injectingSequence = null; - if (params.getInjectingSeq() != null) { - injectingSequence = (SequenceMediator) synCtx.getSequence(params.getInjectingSeq()); - } - if (injectingSequence == null) { - injectingSequence = (SequenceMediator) synCtx.getMainSequence(); - } - - synCtx.getEnvironment().injectMessage(synCtx, injectingSequence); - - } - - private SequenceMediator getFaultSequence(org.apache.synapse.MessageContext synCtx) { - SequenceMediator faultSequence = null; - if (params.getOnErrorSeq() != null) { - faultSequence = (SequenceMediator) synCtx.getSequence(params.getOnErrorSeq()); - } - if (faultSequence == null) { - faultSequence = (SequenceMediator) synCtx.getFaultSequence(); - } - return faultSequence; - } - -} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java new file mode 100644 index 0000000000..4795fab44a --- /dev/null +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.carbon.inbound.endpoint.protocol.cdc; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.synapse.core.SynapseEnvironment; +import org.wso2.carbon.inbound.endpoint.common.InboundTask; + +import java.util.Properties; + +/** + * CDCTask class is used to schedule tasks for inbound CDC processor when + * required (coordination==true) + */ +public class CDCTask extends InboundTask { + + private static final Log logger = LogFactory.getLog(CDCTask.class.getName()); + + private CDCPollingConsumer pollingConsumer; + + public CDCTask(CDCPollingConsumer pollingConsumer, long interval) { + logger.debug("CDC Task initialize."); + this.interval = interval; + this.pollingConsumer = pollingConsumer; + } + + protected void taskExecute() { + if (logger.isDebugEnabled()) { + logger.debug("CDC Task executing."); + } + pollingConsumer.execute(); + } + + @Override + public Properties getInboundProperties() { + return pollingConsumer.getInboundProperties(); + } + + public void init(SynapseEnvironment synapseEnvironment) { + if (logger.isDebugEnabled()) { + logger.debug("Initializing Task."); + } + } + + public void destroy() { + if (logger.isDebugEnabled()) { + logger.debug("Destroying Task. "); + } + } +} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java index 156a414e74..c23931d9b2 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java @@ -1,27 +1,31 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.wso2.carbon.inbound.endpoint.protocol.cdc; -public class InboundCDCConstants { +class InboundCDCConstants { /** Inbound Endpoint Parameters **/ - public static final String CDC = "cdc"; - public static final String INBOUND_ENDPOINT_PARAMETER_CDC_PORT = "inbound.cdc.port"; - //debezium - public static final String DEBEZIUM_NAME = "name"; - public static final String DEBEZIUM_SNAPSHOT_MODE = "snapshot.mode"; - public static final String DEBEZIUM_MAX_THREADS = "snapshot.max.threads"; public static final String DEBEZIUM_OFFSET_STORAGE = "offset.storage"; public static final String DEBEZIUM_OFFSET_STORAGE_FILE_FILENAME = "offset.storage.file.filename"; public static final String DEBEZIUM_OFFSET_FLUSH_INTERVAL_MS = "offset.flush.interval.ms"; - public static final String DEBEZIUM_CONNECTOR_CLASS = "connector.class"; - public static final String DEBEZIUM_DATABASE_HOSTNAME = "database.hostname"; - public static final String DEBEZIUM_DATABASE_PORT = "database.port"; - public static final String DEBEZIUM_DATABASE_USER = "database.user"; public static final String DEBEZIUM_DATABASE_PASSWORD = "database.password"; - public static final String DEBEZIUM_DATABASE_DBNAME = "database.dbname"; - public static final String DEBEZIUM_TABLES_INCLUDE_LIST = "table.include.list"; - public static final String DEBEZIUM_OPERATIONS_EXCLUDE_LIST = "skipped.operations"; - public static final String DEBEZIUM_DATABASE_SERVER_ID = "database.server.id"; public static final String DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL = "database.allowPublicKeyRetrieval"; public static final String DEBEZIUM_TOPIC_PREFIX = "topic.prefix"; @@ -48,4 +52,5 @@ public class InboundCDCConstants { public static final String TABLE = "table"; public static final String TRUE = "true"; + } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java deleted file mode 100644 index 37e3fbd70d..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCEventExecutor.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2019, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. - * - * WSO2 Inc. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.wso2.carbon.inbound.endpoint.protocol.cdc; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -public class InboundCDCEventExecutor { - - private ExecutorService executorService; - - public InboundCDCEventExecutor () { - executorService = Executors.newSingleThreadExecutor(); - } - - public ExecutorService getExecutorService() { - return this.executorService; - } - - public void shutdownEventExecutor() { - executorService.shutdown(); - try { - if (!executorService.awaitTermination(800, TimeUnit.MILLISECONDS)) { - executorService.shutdownNow(); - } - } catch (InterruptedException e) { - executorService.shutdownNow(); - } - } - - -} diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties deleted file mode 100644 index 1da8230168..0000000000 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/resources/log4j.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Root logger option -log4j.rootLogger=INFO, stdout - -# Direct log messages to stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/pom.xml b/pom.xml index d184d4e4cf..0a87249f76 100644 --- a/pom.xml +++ b/pom.xml @@ -1104,6 +1104,12 @@ ${mysql.connector.version} test + + com.oracle.database.jdbc + ojdbc8 + ${ojdbc8.version} + test + org.apache.derby.wso2 derby @@ -1465,6 +1471,21 @@ debezium-connector-mysql ${debezium.version} + + io.debezium + debezium-connector-sqlserver + ${debezium.version} + + + io.debezium + debezium-connector-postgres + ${debezium.version} + + + io.debezium + debezium-connector-oracle + ${debezium.version} + io.debezium debezium-core @@ -1644,11 +1665,12 @@ 2.4.1.Final + 21.3.0.0 3.3.1 - 4.0.0-wso2v38 + 4.0.0-wso2v40 [4.0.0, 4.0.1) 4.7.175 1.1.3 From 67883e384dcc041516d072cdbae5e3ded8d35b38 Mon Sep 17 00:00:00 2001 From: rusirijayodaillesinghe Date: Fri, 28 Jul 2023 13:28:17 +0530 Subject: [PATCH 3/4] Set the output format as json in a way a user cannot override Get the param as allowed.operations instead of skipped.operations Change the property names of the sequence Add isDebugEnable check where string concaterntions are done. Change Json library to Gson and resolve the project snapshot version Directly throw the exceptions without using fault sequence Add bundled dependencies --- .../pom.xml | 2 +- components/business-adaptors/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- components/crypto-service/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- components/data/data-services/pom.xml | 2 +- components/data/pom.xml | 2 +- components/javax.cache/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/data-publishers/pom.xml | 2 +- .../pom.xml | 2 +- .../utsecurity/pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/extensions/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 29 ++-- .../endpoint/protocol/cdc/CDCEventOutput.java | 51 +++--- .../protocol/cdc/CDCInjectHandler.java | 150 +++++++----------- .../protocol/cdc/CDCPollingConsumer.java | 81 ++++++---- .../endpoint/protocol/cdc/CDCProcessor.java | 137 +++++++--------- .../endpoint/protocol/cdc/CDCTask.java | 12 +- .../protocol/cdc/InboundCDCConstants.java | 10 +- .../CDCInbound/cdc-inbound-endpoint.xml | 17 +- .../resources/CDCInbound/cdc_process_seq.xml | 8 +- .../mediation/inbound-endpoints/pom.xml | 2 +- .../org.wso2.carbon.mediator.cache/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/mediators/pom.xml | 2 +- components/mediation/pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/registry/pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/security/pom.xml | 2 +- .../startup/mediation-startup/pom.xml | 2 +- components/mediation/startup/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/tasks/pom.xml | 2 +- .../pom.xml | 2 +- components/mediation/transports/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 4 +- .../org.wso2.micro.integrator.core/pom.xml | 2 +- .../pom.xml | 2 +- .../org.wso2.micro.integrator.probes/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../org.wso2.micro.integrator.server/pom.xml | 2 +- .../org.wso2.micro.integrator.utils/pom.xml | 2 +- components/pom.xml | 2 +- distribution/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../data-services-feature/pom.xml | 2 +- features/data-features/pom.xml | 2 +- .../pom.xml | 2 +- .../builtin-mediators/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../data-publisher-features/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../inbound-endpoint/pom.xml | 2 +- .../mediation-features/ntask-feature/pom.xml | 2 +- features/mediation-features/pom.xml | 2 +- .../pom.xml | 2 +- .../security-features/pom.xml | 2 +- .../pom.xml | 2 +- .../transport-features/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- features/pom.xml | 2 +- features/wso2mi-hl7/pom.xml | 2 +- integration/automation-extensions/pom.xml | 2 +- integration/dataservice-hosting-tests/pom.xml | 2 +- .../samples/data-services/clients/pom.xml | 2 +- .../samples/data-services/pom.xml | 2 +- .../dataservice-hosting-tests/samples/pom.xml | 2 +- .../tests-common/admin-clients/pom.xml | 2 +- .../integration-test-utils/pom.xml | 2 +- .../tests-common/pom.xml | 2 +- .../tests-integration/pom.xml | 2 +- .../tests-integration/tests/pom.xml | 2 +- integration/management-api-tests/pom.xml | 2 +- .../mediation-tests/coverage-report/pom.xml | 2 +- integration/mediation-tests/pom.xml | 2 +- .../mediation-tests/service-samples/pom.xml | 2 +- .../mediation-tests/tests-http/pom.xml | 2 +- .../mediation-tests/tests-mediator-1/pom.xml | 2 +- .../mediation-tests/tests-mediator-2/pom.xml | 2 +- .../mediation-tests/tests-other/pom.xml | 2 +- .../tests-patches-with-smb2/pom.xml | 2 +- .../mediation-tests/tests-patches/pom.xml | 2 +- .../mediation-tests/tests-platform/pom.xml | 2 +- .../tests-platform/tests-coordination/pom.xml | 2 +- .../tests-platform/tests-rabbitmq/pom.xml | 2 +- .../tests-platform/tests-transaction/pom.xml | 2 +- .../tests-platform/tests-userstore/pom.xml | 2 +- .../mediation-tests/tests-sample/pom.xml | 2 +- .../mediation-tests/tests-service/pom.xml | 2 +- .../mediation-tests/tests-transport-2/pom.xml | 2 +- .../mediation-tests/tests-transport/pom.xml | 2 +- integration/pom.xml | 2 +- integration/samples/product/pom.xml | 2 +- .../tests-common/admin-clients/pom.xml | 2 +- .../integration-test-utils/pom.xml | 2 +- integration/tests-common/pom.xml | 2 +- p2-profile/pom.xml | 2 +- pom.xml | 130 ++------------- 129 files changed, 348 insertions(+), 517 deletions(-) diff --git a/components/business-adaptors/org.wso2.micro.integrator.business.messaging.hl7/pom.xml b/components/business-adaptors/org.wso2.micro.integrator.business.messaging.hl7/pom.xml index 0a69bbd758..8b59f85737 100644 --- a/components/business-adaptors/org.wso2.micro.integrator.business.messaging.hl7/pom.xml +++ b/components/business-adaptors/org.wso2.micro.integrator.business.messaging.hl7/pom.xml @@ -21,7 +21,7 @@ business-adaptors org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/business-adaptors/pom.xml b/components/business-adaptors/pom.xml index 6e0757b4f7..50855f5720 100644 --- a/components/business-adaptors/pom.xml +++ b/components/business-adaptors/pom.xml @@ -21,7 +21,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/crypto-service/org.wso2.micro.integrator.crypto.impl/pom.xml b/components/crypto-service/org.wso2.micro.integrator.crypto.impl/pom.xml index 56fd0070b6..dc4f5931e4 100644 --- a/components/crypto-service/org.wso2.micro.integrator.crypto.impl/pom.xml +++ b/components/crypto-service/org.wso2.micro.integrator.crypto.impl/pom.xml @@ -22,7 +22,7 @@ crypto-service org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/crypto-service/org.wso2.micro.integrator.crypto.provider/pom.xml b/components/crypto-service/org.wso2.micro.integrator.crypto.provider/pom.xml index 3be79fceb1..458b982aee 100644 --- a/components/crypto-service/org.wso2.micro.integrator.crypto.provider/pom.xml +++ b/components/crypto-service/org.wso2.micro.integrator.crypto.provider/pom.xml @@ -22,7 +22,7 @@ crypto-service org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/crypto-service/pom.xml b/components/crypto-service/pom.xml index 49edd9d603..2e1d0590e2 100644 --- a/components/crypto-service/pom.xml +++ b/components/crypto-service/pom.xml @@ -22,7 +22,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.capp.deployer/pom.xml b/components/data/data-services/org.wso2.micro.integrator.dataservices.capp.deployer/pom.xml index ca19abee70..44be98d96c 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.capp.deployer/pom.xml +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.capp.deployer/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei data-services - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.common/pom.xml b/components/data/data-services/org.wso2.micro.integrator.dataservices.common/pom.xml index 33b5cb3545..cee5e7c931 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.common/pom.xml +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.common/pom.xml @@ -19,7 +19,7 @@ org.wso2.ei data-services - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/pom.xml b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/pom.xml index a21c7c6ed9..90bf7482c0 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.core/pom.xml +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.core/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei data-services - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.odata.endpoint/pom.xml b/components/data/data-services/org.wso2.micro.integrator.dataservices.odata.endpoint/pom.xml index 3ff1f34f98..b21c1cd195 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.odata.endpoint/pom.xml +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.odata.endpoint/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei data-services - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/org.wso2.micro.integrator.dataservices.sql.driver/pom.xml b/components/data/data-services/org.wso2.micro.integrator.dataservices.sql.driver/pom.xml index 2b78aad02f..4f4db79bdd 100644 --- a/components/data/data-services/org.wso2.micro.integrator.dataservices.sql.driver/pom.xml +++ b/components/data/data-services/org.wso2.micro.integrator.dataservices.sql.driver/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei data-services - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/data-services/pom.xml b/components/data/data-services/pom.xml index 3aecd10820..6f30cb90e3 100644 --- a/components/data/data-services/pom.xml +++ b/components/data/data-services/pom.xml @@ -21,7 +21,7 @@ data org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/data/pom.xml b/components/data/pom.xml index b66399951f..475ed65475 100644 --- a/components/data/pom.xml +++ b/components/data/pom.xml @@ -21,7 +21,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/javax.cache/pom.xml b/components/javax.cache/pom.xml index 746253fd9d..779c7dcb44 100644 --- a/components/javax.cache/pom.xml +++ b/components/javax.cache/pom.xml @@ -21,7 +21,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.data.publisher.util/pom.xml b/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.data.publisher.util/pom.xml index aff407c3d5..6fa41c7c08 100644 --- a/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.data.publisher.util/pom.xml +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.data.publisher.util/pom.xml @@ -23,7 +23,7 @@ org.wso2.ei data-publishers - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.messageflow.data.publisher/pom.xml b/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.messageflow.data.publisher/pom.xml index c2b9773fdd..95504f6199 100644 --- a/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.messageflow.data.publisher/pom.xml +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.analytics.messageflow.data.publisher/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei data-publishers - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/pom.xml b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/pom.xml index a953f5e679..db5bf3f675 100644 --- a/components/mediation/data-publishers/org.wso2.micro.integrator.observability/pom.xml +++ b/components/mediation/data-publishers/org.wso2.micro.integrator.observability/pom.xml @@ -21,7 +21,7 @@ data-publishers org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/data-publishers/pom.xml b/components/mediation/data-publishers/pom.xml index ea85db155c..73ea65aee4 100644 --- a/components/mediation/data-publishers/pom.xml +++ b/components/mediation/data-publishers/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/pom.xml b/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/pom.xml index 5f096b43c8..cdd74bbc72 100644 --- a/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/pom.xml +++ b/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/pom.xml @@ -21,7 +21,7 @@ extensions org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/utsecurity/pom.xml b/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/utsecurity/pom.xml index 6a5fd018b9..bc4c6f98b1 100644 --- a/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/utsecurity/pom.xml +++ b/components/mediation/extensions/org.wso2.micro.integrator.security.handlers/utsecurity/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei org.wso2.micro.integrator.security.handlers - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml b/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml index 53da2fd2a4..a796cabfbc 100644 --- a/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml +++ b/components/mediation/extensions/org.wso2.micro.integrator.transport.handlers/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei extensions - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/extensions/pom.xml b/components/mediation/extensions/pom.xml index 392b190e51..569fe91a6d 100644 --- a/components/mediation/extensions/pom.xml +++ b/components/mediation/extensions/pom.xml @@ -22,7 +22,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.osgi/pom.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.osgi/pom.xml index ccfcb449a6..2231b1fd56 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.osgi/pom.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.osgi/pom.xml @@ -23,7 +23,7 @@ inbound-endpoints org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.persistence/pom.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.persistence/pom.xml index a3a81d1e2d..06a1fa1d5d 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.persistence/pom.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint.persistence/pom.xml @@ -21,7 +21,7 @@ inbound-endpoints org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml index bb7dd45d96..20b678602e 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/pom.xml @@ -20,7 +20,7 @@ inbound-endpoints org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml @@ -170,6 +170,10 @@ org.apache.axis2.transport axis2-transport-rabbitmq-amqp + + org.wso2.orbit.debezium + debezium + org.apache.logging.log4j log4j-core @@ -181,24 +185,12 @@ test - org.wso2.carbon.mediation - org.wso2.carbon.inbound.endpoint.persistence - - - io.debezium - debezium-embedded - - - io.debezium - debezium-api - - - io.debezium - debezium-connector-mysql + com.google.code.gson + gson - org.slf4j - slf4j-api + org.wso2.carbon + org.wso2.carbon.securevault @@ -232,7 +224,8 @@ org.wso2.carbon.inbound.endpoint.persistence, - org.wso2.carbon.inbound.endpoint.osgi.service; + org.wso2.carbon.inbound.endpoint.osgi.service, + *;resolution:=optional * diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java index ec3af80f6a..cdb52b0b9d 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java @@ -18,8 +18,10 @@ package org.wso2.carbon.inbound.endpoint.protocol.cdc; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import io.debezium.engine.ChangeEvent; -import org.json.JSONObject; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.AFTER; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.BEFORE; @@ -32,36 +34,27 @@ public class CDCEventOutput { - private JSONObject payload; + private JsonObject payload; private enum operations {c, r, u, d}; CDCEventOutput(ChangeEvent event) { String valueString = event.value().toString(); - JSONObject value = new JSONObject(valueString); - this.payload = value.getJSONObject(PAYLOAD); + JsonObject value = new Gson().fromJson(valueString, JsonObject.class); + this.payload = value.getAsJsonObject(PAYLOAD); } - public Object getJsonPayloadBeforeEvent() { - Object beforeObject = null; - if (payload.has(BEFORE)) { - beforeObject = payload.get(BEFORE); - } - return beforeObject; - + public JsonElement getJsonPayloadBeforeEvent() { + return payload.get(BEFORE); } - public Object getJsonPayloadAfterEvent() { - Object afterObject = null; - if (payload.has(AFTER)) { - afterObject = payload.get(AFTER); - } - return afterObject; + public JsonElement getJsonPayloadAfterEvent() { + return payload.get(AFTER); } public Long getTs_ms() { if (payload.has(TS_MS)) { - return payload.getLong(TS_MS); + return payload.get(TS_MS).getAsLong(); } return null; } @@ -69,15 +62,15 @@ public Long getTs_ms() { public String getDatabase() { if (getSource() != null) { if (getSource().has(DB)) { - return getSource().getString(DB); + return getSource().get(DB).getAsString(); } return null; } return null; } - public Object getTable() { - Object tableObject = null; + public JsonElement getTable() { + JsonElement tableObject = null; if (getSource() != null) { if (getSource().has(TABLE)) { tableObject = getSource().get(TABLE); @@ -86,16 +79,16 @@ public Object getTable() { return tableObject; } - private JSONObject getSource () { + private JsonObject getSource () { if (payload.has(SOURCE)) { - return payload.getJSONObject(SOURCE); + return payload.getAsJsonObject(SOURCE); } return null; } public String getOp() { if (payload.has(OP)) { - return getOpString(payload.getString(OP)); + return getOpString(payload.get(OP).getAsString()); } return null; } @@ -116,14 +109,14 @@ private String getOpString(String op) { return null; } - public JSONObject getOutputJsonPayload () { + public JsonObject getOutputJsonPayload () { if (payload == null) { return null; } - JSONObject jsonPayload = new JSONObject(); - jsonPayload.put(OP, getOp()); - jsonPayload.put(BEFORE, getJsonPayloadBeforeEvent()); - jsonPayload.put(AFTER, getJsonPayloadAfterEvent()); + JsonObject jsonPayload = new JsonObject(); + jsonPayload.addProperty(OP, getOp()); + jsonPayload.add(BEFORE, getJsonPayloadBeforeEvent()); + jsonPayload.add(AFTER, getJsonPayloadAfterEvent()); return jsonPayload; } } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java index 00e5fe2127..6fa691502f 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCInjectHandler.java @@ -24,26 +24,22 @@ import org.apache.axis2.AxisFault; import org.apache.axis2.context.MessageContext; import org.apache.axis2.transport.TransportUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.synapse.SynapseConstants; -import org.apache.synapse.SynapseException; import org.apache.synapse.commons.json.JsonUtil; import org.apache.synapse.core.SynapseEnvironment; import org.apache.synapse.inbound.InboundEndpoint; -import org.apache.synapse.mediators.MediatorFaultHandler; import org.apache.synapse.mediators.base.SequenceMediator; import org.apache.synapse.transport.customlogsetter.CustomLogSetter; -import org.wso2.carbon.inbound.endpoint.protocol.generic.GenericConstants; -import java.io.InputStream; -import java.util.Map; import java.util.Properties; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DATABASE_NAME; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.OPERATIONS; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TABLES; -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TS_MS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.CDC_DATABASE_NAME; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.CDC_OPERATIONS; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.CDC_TABLES; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.CDC_TS_MS; public class CDCInjectHandler { @@ -54,8 +50,6 @@ public class CDCInjectHandler { private boolean sequential; private Properties cdcProperties; private SynapseEnvironment synapseEnvironment; - private Map transportHeaders; - private static final String contentType = "application/json"; public CDCInjectHandler(String injectingSeq, String onErrorSeq, boolean sequential, SynapseEnvironment synapseEnvironment, Properties cdcProperties) { @@ -69,103 +63,76 @@ public CDCInjectHandler(String injectingSeq, String onErrorSeq, boolean sequenti /** * Inject the message to the sequence */ - public boolean invoke(Object object, String inboundEndpointName) throws SynapseException { + public boolean invoke(Object object, String inboundEndpointName) { ChangeEvent eventRecord = (ChangeEvent) object; if (eventRecord == null || eventRecord.value() == null) { + logger.debug("CDC Source Handler received empty event record"); + } else { + org.apache.synapse.MessageContext msgCtx = createMessageContext(); + msgCtx.setProperty(SynapseConstants.INBOUND_ENDPOINT_NAME, inboundEndpointName); + msgCtx.setProperty(SynapseConstants.ARTIFACT_NAME, SynapseConstants.FAIL_SAFE_MODE_INBOUND_ENDPOINT + inboundEndpointName); + msgCtx.setProperty(SynapseConstants.IS_INBOUND, true); + InboundEndpoint inboundEndpoint = msgCtx.getConfiguration().getInboundEndpoint(inboundEndpointName); + CustomLogSetter.getInstance().setLogAppender(inboundEndpoint.getArtifactContainerName()); + + CDCEventOutput cdcEventOutput = new CDCEventOutput(eventRecord); + msgCtx.setProperty(CDC_DATABASE_NAME, cdcEventOutput.getDatabase()); + msgCtx.setProperty(CDC_TABLES, cdcEventOutput.getTable().toString()); + msgCtx.setProperty(CDC_OPERATIONS, cdcEventOutput.getOp()); + msgCtx.setProperty(CDC_TS_MS, cdcEventOutput.getTs_ms().toString()); + if (logger.isDebugEnabled()) { - logger.debug("CDC Source Handler received empty event record"); + logger.debug("Processed event : " + eventRecord); } - } else { - InputStream in = null; + MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) msgCtx) + .getAxis2MessageContext(); + + OMElement documentElement = null; try { - org.apache.synapse.MessageContext msgCtx = createMessageContext(); - msgCtx.setProperty(SynapseConstants.INBOUND_ENDPOINT_NAME, inboundEndpointName); - msgCtx.setProperty(SynapseConstants.ARTIFACT_NAME, SynapseConstants.FAIL_SAFE_MODE_INBOUND_ENDPOINT + inboundEndpointName); - msgCtx.setProperty(SynapseConstants.IS_INBOUND, true); - InboundEndpoint inboundEndpoint = msgCtx.getConfiguration().getInboundEndpoint(inboundEndpointName); - CustomLogSetter.getInstance().setLogAppender(inboundEndpoint.getArtifactContainerName()); - - CDCEventOutput cdcEventOutput = new CDCEventOutput(eventRecord); - msgCtx.setProperty(DATABASE_NAME, cdcEventOutput.getDatabase()); - msgCtx.setProperty(TABLES, cdcEventOutput.getTable().toString()); - msgCtx.setProperty(OPERATIONS, cdcEventOutput.getOp()); - msgCtx.setProperty(TS_MS, cdcEventOutput.getTs_ms().toString()); + documentElement = JsonUtil.getNewJsonPayload(axis2MsgCtx, + cdcEventOutput.getOutputJsonPayload().toString(), true, true); - if (logger.isDebugEnabled()) { - logger.debug("Processed event : " + eventRecord); - } - MessageContext axis2MsgCtx = ((org.apache.synapse.core.axis2.Axis2MessageContext) msgCtx) - .getAxis2MessageContext(); - - //Builder builder = null; - OMElement documentElement = null; - try { - documentElement = JsonUtil.getNewJsonPayload(axis2MsgCtx, - cdcEventOutput.getOutputJsonPayload().toString(), true, true); - - } catch (AxisFault ex) { - logger.error("Error while creating the OMElement", ex); - msgCtx.setProperty(SynapseConstants.ERROR_CODE, GenericConstants.INBOUND_BUILD_ERROR); - msgCtx.setProperty(SynapseConstants.ERROR_MESSAGE, ex.getMessage()); - SequenceMediator faultSequence = getFaultSequence(msgCtx); - faultSequence.mediate(msgCtx); - return true; - } + } catch (AxisFault ex) { + handleError("Error while creating the OMElement"); + } - // Inject the message to the sequence. + // Inject the message to the sequence. + try { msgCtx.setEnvelope(TransportUtils.createSOAPEnvelope(documentElement)); - if (injectingSeq == null || injectingSeq.equals("")) { - logger.error("Sequence name not specified. Sequence : " + injectingSeq); - return false; + } catch (AxisFault e) { + handleError("Error while creating the SOAP Envelop"); + } + + if (StringUtils.isBlank(injectingSeq)) { + handleError("Injecting sequence name not specified"); + } + SequenceMediator seq = (SequenceMediator) synapseEnvironment.getSynapseConfiguration() + .getSequence(injectingSeq); + if (seq != null) { + if (logger.isDebugEnabled()) { + logger.debug("injecting message to sequence : " + injectingSeq); } - SequenceMediator seq = (SequenceMediator) synapseEnvironment.getSynapseConfiguration() - .getSequence(injectingSeq); - if (seq != null) { - if (logger.isDebugEnabled()) { - logger.debug("injecting message to sequence : " + injectingSeq); - } - if (!seq.isInitialized()) { - seq.init(synapseEnvironment); - } - SequenceMediator faultSequence = getFaultSequence(msgCtx); - MediatorFaultHandler mediatorFaultHandler = new MediatorFaultHandler(faultSequence); - msgCtx.pushFaultHandler(mediatorFaultHandler); - - if (!synapseEnvironment.injectInbound(msgCtx, seq, sequential)) { - return false; - } - } else { - logger.error("Sequence: " + injectingSeq + " not found"); + if (!seq.isInitialized()) { + seq.init(synapseEnvironment); } - } catch (AxisFault e) { - throw new RuntimeException(e); + seq.setErrorHandler(onErrorSeq); + + if (!synapseEnvironment.injectInbound(msgCtx, seq, sequential)) { + handleError("Failed to inject the sequence"); + } + } else { + handleError("Sequence:" + injectingSeq + " not found"); } } return true; } - private SequenceMediator getFaultSequence(org.apache.synapse.MessageContext synCtx) { - SequenceMediator faultSequence = null; - if (this.onErrorSeq != null) { - faultSequence = (SequenceMediator) synCtx.getSequence(this.onErrorSeq); - } - - if (faultSequence == null) { - faultSequence = (SequenceMediator) synCtx.getFaultSequence(); - } - - return faultSequence; - } - - - /** - * @param transportHeaders the transportHeaders to set - */ - public void setTransportHeaders(Map transportHeaders) { - this.transportHeaders = transportHeaders; - } + private void handleError(String msg) { + logger.error(msg); + throw new RuntimeException(msg); + } /** * Create the initial message context for the file @@ -177,7 +144,6 @@ private org.apache.synapse.MessageContext createMessageContext() { .getAxis2MessageContext(); axis2MsgCtx.setServerSide(true); axis2MsgCtx.setMessageID(UUIDGenerator.getUUID()); - axis2MsgCtx.setProperty(MessageContext.TRANSPORT_HEADERS, transportHeaders); msgCtx.setProperty(MessageContext.CLIENT_API_NON_BLOCKING, true); return msgCtx; } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java index dbd56d18f2..906982c1ac 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCPollingConsumer.java @@ -18,15 +18,17 @@ package org.wso2.carbon.inbound.endpoint.protocol.cdc; import io.debezium.engine.ChangeEvent; +import io.debezium.engine.DebeziumEngine; +import io.debezium.engine.format.Json; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.synapse.core.SynapseEnvironment; +import java.io.IOException; import java.util.Date; import java.util.Properties; -import java.util.concurrent.BlockingQueue; - -import static org.wso2.carbon.inbound.endpoint.protocol.cdc.CDCProcessor.inboundEpEventQueueMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** * This class implement the processing logic related to inbound CDC protocol. @@ -42,6 +44,8 @@ public class CDCPollingConsumer { private long scanInterval; private Long lastRanTime; private CDCInjectHandler injectHandler; + private ExecutorService executorService = null; + private DebeziumEngine> engine = null; public CDCPollingConsumer(Properties cdcProperties, String inboundEndpointName, SynapseEnvironment synapseEnvironment, long scanInterval) { @@ -68,25 +72,21 @@ public void registerHandler(CDCInjectHandler injectHandler) { * interval. Timestamp based check is done to avoid that. */ public void execute() { - try { - if (logger.isDebugEnabled()) { - logger.debug("Start : CDC Inbound EP : " + inboundEndpointName); - } - // Check if the cycles are running in correct interval and start - // scan - long currentTime = (new Date()).getTime(); - if (lastRanTime == null || ((lastRanTime + (scanInterval)) <= currentTime)) { - lastRanTime = currentTime; - poll(); - } else if (logger.isDebugEnabled()) { - logger.debug( - "Skip cycle since cuncurrent rate is higher than the scan interval : CDC Inbound EP : " + inboundEndpointName); - } - if (logger.isDebugEnabled()) { - logger.debug("End : CDC Inbound EP : " + inboundEndpointName); - } - } catch (Exception e) { - logger.error("Error while getting events. " + e.getMessage(), e); + if (logger.isDebugEnabled()) { + logger.debug("Start : CDC Inbound EP : " + inboundEndpointName); + } + // Check if the cycles are running in correct interval and start + // scan + long currentTime = (new Date()).getTime(); + if (lastRanTime == null || ((lastRanTime + (scanInterval)) <= currentTime)) { + lastRanTime = currentTime; + poll(); + } else if (logger.isDebugEnabled()) { + logger.debug( + "Skip cycle since concurrent rate is higher than the scan interval : CDC Inbound EP : " + inboundEndpointName); + } + if (logger.isDebugEnabled()) { + logger.debug("End : CDC Inbound EP : " + inboundEndpointName); } } @@ -95,24 +95,41 @@ public void execute() { * according to the registered handler */ public ChangeEvent poll() { + logger.debug("Start : listening to DB events : "); + listenDataChanges(); + logger.debug("End : Listening to DB events : "); + return null; + } - if (logger.isDebugEnabled()) { - logger.debug("Start : listening to DB events : "); - } + private void listenDataChanges () { + executorService = Executors.newSingleThreadExecutor(); - BlockingQueue> eventQueue = inboundEpEventQueueMap.get(inboundEndpointName); - while (!eventQueue.isEmpty()) { - injectHandler.invoke(eventQueue.poll(), inboundEndpointName); - } + if (engine == null || executorService.isShutdown()) { + engine = DebeziumEngine.create(Json.class) + .using(this.cdcProperties) + .notifying(record -> { + injectHandler.invoke(record, this.inboundEndpointName); + }).build(); - if (logger.isDebugEnabled()) { - logger.debug("End : Listening to DB events : "); + executorService.execute(engine); } - return null; } protected Properties getInboundProperties() { return cdcProperties; } + protected void destroy () { + if (!executorService.isShutdown()) { + executorService.shutdown(); + } + try { + if (engine != null) { + engine.close(); + } + } catch (IOException e) { + throw new RuntimeException("Error while closing the Debezium Engine", e); + } + } + } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java index f5353e2acd..2219fbd80f 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java @@ -18,34 +18,27 @@ package org.wso2.carbon.inbound.endpoint.protocol.cdc; -import io.debezium.engine.ChangeEvent; -import io.debezium.engine.DebeziumEngine; -import io.debezium.engine.format.Json; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.synapse.MessageContext; import org.apache.synapse.SynapseException; import org.apache.synapse.core.SynapseEnvironment; import org.apache.synapse.inbound.InboundProcessorParams; import org.apache.synapse.inbound.InboundTaskProcessor; -import org.apache.synapse.mediators.Value; import org.apache.synapse.task.TaskStartupObserver; -import org.apache.synapse.util.xpath.SynapseXPath; -import org.jaxen.JaxenException; +import org.apache.synapse.util.resolver.SecureVaultResolver; import org.wso2.carbon.inbound.endpoint.common.InboundRequestProcessorImpl; import org.wso2.carbon.inbound.endpoint.common.InboundTask; import org.wso2.carbon.inbound.endpoint.protocol.PollingConstants; import java.io.File; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; +import java.util.List; import java.util.Properties; -import java.util.concurrent.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_ALLOWED_OPERATIONS; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_DATABASE_PASSWORD; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_KEY_CONVERTER; @@ -58,6 +51,7 @@ import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_TOPIC_PREFIX; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE; +import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.DEBEZIUM_SKIPPED_OPERATIONS; import static org.wso2.carbon.inbound.endpoint.protocol.cdc.InboundCDCConstants.TRUE; public class CDCProcessor extends InboundRequestProcessorImpl implements TaskStartupObserver, InboundTaskProcessor { @@ -67,15 +61,13 @@ public class CDCProcessor extends InboundRequestProcessorImpl implements TaskSta private String injectingSeq; private String onErrorSeq; private boolean sequential; - - private static final String SECURE_VAULT_REGEX = "(wso2:vault-lookup\\('(.*?)'\\))"; - private static Pattern vaultLookupPattern = Pattern.compile(SECURE_VAULT_REGEX); private static final String ENDPOINT_POSTFIX = "CDC" + COMMON_ENDPOINT_POSTFIX; private static final String FILE_OFFSET_STORAGE_CLASS = "org.apache.kafka.connect.storage.FileOffsetBackingStore"; private static final String FILE_SCHEMA_HISTORY_STORAGE_CLASS = "io.debezium.storage.file.history.FileSchemaHistory"; private static final Log LOGGER = LogFactory.getLog(CDCProcessor.class); - protected static Map inboundEpEventQueueMap = new HashMap(); - private ExecutorService executorService = null; + + private enum operations {create, update, delete, truncate}; + private enum opCodes {c, u, d, t}; public CDCProcessor(InboundProcessorParams params) { this.name = params.getName(); @@ -88,8 +80,6 @@ public CDCProcessor(InboundProcessorParams params) { this.interval = Long.parseLong(cdcProperties.getProperty(PollingConstants.INBOUND_ENDPOINT_INTERVAL)); } catch (NumberFormatException nfe) { throw new SynapseException("Invalid numeric value for interval.", nfe); - } catch (Exception e) { - throw new SynapseException("Invalid value for interval.", e); } this.sequential = true; if (cdcProperties.getProperty(PollingConstants.INBOUND_ENDPOINT_SEQUENTIAL) != null) { @@ -100,10 +90,6 @@ public CDCProcessor(InboundProcessorParams params) { if (cdcProperties.getProperty(PollingConstants.INBOUND_COORDINATION) != null) { this.coordination = Boolean.parseBoolean(cdcProperties.getProperty(PollingConstants.INBOUND_COORDINATION)); } - if (!inboundEpEventQueueMap.containsKey(this.name)) { - BlockingQueue> eventQueue = new LinkedBlockingQueue<>(); - inboundEpEventQueueMap.put(this.name, eventQueue); - } } private void setProperties () { @@ -129,30 +115,27 @@ private void setProperties () { String passwordString = this.cdcProperties.getProperty(DEBEZIUM_DATABASE_PASSWORD); SynapseEnvironment synapseEnvironment = this.synapseEnvironment; - MessageContext messageContext = synapseEnvironment.createMessageContext(); - this.cdcProperties.setProperty(DEBEZIUM_DATABASE_PASSWORD, resolveSecureVault(messageContext, passwordString)); + this.cdcProperties.setProperty(DEBEZIUM_DATABASE_PASSWORD, SecureVaultResolver.resolve(synapseEnvironment, passwordString)); if (this.cdcProperties.getProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL) == null) { this.cdcProperties.setProperty(DEBEZIUM_DATABASE_ALLOW_PUBLIC_KEY_RETRIEVAL, TRUE); } + if (this.cdcProperties.getProperty(DEBEZIUM_ALLOWED_OPERATIONS) != null) { + this.cdcProperties.setProperty(DEBEZIUM_SKIPPED_OPERATIONS, + getSkippedOperationsString(this.cdcProperties.getProperty(DEBEZIUM_ALLOWED_OPERATIONS))); + } + if (this.cdcProperties.getProperty(DEBEZIUM_TOPIC_PREFIX) == null) { this.cdcProperties.setProperty(DEBEZIUM_TOPIC_PREFIX, this.name +"_topic"); } - if (this.cdcProperties.getProperty(DEBEZIUM_VALUE_CONVERTER) == null) { - this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); - } - if (this.cdcProperties.getProperty(DEBEZIUM_KEY_CONVERTER) == null) { - this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); - } - if (this.cdcProperties.getProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE) == null) { - this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE, TRUE); - } - if (this.cdcProperties.getProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE) == null) { - this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE, TRUE); - } + // set the output format as json in a way a user cannot override + this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER, "org.apache.kafka.connect.json.JsonConverter"); + this.cdcProperties.setProperty(DEBEZIUM_KEY_CONVERTER_SCHEMAS_ENABLE, TRUE); + this.cdcProperties.setProperty(DEBEZIUM_VALUE_CONVERTER_SCHEMAS_ENABLE, TRUE); if (this.cdcProperties.getProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL) == null) { this.cdcProperties.setProperty(DEBEZIUM_SCHEMA_HISTORY_INTERNAL, FILE_SCHEMA_HISTORY_STORAGE_CLASS); @@ -170,10 +153,9 @@ private void setProperties () { } } catch (IOException e) { - throw new RuntimeException(e); - } catch (NullPointerException e) { - LOGGER.error("A required property value is not defined", e); - throw new RuntimeException(e); + String msg = "Error While creating the SCHEMAHISTORY or OFFSET storage file"; + LOGGER.error(msg); + throw new RuntimeException(msg, e); } } @@ -185,32 +167,6 @@ private void createFile (String filePath) throws IOException { } } - private static synchronized String resolveSecureVault(MessageContext messageContext, String passwordString) { - if (passwordString == null) { - return null; - } - Matcher lookupMatcher = vaultLookupPattern.matcher(passwordString); - String resolvedValue = ""; - if (lookupMatcher.find()) { - Value expression; - String expressionStr = lookupMatcher.group(1); - try { - expression = new Value(new SynapseXPath(expressionStr)); - - } catch (JaxenException e) { - throw new SynapseException("Error while building the expression : " + expressionStr, e); - } - resolvedValue = expression.evaluateValue(messageContext); - if (StringUtils.isEmpty(resolvedValue)) { - LOGGER.warn("Found Empty value for expression : " + expression.getExpression()); - resolvedValue = ""; - } - } else { - resolvedValue = passwordString; - } - return resolvedValue; - } - /** * This will be called at the time of synapse artifact deployment. @@ -220,19 +176,6 @@ public void init() { pollingConsumer = new CDCPollingConsumer(cdcProperties, name, synapseEnvironment, interval); pollingConsumer.registerHandler(new CDCInjectHandler(injectingSeq, onErrorSeq, sequential, synapseEnvironment, cdcProperties)); - - DebeziumEngine> engine = DebeziumEngine.create(Json.class) - .using(this.cdcProperties) - .notifying(record -> { - try { - inboundEpEventQueueMap.get(this.name).offer(record, interval, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - }).build(); - - executorService = Executors.newSingleThreadExecutor(); - executorService.execute(engine); start(); } @@ -254,7 +197,7 @@ public void start() { public void destroy(boolean removeTask) { if (removeTask) { destroy(); - executorService.shutdown(); + pollingConsumer.destroy(); } } @@ -262,4 +205,34 @@ public void destroy(boolean removeTask) { public void update() { // This will not be called for inbound endpoints } + + private String getOpCode(String op) { + if (op != null) { + switch (operations.valueOf(op)) { + case create: + return opCodes.c.toString(); + case update: + return opCodes.u.toString(); + case delete: + return opCodes.d.toString(); + case truncate: + return opCodes.t.toString(); + } + } + return ""; + } + + /** + * Get the comma separated list containing allowed operations and returns the string of skipped operation codes + * @param allowedOperationsString string + * @return the coma separated string of skipped operation codes + */ + private String getSkippedOperationsString(String allowedOperationsString) { + List allOperations = Stream.of(opCodes.values()).map(Enum :: toString).collect(Collectors.toList()); + Set allowedOperationsSet = Stream.of(allowedOperationsString.split(",")). + map(String :: trim).map(String :: toLowerCase).map(op -> getOpCode(op)). + collect(Collectors.toSet()); + allOperations.removeAll(allowedOperationsSet); + return String.join(",", allOperations); + } } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java index 4795fab44a..a81d0b6fa9 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCTask.java @@ -42,9 +42,7 @@ public CDCTask(CDCPollingConsumer pollingConsumer, long interval) { } protected void taskExecute() { - if (logger.isDebugEnabled()) { - logger.debug("CDC Task executing."); - } + logger.debug("CDC Task executing."); pollingConsumer.execute(); } @@ -54,14 +52,10 @@ public Properties getInboundProperties() { } public void init(SynapseEnvironment synapseEnvironment) { - if (logger.isDebugEnabled()) { - logger.debug("Initializing Task."); - } + logger.debug("Initializing Task."); } public void destroy() { - if (logger.isDebugEnabled()) { - logger.debug("Destroying Task. "); - } + logger.debug("Destroying Task. "); } } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java index c23931d9b2..386cb7bae7 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/InboundCDCConstants.java @@ -36,11 +36,15 @@ class InboundCDCConstants { public static final String DEBEZIUM_SCHEMA_HISTORY_INTERNAL = "schema.history.internal"; public static final String DEBEZIUM_SCHEMA_HISTORY_INTERNAL_FILE_FILENAME = "schema.history.internal.file.filename"; + public static final String DEBEZIUM_SKIPPED_OPERATIONS = "skipped.operations"; + public static final String DEBEZIUM_ALLOWED_OPERATIONS = "allowed.operations"; + /** Output Properties **/ - public static final String DATABASE_NAME = "database"; - public static final String TABLES ="tables"; - public static final String OPERATIONS ="operations"; + public static final String CDC_DATABASE_NAME = "cdc.database"; + public static final String CDC_TABLES ="cdc.tables"; + public static final String CDC_OPERATIONS ="cdc.operations"; + public static final String CDC_TS_MS = "cdc.ts_ms"; public static final String TS_MS = "ts_ms"; public static final String BEFORE = "before"; diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml index e3caf7bc88..606e5c8876 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc-inbound-endpoint.xml @@ -1,19 +1,22 @@ - 9091 + 1000 engine + initial org.apache.kafka.connect.storage.FileOffsetBackingStore - + cdc/offsetStorage/offsets1_.dat io.debezium.connector.mysql.MySqlConnector localhost 3306 root - rusiri@wso2 + {wso2:vault-lookup('mysql_password')} students - 85744 + 8574444 + server_1 + topic2 io.debezium.storage.file.history.FileSchemaHistory - + cdc/schemaHistory/schema_history1_.dat students.marks - c + create - \ No newline at end of file + diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml index 16bab87ac4..982d9998d9 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/test/resources/CDCInbound/cdc_process_seq.xml @@ -2,15 +2,15 @@ - + - + - + - + diff --git a/components/mediation/inbound-endpoints/pom.xml b/components/mediation/inbound-endpoints/pom.xml index 66833fbb3c..5ce4036683 100644 --- a/components/mediation/inbound-endpoints/pom.xml +++ b/components/mediation/inbound-endpoints/pom.xml @@ -22,7 +22,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/mediators/cache-mediator/org.wso2.carbon.mediator.cache/pom.xml b/components/mediation/mediators/cache-mediator/org.wso2.carbon.mediator.cache/pom.xml index 4e16af2941..cdc613863b 100644 --- a/components/mediation/mediators/cache-mediator/org.wso2.carbon.mediator.cache/pom.xml +++ b/components/mediation/mediators/cache-mediator/org.wso2.carbon.mediator.cache/pom.xml @@ -19,7 +19,7 @@ mediators org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/components/mediation/mediators/dataservices-mediator/org.wso2.micro.integrator.mediator.dataservice/pom.xml b/components/mediation/mediators/dataservices-mediator/org.wso2.micro.integrator.mediator.dataservice/pom.xml index dff14d9ad2..efb477fd35 100644 --- a/components/mediation/mediators/dataservices-mediator/org.wso2.micro.integrator.mediator.dataservice/pom.xml +++ b/components/mediation/mediators/dataservices-mediator/org.wso2.micro.integrator.mediator.dataservice/pom.xml @@ -19,7 +19,7 @@ mediators org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/components/mediation/mediators/entitlement-mediator/org.wso2.micro.integrator.identity.entitlement.mediator/pom.xml b/components/mediation/mediators/entitlement-mediator/org.wso2.micro.integrator.identity.entitlement.mediator/pom.xml index 5394cf59e8..7a35d3c5e7 100644 --- a/components/mediation/mediators/entitlement-mediator/org.wso2.micro.integrator.identity.entitlement.mediator/pom.xml +++ b/components/mediation/mediators/entitlement-mediator/org.wso2.micro.integrator.identity.entitlement.mediator/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mediators - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/components/mediation/mediators/oauth-mediator/org.wso2.micro.integrator.mediator.oauth/pom.xml b/components/mediation/mediators/oauth-mediator/org.wso2.micro.integrator.mediator.oauth/pom.xml index 81d0947c06..2009bf2ad2 100644 --- a/components/mediation/mediators/oauth-mediator/org.wso2.micro.integrator.mediator.oauth/pom.xml +++ b/components/mediation/mediators/oauth-mediator/org.wso2.micro.integrator.mediator.oauth/pom.xml @@ -18,7 +18,7 @@ mediators org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/components/mediation/mediators/pom.xml b/components/mediation/mediators/pom.xml index 59ac91f643..4c2eca448b 100644 --- a/components/mediation/mediators/pom.xml +++ b/components/mediation/mediators/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/pom.xml b/components/mediation/pom.xml index 7106440b85..e1b09e5e05 100644 --- a/components/mediation/pom.xml +++ b/components/mediation/pom.xml @@ -21,7 +21,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/registry/org.wso2.micro.integrator.registry/pom.xml b/components/mediation/registry/org.wso2.micro.integrator.registry/pom.xml index 518dda038c..20666dc9ee 100644 --- a/components/mediation/registry/org.wso2.micro.integrator.registry/pom.xml +++ b/components/mediation/registry/org.wso2.micro.integrator.registry/pom.xml @@ -21,7 +21,7 @@ registry org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/registry/pom.xml b/components/mediation/registry/pom.xml index 32ad58347c..c4d8540be5 100644 --- a/components/mediation/registry/pom.xml +++ b/components/mediation/registry/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/security/org.wso2.micro.integrator.mediation.security/pom.xml b/components/mediation/security/org.wso2.micro.integrator.mediation.security/pom.xml index b1b8b2d4df..975d584d03 100644 --- a/components/mediation/security/org.wso2.micro.integrator.mediation.security/pom.xml +++ b/components/mediation/security/org.wso2.micro.integrator.mediation.security/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei security - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/components/mediation/security/pom.xml b/components/mediation/security/pom.xml index 1aec46d0fe..83fed6869e 100644 --- a/components/mediation/security/pom.xml +++ b/components/mediation/security/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/components/mediation/startup/mediation-startup/pom.xml b/components/mediation/startup/mediation-startup/pom.xml index e28101052e..f1c45d8b02 100644 --- a/components/mediation/startup/mediation-startup/pom.xml +++ b/components/mediation/startup/mediation-startup/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei startup - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/startup/pom.xml b/components/mediation/startup/pom.xml index cc09b17595..c8395abe36 100644 --- a/components/mediation/startup/pom.xml +++ b/components/mediation/startup/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/tasks/org.wso2.micro.integrator.mediation.ntask/pom.xml b/components/mediation/tasks/org.wso2.micro.integrator.mediation.ntask/pom.xml index a83fbc44fe..1444c7a9f4 100644 --- a/components/mediation/tasks/org.wso2.micro.integrator.mediation.ntask/pom.xml +++ b/components/mediation/tasks/org.wso2.micro.integrator.mediation.ntask/pom.xml @@ -21,7 +21,7 @@ tasks org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/tasks/org.wso2.micro.integrator.ntask.core/pom.xml b/components/mediation/tasks/org.wso2.micro.integrator.ntask.core/pom.xml index 5f076c79d6..caa1ea1f69 100644 --- a/components/mediation/tasks/org.wso2.micro.integrator.ntask.core/pom.xml +++ b/components/mediation/tasks/org.wso2.micro.integrator.ntask.core/pom.xml @@ -21,7 +21,7 @@ tasks org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/tasks/pom.xml b/components/mediation/tasks/pom.xml index 0453ce0ecd..103f572b9f 100644 --- a/components/mediation/tasks/pom.xml +++ b/components/mediation/tasks/pom.xml @@ -22,7 +22,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/transports/org.wso2.micro.integrator.websocket.transport/pom.xml b/components/mediation/transports/org.wso2.micro.integrator.websocket.transport/pom.xml index 3c07ca7f05..5c49872413 100644 --- a/components/mediation/transports/org.wso2.micro.integrator.websocket.transport/pom.xml +++ b/components/mediation/transports/org.wso2.micro.integrator.websocket.transport/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei transports - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/mediation/transports/pom.xml b/components/mediation/transports/pom.xml index ff4e8043fd..b146732d53 100644 --- a/components/mediation/transports/pom.xml +++ b/components/mediation/transports/pom.xml @@ -21,7 +21,7 @@ mediation org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.bootstrap/pom.xml b/components/org.wso2.micro.integrator.bootstrap/pom.xml index 832ce0bfbe..1a44dd26f9 100644 --- a/components/org.wso2.micro.integrator.bootstrap/pom.xml +++ b/components/org.wso2.micro.integrator.bootstrap/pom.xml @@ -21,7 +21,7 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.coordination/pom.xml b/components/org.wso2.micro.integrator.coordination/pom.xml index f20eaa0e85..50a65e072c 100644 --- a/components/org.wso2.micro.integrator.coordination/pom.xml +++ b/components/org.wso2.micro.integrator.coordination/pom.xml @@ -20,14 +20,14 @@ mi-component-parent org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml 4.0.0 org.wso2.micro.integrator.coordination bundle - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT org.wso2.ei diff --git a/components/org.wso2.micro.integrator.core/pom.xml b/components/org.wso2.micro.integrator.core/pom.xml index 037e10722d..28b06c6eb5 100644 --- a/components/org.wso2.micro.integrator.core/pom.xml +++ b/components/org.wso2.micro.integrator.core/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/pom.xml b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/pom.xml index 5173d84d66..796ec2f9dc 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/pom.xml +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.management.apis/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei org.wso2.micro.integrator.extensions - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.probes/pom.xml b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.probes/pom.xml index 286307adf9..4f6b438c97 100644 --- a/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.probes/pom.xml +++ b/components/org.wso2.micro.integrator.extensions/org.wso2.micro.integrator.probes/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei org.wso2.micro.integrator.extensions - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.extensions/pom.xml b/components/org.wso2.micro.integrator.extensions/pom.xml index fb0db4285d..5423075175 100644 --- a/components/org.wso2.micro.integrator.extensions/pom.xml +++ b/components/org.wso2.micro.integrator.extensions/pom.xml @@ -23,7 +23,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.initializer/pom.xml b/components/org.wso2.micro.integrator.initializer/pom.xml index 56150bcc78..55c56f25b6 100755 --- a/components/org.wso2.micro.integrator.initializer/pom.xml +++ b/components/org.wso2.micro.integrator.initializer/pom.xml @@ -24,7 +24,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.log4j2.plugins/pom.xml b/components/org.wso2.micro.integrator.log4j2.plugins/pom.xml index 10a697aab5..517b5a895d 100644 --- a/components/org.wso2.micro.integrator.log4j2.plugins/pom.xml +++ b/components/org.wso2.micro.integrator.log4j2.plugins/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.logging.updater/pom.xml b/components/org.wso2.micro.integrator.logging.updater/pom.xml index 8c58da3ea4..bcb41a3603 100644 --- a/components/org.wso2.micro.integrator.logging.updater/pom.xml +++ b/components/org.wso2.micro.integrator.logging.updater/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.ndatasource.capp.deployer/pom.xml b/components/org.wso2.micro.integrator.ndatasource.capp.deployer/pom.xml index 5439d19261..9ae38e9def 100644 --- a/components/org.wso2.micro.integrator.ndatasource.capp.deployer/pom.xml +++ b/components/org.wso2.micro.integrator.ndatasource.capp.deployer/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.ndatasource.common/pom.xml b/components/org.wso2.micro.integrator.ndatasource.common/pom.xml index 5dc73c3e64..9a1c25c982 100644 --- a/components/org.wso2.micro.integrator.ndatasource.common/pom.xml +++ b/components/org.wso2.micro.integrator.ndatasource.common/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.ndatasource.core/pom.xml b/components/org.wso2.micro.integrator.ndatasource.core/pom.xml index 4c96487cb4..5d0e141e5f 100644 --- a/components/org.wso2.micro.integrator.ndatasource.core/pom.xml +++ b/components/org.wso2.micro.integrator.ndatasource.core/pom.xml @@ -19,7 +19,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.ndatasource.rdbms/pom.xml b/components/org.wso2.micro.integrator.ndatasource.rdbms/pom.xml index 35a65f4a7e..5b369f3f0f 100644 --- a/components/org.wso2.micro.integrator.ndatasource.rdbms/pom.xml +++ b/components/org.wso2.micro.integrator.ndatasource.rdbms/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.security/pom.xml b/components/org.wso2.micro.integrator.security/pom.xml index f0176c0e5d..a196f2a3d8 100644 --- a/components/org.wso2.micro.integrator.security/pom.xml +++ b/components/org.wso2.micro.integrator.security/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.server/pom.xml b/components/org.wso2.micro.integrator.server/pom.xml index 7df41940ec..df77746ccd 100644 --- a/components/org.wso2.micro.integrator.server/pom.xml +++ b/components/org.wso2.micro.integrator.server/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/org.wso2.micro.integrator.utils/pom.xml b/components/org.wso2.micro.integrator.utils/pom.xml index 7f0de07457..c245e25e1c 100644 --- a/components/org.wso2.micro.integrator.utils/pom.xml +++ b/components/org.wso2.micro.integrator.utils/pom.xml @@ -19,7 +19,7 @@ org.wso2.ei mi-component-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/components/pom.xml b/components/pom.xml index 4e79757887..8f9f7eaabb 100644 --- a/components/pom.xml +++ b/components/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei wso2-micro-integrator-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/distribution/pom.xml b/distribution/pom.xml index c98a375b15..411a80bbf0 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei wso2-micro-integrator-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/crypto-feature/org.wso2.micro.integrator.crypto.feature/pom.xml b/features/crypto-feature/org.wso2.micro.integrator.crypto.feature/pom.xml index ee8d87b051..7817c6489c 100644 --- a/features/crypto-feature/org.wso2.micro.integrator.crypto.feature/pom.xml +++ b/features/crypto-feature/org.wso2.micro.integrator.crypto.feature/pom.xml @@ -21,7 +21,7 @@ micro-integrator-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/features/data-features/data-services-feature/org.wso2.micro.integrator.dataservices.server.feature/pom.xml b/features/data-features/data-services-feature/org.wso2.micro.integrator.dataservices.server.feature/pom.xml index 80063f6e62..906e496f7e 100644 --- a/features/data-features/data-services-feature/org.wso2.micro.integrator.dataservices.server.feature/pom.xml +++ b/features/data-features/data-services-feature/org.wso2.micro.integrator.dataservices.server.feature/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei data-services-feature - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/data-features/data-services-feature/pom.xml b/features/data-features/data-services-feature/pom.xml index a6e04fa737..d73f25cd47 100644 --- a/features/data-features/data-services-feature/pom.xml +++ b/features/data-features/data-services-feature/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei data-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/data-features/pom.xml b/features/data-features/pom.xml index e99acc9081..78d0fac8fe 100644 --- a/features/data-features/pom.xml +++ b/features/data-features/pom.xml @@ -21,7 +21,7 @@ micro-integrator-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/builtin-mediators/org.wso2.micro.integrator.mediators.server.feature/pom.xml b/features/mediation-features/builtin-mediators/org.wso2.micro.integrator.mediators.server.feature/pom.xml index 9d8673499b..179f499a89 100644 --- a/features/mediation-features/builtin-mediators/org.wso2.micro.integrator.mediators.server.feature/pom.xml +++ b/features/mediation-features/builtin-mediators/org.wso2.micro.integrator.mediators.server.feature/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei builtin-mediators - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/builtin-mediators/pom.xml b/features/mediation-features/builtin-mediators/pom.xml index b988d2485a..8083b0e4d9 100644 --- a/features/mediation-features/builtin-mediators/pom.xml +++ b/features/mediation-features/builtin-mediators/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.analytics.messageflow.data.publisher.feature/pom.xml b/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.analytics.messageflow.data.publisher.feature/pom.xml index 2604162d08..411119de59 100644 --- a/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.analytics.messageflow.data.publisher.feature/pom.xml +++ b/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.analytics.messageflow.data.publisher.feature/pom.xml @@ -22,7 +22,7 @@ data-publisher-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.observability.feature/pom.xml b/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.observability.feature/pom.xml index d86b3960cb..bb7e55a1c7 100644 --- a/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.observability.feature/pom.xml +++ b/features/mediation-features/data-publisher-features/org.wso2.micro.integrator.observability.feature/pom.xml @@ -21,7 +21,7 @@ data-publisher-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/data-publisher-features/pom.xml b/features/mediation-features/data-publisher-features/pom.xml index 7258167c63..79ef4f8d1e 100644 --- a/features/mediation-features/data-publisher-features/pom.xml +++ b/features/mediation-features/data-publisher-features/pom.xml @@ -21,7 +21,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/dataservice-mediator-feature/org.wso2.micro.integrator.mediator.dataservice/pom.xml b/features/mediation-features/dataservice-mediator-feature/org.wso2.micro.integrator.mediator.dataservice/pom.xml index 0c788efecb..c59c4179bf 100644 --- a/features/mediation-features/dataservice-mediator-feature/org.wso2.micro.integrator.mediator.dataservice/pom.xml +++ b/features/mediation-features/dataservice-mediator-feature/org.wso2.micro.integrator.mediator.dataservice/pom.xml @@ -20,7 +20,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/features/mediation-features/entitlement-mediator-feature/org.wso2.micro.integrator.identity.xacml.mediator/pom.xml b/features/mediation-features/entitlement-mediator-feature/org.wso2.micro.integrator.identity.xacml.mediator/pom.xml index 2a1af54e87..5c865654a0 100644 --- a/features/mediation-features/entitlement-mediator-feature/org.wso2.micro.integrator.identity.xacml.mediator/pom.xml +++ b/features/mediation-features/entitlement-mediator-feature/org.wso2.micro.integrator.identity.xacml.mediator/pom.xml @@ -20,7 +20,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/features/mediation-features/inbound-endpoint/org.wso2.micro.integrator.inbound.endpoints.server.feature/pom.xml b/features/mediation-features/inbound-endpoint/org.wso2.micro.integrator.inbound.endpoints.server.feature/pom.xml index 3cd44e86e8..8e22c3533a 100644 --- a/features/mediation-features/inbound-endpoint/org.wso2.micro.integrator.inbound.endpoints.server.feature/pom.xml +++ b/features/mediation-features/inbound-endpoint/org.wso2.micro.integrator.inbound.endpoints.server.feature/pom.xml @@ -22,7 +22,7 @@ inbound-endpoint org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/inbound-endpoint/pom.xml b/features/mediation-features/inbound-endpoint/pom.xml index 19625648bf..50975f771a 100644 --- a/features/mediation-features/inbound-endpoint/pom.xml +++ b/features/mediation-features/inbound-endpoint/pom.xml @@ -21,7 +21,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/ntask-feature/pom.xml b/features/mediation-features/ntask-feature/pom.xml index 13f2fd644f..959c45d7aa 100644 --- a/features/mediation-features/ntask-feature/pom.xml +++ b/features/mediation-features/ntask-feature/pom.xml @@ -21,7 +21,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/pom.xml b/features/mediation-features/pom.xml index e09b63d26c..6d38fe1b47 100644 --- a/features/mediation-features/pom.xml +++ b/features/mediation-features/pom.xml @@ -21,7 +21,7 @@ micro-integrator-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/security-features/org.wso2.micro.integrator.security.server.feature/pom.xml b/features/mediation-features/security-features/org.wso2.micro.integrator.security.server.feature/pom.xml index 0db9cea038..173d908fb0 100644 --- a/features/mediation-features/security-features/org.wso2.micro.integrator.security.server.feature/pom.xml +++ b/features/mediation-features/security-features/org.wso2.micro.integrator.security.server.feature/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei security-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/security-features/pom.xml b/features/mediation-features/security-features/pom.xml index e6cb585676..add4a4717b 100644 --- a/features/mediation-features/security-features/pom.xml +++ b/features/mediation-features/security-features/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mediation-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/transport-features/org.wso2.micro.integrator.websocket.feature/pom.xml b/features/mediation-features/transport-features/org.wso2.micro.integrator.websocket.feature/pom.xml index 2f2abf0081..4063dc455f 100644 --- a/features/mediation-features/transport-features/org.wso2.micro.integrator.websocket.feature/pom.xml +++ b/features/mediation-features/transport-features/org.wso2.micro.integrator.websocket.feature/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei transport-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/mediation-features/transport-features/pom.xml b/features/mediation-features/transport-features/pom.xml index 65fd33e73b..93697a9c02 100644 --- a/features/mediation-features/transport-features/pom.xml +++ b/features/mediation-features/transport-features/pom.xml @@ -21,7 +21,7 @@ mediation-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/features/org.wso2.micro.integrator.initializer.feature/pom.xml b/features/org.wso2.micro.integrator.initializer.feature/pom.xml index 81946ea1fc..4098e2f2d2 100644 --- a/features/org.wso2.micro.integrator.initializer.feature/pom.xml +++ b/features/org.wso2.micro.integrator.initializer.feature/pom.xml @@ -24,7 +24,7 @@ org.wso2.ei micro-integrator-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/org.wso2.micro.integrator.logging.updater.feature/pom.xml b/features/org.wso2.micro.integrator.logging.updater.feature/pom.xml index c8c7659742..adca9f84eb 100644 --- a/features/org.wso2.micro.integrator.logging.updater.feature/pom.xml +++ b/features/org.wso2.micro.integrator.logging.updater.feature/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei micro-integrator-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/org.wso2.micro.integrator.ntask.core.feature/pom.xml b/features/org.wso2.micro.integrator.ntask.core.feature/pom.xml index 81944ae1a1..b4ca053a8c 100644 --- a/features/org.wso2.micro.integrator.ntask.core.feature/pom.xml +++ b/features/org.wso2.micro.integrator.ntask.core.feature/pom.xml @@ -23,7 +23,7 @@ org.wso2.ei micro-integrator-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/org.wso2.micro.integrator.server.feature/pom.xml b/features/org.wso2.micro.integrator.server.feature/pom.xml index 46855f9ed7..502e0d720d 100644 --- a/features/org.wso2.micro.integrator.server.feature/pom.xml +++ b/features/org.wso2.micro.integrator.server.feature/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei micro-integrator-features - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/pom.xml b/features/pom.xml index ed4d6e342c..1b7701de8d 100644 --- a/features/pom.xml +++ b/features/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei wso2-micro-integrator-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/features/wso2mi-hl7/pom.xml b/features/wso2mi-hl7/pom.xml index e4953ed90a..f81d709ec6 100644 --- a/features/wso2mi-hl7/pom.xml +++ b/features/wso2mi-hl7/pom.xml @@ -21,7 +21,7 @@ micro-integrator-features org.wso2.ei - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT 4.0.0 diff --git a/integration/automation-extensions/pom.xml b/integration/automation-extensions/pom.xml index f4c18501ef..1d7fc44da2 100644 --- a/integration/automation-extensions/pom.xml +++ b/integration/automation-extensions/pom.xml @@ -23,7 +23,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/pom.xml b/integration/dataservice-hosting-tests/pom.xml index 53e71d7b21..b902e4cb8f 100755 --- a/integration/dataservice-hosting-tests/pom.xml +++ b/integration/dataservice-hosting-tests/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/samples/data-services/clients/pom.xml b/integration/dataservice-hosting-tests/samples/data-services/clients/pom.xml index bc5cd14f59..8f1525b1ac 100755 --- a/integration/dataservice-hosting-tests/samples/data-services/clients/pom.xml +++ b/integration/dataservice-hosting-tests/samples/data-services/clients/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei dataservices-samples-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/samples/data-services/pom.xml b/integration/dataservice-hosting-tests/samples/data-services/pom.xml index d8a0f2c12f..00b06d17b8 100755 --- a/integration/dataservice-hosting-tests/samples/data-services/pom.xml +++ b/integration/dataservice-hosting-tests/samples/data-services/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei mi-samples-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/samples/pom.xml b/integration/dataservice-hosting-tests/samples/pom.xml index 0a7f4b11c3..12e4a59f98 100644 --- a/integration/dataservice-hosting-tests/samples/pom.xml +++ b/integration/dataservice-hosting-tests/samples/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei dataservice-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/tests-common/admin-clients/pom.xml b/integration/dataservice-hosting-tests/tests-common/admin-clients/pom.xml index fdcec13e4c..d3073f425c 100644 --- a/integration/dataservice-hosting-tests/tests-common/admin-clients/pom.xml +++ b/integration/dataservice-hosting-tests/tests-common/admin-clients/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei dataservice-integration-tests-common - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/tests-common/integration-test-utils/pom.xml b/integration/dataservice-hosting-tests/tests-common/integration-test-utils/pom.xml index 6d650f17dc..30217d7110 100644 --- a/integration/dataservice-hosting-tests/tests-common/integration-test-utils/pom.xml +++ b/integration/dataservice-hosting-tests/tests-common/integration-test-utils/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei dataservice-integration-tests-common - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/tests-common/pom.xml b/integration/dataservice-hosting-tests/tests-common/pom.xml index 76a9dcecfd..a202590487 100644 --- a/integration/dataservice-hosting-tests/tests-common/pom.xml +++ b/integration/dataservice-hosting-tests/tests-common/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei dataservice-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/tests-integration/pom.xml b/integration/dataservice-hosting-tests/tests-integration/pom.xml index b32dd14b3b..ae3adb7777 100644 --- a/integration/dataservice-hosting-tests/tests-integration/pom.xml +++ b/integration/dataservice-hosting-tests/tests-integration/pom.xml @@ -20,7 +20,7 @@ org.wso2.ei dataservice-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/dataservice-hosting-tests/tests-integration/tests/pom.xml b/integration/dataservice-hosting-tests/tests-integration/tests/pom.xml index cf30ba11d1..fe23e4dfd6 100644 --- a/integration/dataservice-hosting-tests/tests-integration/tests/pom.xml +++ b/integration/dataservice-hosting-tests/tests-integration/tests/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei dataservice-integration-tests - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/management-api-tests/pom.xml b/integration/management-api-tests/pom.xml index 8899308fc5..4113715ef6 100644 --- a/integration/management-api-tests/pom.xml +++ b/integration/management-api-tests/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/coverage-report/pom.xml b/integration/mediation-tests/coverage-report/pom.xml index 7298ac2e8e..f77e6fd99e 100644 --- a/integration/mediation-tests/coverage-report/pom.xml +++ b/integration/mediation-tests/coverage-report/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/pom.xml b/integration/mediation-tests/pom.xml index 90c7489f19..917145041a 100755 --- a/integration/mediation-tests/pom.xml +++ b/integration/mediation-tests/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/service-samples/pom.xml b/integration/mediation-tests/service-samples/pom.xml index 82f8b58244..5548c7c112 100644 --- a/integration/mediation-tests/service-samples/pom.xml +++ b/integration/mediation-tests/service-samples/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-http/pom.xml b/integration/mediation-tests/tests-http/pom.xml index 3d28f17abe..1064a56354 100644 --- a/integration/mediation-tests/tests-http/pom.xml +++ b/integration/mediation-tests/tests-http/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-mediator-1/pom.xml b/integration/mediation-tests/tests-mediator-1/pom.xml index 011c9fd680..929a237385 100644 --- a/integration/mediation-tests/tests-mediator-1/pom.xml +++ b/integration/mediation-tests/tests-mediator-1/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-mediator-2/pom.xml b/integration/mediation-tests/tests-mediator-2/pom.xml index b3c728e6a1..630120bd26 100644 --- a/integration/mediation-tests/tests-mediator-2/pom.xml +++ b/integration/mediation-tests/tests-mediator-2/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-other/pom.xml b/integration/mediation-tests/tests-other/pom.xml index 98813da842..18296c04c3 100644 --- a/integration/mediation-tests/tests-other/pom.xml +++ b/integration/mediation-tests/tests-other/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-patches-with-smb2/pom.xml b/integration/mediation-tests/tests-patches-with-smb2/pom.xml index 6bf367e418..81573eeec3 100644 --- a/integration/mediation-tests/tests-patches-with-smb2/pom.xml +++ b/integration/mediation-tests/tests-patches-with-smb2/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-patches/pom.xml b/integration/mediation-tests/tests-patches/pom.xml index 784a735ac8..8cef21f449 100644 --- a/integration/mediation-tests/tests-patches/pom.xml +++ b/integration/mediation-tests/tests-patches/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-platform/pom.xml b/integration/mediation-tests/tests-platform/pom.xml index 49ebf460de..e30198d2f3 100644 --- a/integration/mediation-tests/tests-platform/pom.xml +++ b/integration/mediation-tests/tests-platform/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-platform/tests-coordination/pom.xml b/integration/mediation-tests/tests-platform/tests-coordination/pom.xml index 27bf436a78..77272c9161 100644 --- a/integration/mediation-tests/tests-platform/tests-coordination/pom.xml +++ b/integration/mediation-tests/tests-platform/tests-coordination/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/integration/mediation-tests/tests-platform/tests-rabbitmq/pom.xml b/integration/mediation-tests/tests-platform/tests-rabbitmq/pom.xml index ef2f503463..2809a29834 100644 --- a/integration/mediation-tests/tests-platform/tests-rabbitmq/pom.xml +++ b/integration/mediation-tests/tests-platform/tests-rabbitmq/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei org.wso2.carbon.ei.tests.platform - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-platform/tests-transaction/pom.xml b/integration/mediation-tests/tests-platform/tests-transaction/pom.xml index 405387b6ec..1beeadb05a 100644 --- a/integration/mediation-tests/tests-platform/tests-transaction/pom.xml +++ b/integration/mediation-tests/tests-platform/tests-transaction/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/integration/mediation-tests/tests-platform/tests-userstore/pom.xml b/integration/mediation-tests/tests-platform/tests-userstore/pom.xml index f05ab2f352..1794ebcaed 100644 --- a/integration/mediation-tests/tests-platform/tests-userstore/pom.xml +++ b/integration/mediation-tests/tests-platform/tests-userstore/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/integration/mediation-tests/tests-sample/pom.xml b/integration/mediation-tests/tests-sample/pom.xml index 52693e1d41..f7b95f465e 100644 --- a/integration/mediation-tests/tests-sample/pom.xml +++ b/integration/mediation-tests/tests-sample/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-service/pom.xml b/integration/mediation-tests/tests-service/pom.xml index bdd177c55e..acfb7cea56 100644 --- a/integration/mediation-tests/tests-service/pom.xml +++ b/integration/mediation-tests/tests-service/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-transport-2/pom.xml b/integration/mediation-tests/tests-transport-2/pom.xml index 3db8fdbda6..0bf1a4de2d 100644 --- a/integration/mediation-tests/tests-transport-2/pom.xml +++ b/integration/mediation-tests/tests-transport-2/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/mediation-tests/tests-transport/pom.xml b/integration/mediation-tests/tests-transport/pom.xml index 1edac05e38..bf35eba37c 100644 --- a/integration/mediation-tests/tests-transport/pom.xml +++ b/integration/mediation-tests/tests-transport/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei mediation-integration - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/pom.xml b/integration/pom.xml index ba17ee36b6..69de98222f 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei wso2-micro-integrator-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/samples/product/pom.xml b/integration/samples/product/pom.xml index b410214473..5b0c180582 100755 --- a/integration/samples/product/pom.xml +++ b/integration/samples/product/pom.xml @@ -22,7 +22,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../../pom.xml diff --git a/integration/tests-common/admin-clients/pom.xml b/integration/tests-common/admin-clients/pom.xml index 4ecd4529ae..859491eb5c 100644 --- a/integration/tests-common/admin-clients/pom.xml +++ b/integration/tests-common/admin-clients/pom.xml @@ -5,7 +5,7 @@ org.wso2.ei integration-test-common - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/tests-common/integration-test-utils/pom.xml b/integration/tests-common/integration-test-utils/pom.xml index b3397cce83..d89a703ca8 100644 --- a/integration/tests-common/integration-test-utils/pom.xml +++ b/integration/tests-common/integration-test-utils/pom.xml @@ -8,7 +8,7 @@ org.wso2.ei integration-test-common - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/integration/tests-common/pom.xml b/integration/tests-common/pom.xml index c786674e20..9d767f963e 100644 --- a/integration/tests-common/pom.xml +++ b/integration/tests-common/pom.xml @@ -4,7 +4,7 @@ org.wso2.ei micro-integrator-integration-test - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/p2-profile/pom.xml b/p2-profile/pom.xml index 9a7f9e383c..89edb8aaec 100644 --- a/p2-profile/pom.xml +++ b/p2-profile/pom.xml @@ -21,7 +21,7 @@ org.wso2.ei wso2-micro-integrator-parent - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 0a87249f76..83c0ef3a5f 100644 --- a/pom.xml +++ b/pom.xml @@ -1104,12 +1104,6 @@ ${mysql.connector.version} test - - com.oracle.database.jdbc - ojdbc8 - ${ojdbc8.version} - test - org.apache.derby.wso2 derby @@ -1268,7 +1262,7 @@ org.wso2.ei org.wso2.micro.integrator.coordination - 4.2.0-SNAPSHOT + 4.3.0-SNAPSHOT @@ -1457,116 +1451,14 @@ ${opencensus.orbit.version} - io.debezium - debezium-embedded - ${debezium.version} - - - io.debezium - debezium-api - ${debezium.version} - - - io.debezium - debezium-connector-mysql - ${debezium.version} - - - io.debezium - debezium-connector-sqlserver - ${debezium.version} - - - io.debezium - debezium-connector-postgres - ${debezium.version} - - - io.debezium - debezium-connector-oracle - ${debezium.version} - - - io.debezium - debezium-core - ${debezium.version} - - - org.slf4j - slf4j-api - - - - - io.debezium - debezium-storage-kafka - ${debezium.version} - - - org.slf4j - slf4j-api - - - - - io.debezium - debezium-storage-file - ${debezium.version} - - - org.slf4j - slf4j-api - - - - - io.debezium - debezium-ddl-parser + org.wso2.orbit.debezium + debezium ${debezium.version} - - - org.slf4j - slf4j-api - - - - - com.github.shyiko - mysql-binlog-connector-java - 0.27.2 - - - org.apache.kafka - kafka-clients - ${kafka.version} - - - org.slf4j - slf4j-api - - - - - org.apache.kafka - connect-runtime - ${kafka.version} - - - org.slf4j - slf4j-api - - - org.apache.kafka - connect-json - ${kafka.version} - - - org.slf4j - slf4j-api - - + com.google.code.gson + gson + ${version.com.google.code.gson} @@ -1663,14 +1555,10 @@ 2.3.0 2.3.1 - - 2.4.1.Final - 21.3.0.0 - - - 3.3.1 + + 2.1.4.Final.wso2v1 - 4.0.0-wso2v40 + 4.0.0-wso2v47 [4.0.0, 4.0.1) 4.7.175 1.1.3 From 2df82c43ecc1390fcd0e337d0cddc5830f870446 Mon Sep 17 00:00:00 2001 From: rusirijayodaillesinghe Date: Wed, 20 Sep 2023 16:47:19 +0530 Subject: [PATCH 4/4] Resolve review comments --- .../inbound/endpoint/protocol/cdc/CDCEventOutput.java | 11 +++-------- .../inbound/endpoint/protocol/cdc/CDCProcessor.java | 10 +++++----- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java index cdb52b0b9d..71e143eeb4 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCEventOutput.java @@ -60,21 +60,16 @@ public Long getTs_ms() { } public String getDatabase() { - if (getSource() != null) { - if (getSource().has(DB)) { + if (getSource() != null && getSource().has(DB)) { return getSource().get(DB).getAsString(); - } - return null; } return null; } public JsonElement getTable() { JsonElement tableObject = null; - if (getSource() != null) { - if (getSource().has(TABLE)) { - tableObject = getSource().get(TABLE); - } + if (getSource() != null && getSource().has(TABLE)) { + tableObject = getSource().get(TABLE); } return tableObject; } diff --git a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java index 2219fbd80f..3563d2f01b 100644 --- a/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java +++ b/components/mediation/inbound-endpoints/org.wso2.micro.integrator.inbound.endpoint/src/main/java/org/wso2/carbon/inbound/endpoint/protocol/cdc/CDCProcessor.java @@ -64,7 +64,7 @@ public class CDCProcessor extends InboundRequestProcessorImpl implements TaskSta private static final String ENDPOINT_POSTFIX = "CDC" + COMMON_ENDPOINT_POSTFIX; private static final String FILE_OFFSET_STORAGE_CLASS = "org.apache.kafka.connect.storage.FileOffsetBackingStore"; private static final String FILE_SCHEMA_HISTORY_STORAGE_CLASS = "io.debezium.storage.file.history.FileSchemaHistory"; - private static final Log LOGGER = LogFactory.getLog(CDCProcessor.class); + private static final Log logger = LogFactory.getLog(CDCProcessor.class); private enum operations {create, update, delete, truncate}; private enum opCodes {c, u, d, t}; @@ -93,7 +93,7 @@ public CDCProcessor(InboundProcessorParams params) { } private void setProperties () { - LOGGER.info("Initializing the properties"); + logger.info("Initializing the CDC properties"); try { if (this.cdcProperties.getProperty(DEBEZIUM_OFFSET_STORAGE) == null) { this.cdcProperties.setProperty(DEBEZIUM_OFFSET_STORAGE, FILE_OFFSET_STORAGE_CLASS); @@ -153,8 +153,8 @@ private void setProperties () { } } catch (IOException e) { - String msg = "Error While creating the SCHEMAHISTORY or OFFSET storage file"; - LOGGER.error(msg); + String msg = "Error while setting the CDC Properties"; + logger.error(msg); throw new RuntimeException(msg, e); } } @@ -172,7 +172,7 @@ private void createFile (String filePath) throws IOException { * This will be called at the time of synapse artifact deployment. */ public void init() { - LOGGER.info("Inbound CDC listener " + name + " starting ..."); + logger.info("Inbound CDC listener " + name + " starting ..."); pollingConsumer = new CDCPollingConsumer(cdcProperties, name, synapseEnvironment, interval); pollingConsumer.registerHandler(new CDCInjectHandler(injectingSeq, onErrorSeq, sequential, synapseEnvironment, cdcProperties));