diff --git a/dal-client/pom.xml b/dal-client/pom.xml index 393e2224b..70faad09c 100644 --- a/dal-client/pom.xml +++ b/dal-client/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.ctrip.platform dal-client - 2.0.3 + 2.0.4 1.8 diff --git a/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/ClusterDatabaseSet.java b/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/ClusterDatabaseSet.java index 91c1911b8..03ae3f717 100644 --- a/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/ClusterDatabaseSet.java +++ b/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/ClusterDatabaseSet.java @@ -19,7 +19,7 @@ /** * @author c7ch23en */ -public class ClusterDatabaseSet implements DatabaseSet { +public class ClusterDatabaseSet extends DatabaseSet { private String databaseSetName; private Cluster cluster; diff --git a/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/DatabaseSet.java b/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/DatabaseSet.java index 27dad08b4..b43e50db2 100644 --- a/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/DatabaseSet.java +++ b/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/DatabaseSet.java @@ -14,36 +14,81 @@ import com.ctrip.platform.dal.exceptions.DalException; import com.ctrip.platform.dal.sharding.idgen.IIdGeneratorConfig; -public interface DatabaseSet { - - String getName(); - - String getProvider(); - - DatabaseCategory getDatabaseCategory(); - - boolean isShardingSupported(); - - boolean isTableShardingSupported(String tableName); - - Map getDatabases(); - - void validate(String shard) throws SQLException; - - Set getAllShards(); - - Set getAllTableShards(String tableName) throws SQLException; - - DalShardingStrategy getStrategy() throws SQLException; - - List getMasterDbs(); - - List getSlaveDbs(); - - List getMasterDbs(String shard); - - List getSlaveDbs(String shard); - - IIdGeneratorConfig getIdGenConfig(); +public abstract class DatabaseSet implements IDatabaseSet { + + @Override + public String getName() { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public String getProvider() { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public DatabaseCategory getDatabaseCategory() { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public boolean isShardingSupported() { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public boolean isTableShardingSupported(String tableName) { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public Map getDatabases() { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public void validate(String shard) throws SQLException { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public Set getAllShards() { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public Set getAllTableShards(String tableName) throws SQLException { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public DalShardingStrategy getStrategy() throws SQLException { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public List getMasterDbs() { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public List getSlaveDbs() { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public List getMasterDbs(String shard) { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public List getSlaveDbs(String shard) { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } + + @Override + public IIdGeneratorConfig getIdGenConfig() { + throw new UnsupportedOperationException("This is an abstract DatabaseSet."); + } } diff --git a/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/DefaultDatabaseSet.java b/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/DefaultDatabaseSet.java index ca5bf6800..a93e0c6f8 100644 --- a/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/DefaultDatabaseSet.java +++ b/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/DefaultDatabaseSet.java @@ -14,7 +14,7 @@ import com.ctrip.platform.dal.exceptions.DalException; import com.ctrip.platform.dal.sharding.idgen.IIdGeneratorConfig; -public class DefaultDatabaseSet implements DatabaseSet { +public class DefaultDatabaseSet extends DatabaseSet { private static final String CLASS = "class"; private static final String ENTRY_SEPARATOR = ";"; private static final String KEY_VALUE_SEPARATOR = "="; diff --git a/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/IDatabaseSet.java b/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/IDatabaseSet.java new file mode 100644 index 000000000..57f95413d --- /dev/null +++ b/dal-client/src/main/java/com/ctrip/platform/dal/dao/configure/IDatabaseSet.java @@ -0,0 +1,44 @@ +package com.ctrip.platform.dal.dao.configure; + +import com.ctrip.platform.dal.common.enums.DatabaseCategory; +import com.ctrip.platform.dal.dao.strategy.DalShardingStrategy; +import com.ctrip.platform.dal.sharding.idgen.IIdGeneratorConfig; + +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface IDatabaseSet { + + String getName(); + + String getProvider(); + + DatabaseCategory getDatabaseCategory(); + + boolean isShardingSupported(); + + boolean isTableShardingSupported(String tableName); + + Map getDatabases(); + + void validate(String shard) throws SQLException; + + Set getAllShards(); + + Set getAllTableShards(String tableName) throws SQLException; + + DalShardingStrategy getStrategy() throws SQLException; + + List getMasterDbs(); + + List getSlaveDbs(); + + List getMasterDbs(String shard); + + List getSlaveDbs(String shard); + + IIdGeneratorConfig getIdGenConfig(); + +} diff --git a/dal-client/src/main/java/com/ctrip/platform/dal/dao/datasource/DataSourceLocator.java b/dal-client/src/main/java/com/ctrip/platform/dal/dao/datasource/DataSourceLocator.java index aaf47310c..f4f487973 100644 --- a/dal-client/src/main/java/com/ctrip/platform/dal/dao/datasource/DataSourceLocator.java +++ b/dal-client/src/main/java/com/ctrip/platform/dal/dao/datasource/DataSourceLocator.java @@ -77,6 +77,7 @@ public DataSource getDataSource(DataSourceIdentity id) { public void removeDataSource(DataSourceIdentity id) { DataSource ds = cache.remove(id); + provider.unregister(id); if (ds instanceof RefreshableDataSource) { ((RefreshableDataSource) ds).close(); } @@ -96,8 +97,7 @@ private DataSource createDataSource(DataSourceIdentity id) throws SQLException { return ds; } - public void setup(Cluster cluster) { - } + public void setup(Cluster cluster) {} public static Map getActiveConnectionNumber() { Map map = new HashMap<>(); diff --git a/dal-client/src/main/java/com/ctrip/platform/dal/dao/task/InsertTaskAdapter.java b/dal-client/src/main/java/com/ctrip/platform/dal/dao/task/InsertTaskAdapter.java index cfac50f16..fb692515f 100644 --- a/dal-client/src/main/java/com/ctrip/platform/dal/dao/task/InsertTaskAdapter.java +++ b/dal-client/src/main/java/com/ctrip/platform/dal/dao/task/InsertTaskAdapter.java @@ -139,7 +139,7 @@ public void processIdentityField(DalHints hints, List> pojos) { if (identityInsertDisabled || null == pojo.get(identityFieldName)) { Number id = idGenerator.nextId(); checkIdentityTypes(identityFieldType, id); - pojo.put(identityFieldName, idGenerator.nextId()); + pojo.put(identityFieldName, id); } } } diff --git a/dal-cluster-client/pom.xml b/dal-cluster-client/pom.xml index ff618a62a..f905119b8 100644 --- a/dal-cluster-client/pom.xml +++ b/dal-cluster-client/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.ctrip.framework.dal dal-cluster-client - 2.0.3 + 2.0.4 1.8 diff --git a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/config/ClusterConfigXMLConstants.java b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/config/ClusterConfigXMLConstants.java index 1a72fefc7..6e80c883d 100644 --- a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/config/ClusterConfigXMLConstants.java +++ b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/config/ClusterConfigXMLConstants.java @@ -13,6 +13,7 @@ public interface ClusterConfigXMLConstants { String DATABASE = "Database"; String SHARD_STRATEGIES = "ShardStrategies"; String MOD_STRATEGY = "ModStrategy"; + String USER_HINT_STRATEGY = "UserHintStrategy"; String CUSTOM_STRATEGY = "CustomStrategy"; String PROPERTY = "Property"; String TABLES = "Tables"; diff --git a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/config/ClusterConfigXMLParser.java b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/config/ClusterConfigXMLParser.java index c2c3829c7..27c649411 100644 --- a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/config/ClusterConfigXMLParser.java +++ b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/config/ClusterConfigXMLParser.java @@ -8,6 +8,7 @@ import com.ctrip.framework.dal.cluster.client.sharding.idgen.ClusterIdGeneratorConfig; import com.ctrip.framework.dal.cluster.client.sharding.strategy.ModShardStrategy; import com.ctrip.framework.dal.cluster.client.sharding.strategy.ShardStrategy; +import com.ctrip.framework.dal.cluster.client.sharding.strategy.UserHintStrategy; import com.ctrip.framework.dal.cluster.client.util.ServiceLoaderUtils; import com.ctrip.framework.dal.cluster.client.util.StringUtils; import org.w3c.dom.Document; @@ -139,6 +140,8 @@ private void setAttributesForDatabase(DatabaseConfigImpl databaseConfig, Node da private void parseShardStrategies(ClusterConfigImpl clusterConfig, Node strategiesNode) { for (Node modStrategyNode : getChildNodes(strategiesNode, MOD_STRATEGY)) parseModStrategy(clusterConfig, modStrategyNode); + for (Node modStrategyNode : getChildNodes(strategiesNode, USER_HINT_STRATEGY)) + parseUserHintStrategy(clusterConfig, modStrategyNode); for (Node customStrategyNode : getChildNodes(strategiesNode, CUSTOM_STRATEGY)) parseCustomStrategy(clusterConfig, customStrategyNode); } @@ -147,6 +150,10 @@ private void parseModStrategy(ClusterConfigImpl clusterConfig, Node strategyNode parseShardStrategy(clusterConfig, strategyNode, new ModShardStrategy()); } + private void parseUserHintStrategy(ClusterConfigImpl clusterConfig, Node strategyNode) { + parseShardStrategy(clusterConfig, strategyNode, new UserHintStrategy()); + } + private void parseCustomStrategy(ClusterConfigImpl clusterConfig, Node strategyNode) { String className = getAttribute(strategyNode, CLASS); try { diff --git a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/BaseShardStrategy.java b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/BaseShardStrategy.java index 5437e6496..6d3163021 100644 --- a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/BaseShardStrategy.java +++ b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/BaseShardStrategy.java @@ -1,6 +1,11 @@ package com.ctrip.framework.dal.cluster.client.sharding.strategy; import com.ctrip.framework.dal.cluster.client.config.ShardStrategyElement; +import com.ctrip.framework.dal.cluster.client.sharding.context.DbShardContext; +import com.ctrip.framework.dal.cluster.client.sharding.context.TableShardContext; + +import java.util.HashSet; +import java.util.Set; /** * @author c7ch23en @@ -9,11 +14,40 @@ public abstract class BaseShardStrategy extends ShardStrategyElement implements private static final String TABLE_SHARDING = "tableSharding"; private static final String TABLE_SHARD_SEPARATOR = "tableShardSeparator"; + private static final String DB_SHARD_OFFSET = "dbShardOffset"; + private static final String TABLE_SHARD_OFFSET = "tableShardOffset"; private static final boolean DEFAULT_TABLE_SHARDING = false; private static final String DEFAULT_TABLE_SHARD_SEPARATOR = "_"; + private static final int DEFAULT_DB_SHARD_OFFSET = 0; + private static final int DEFAULT_TABLE_SHARD_OFFSET = 0; + + protected BaseShardStrategy() {} + + protected abstract Integer calcDbShard(String tableName, DbShardContext context); + + protected abstract String calcTableShard(String tableName, TableShardContext context); + + @Override + public Integer getDbShard(String tableName, DbShardContext context) { + Integer shard = calcDbShard(tableName, context); + return shard != null ? (shard + getDbShardOffset(tableName)) : null; + } + + @Override + public String getTableShard(String tableName, TableShardContext context) { + String shard = calcTableShard(tableName, context); + return shard != null ? offsetTableShard(tableName, shard) : null; + } - public BaseShardStrategy() {} + @Override + public Set getAllTableShards(String tableName) { + Set allShards = calcAllTableShards(tableName); + Set offsetShards = new HashSet<>(); + for (String shard : allShards) + offsetShards.add(offsetTableShard(tableName, shard)); + return offsetShards; + } @Override public boolean tableShardingEnabled(String tableName) { @@ -25,4 +59,23 @@ public String getTableShardSeparator(String tableName) { return getTableProperty(tableName, TABLE_SHARD_SEPARATOR, DEFAULT_TABLE_SHARD_SEPARATOR); } + protected int getDbShardOffset(String tableName) { + return getTableIntProperty(tableName, DB_SHARD_OFFSET, DEFAULT_DB_SHARD_OFFSET); + } + + protected int getTableShardOffset(String tableName) { + return getTableIntProperty(tableName, TABLE_SHARD_OFFSET, DEFAULT_TABLE_SHARD_OFFSET); + } + + protected Set calcAllTableShards(String tableName) { + throw new UnsupportedOperationException(String.format("Could not calculate all table shards for table '%s'", tableName)); + } + + private String offsetTableShard(String tableName, String shard) { + int offset = getTableShardOffset(tableName); + if (offset == 0) + return shard; + return String.valueOf(Integer.parseInt(shard) + offset); + } + } diff --git a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/ColumnShardStrategy.java b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/ColumnShardStrategy.java index 6c27d7bea..cdd38353c 100644 --- a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/ColumnShardStrategy.java +++ b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/ColumnShardStrategy.java @@ -15,14 +15,14 @@ public abstract class ColumnShardStrategy extends BaseShardStrategy implements S private static final String DB_SHARD_COLUMN = "dbShardColumn"; private static final String TABLE_SHARD_COLUMN = "tableShardColumn"; - public ColumnShardStrategy() {} + protected ColumnShardStrategy() {} - public abstract Integer calcDbShard(String tableName, Object shardValue); + protected abstract Integer calcDbShard(String tableName, Object shardValue); - public abstract String calcTableShard(String tableName, Object shardValue); + protected abstract String calcTableShard(String tableName, Object shardValue); @Override - public Integer getDbShard(String tableName, DbShardContext context) { + protected Integer calcDbShard(String tableName, DbShardContext context) { Integer shard = context.getShardId(); if (shard != null) return shard; @@ -50,7 +50,7 @@ public Integer getDbShard(String tableName, DbShardContext context) { } @Override - public String getTableShard(String tableName, TableShardContext context) { + protected String calcTableShard(String tableName, TableShardContext context) { String shard = context.getShardId(); if (shard != null) return shard; @@ -77,7 +77,7 @@ public String getTableShard(String tableName, TableShardContext context) { return shard; } - private Object getDbShardValue(String tableName, ShardData shardData) { + protected Object getDbShardValue(String tableName, ShardData shardData) { if (shardData == null) return null; String dbShardColumn = getTableProperty(tableName, DB_SHARD_COLUMN); @@ -86,7 +86,7 @@ private Object getDbShardValue(String tableName, ShardData shardData) { return shardData.getValue(dbShardColumn); } - private Object getTableShardValue(String tableName, ShardData shardData) { + protected Object getTableShardValue(String tableName, ShardData shardData) { if (shardData == null) return null; String tableShardColumn = getTableProperty(tableName, TABLE_SHARD_COLUMN); diff --git a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/ModShardStrategy.java b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/ModShardStrategy.java index 33f1355ad..15b9dbf78 100644 --- a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/ModShardStrategy.java +++ b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/ModShardStrategy.java @@ -11,43 +11,33 @@ public class ModShardStrategy extends ColumnShardStrategy implements ShardStrategy { private static final String DB_SHARD_MOD = "dbShardMod"; - private static final String DB_SHARD_OFFSET = "dbShardOffset"; private static final String TABLE_SHARD_MOD = "tableShardMod"; - private static final String TABLE_SHARD_OFFSET = "tableShardOffset"; - - private static final int DEFAULT_DB_SHARD_OFFSET = 0; - private static final int DEFAULT_TABLE_SHARD_OFFSET = 0; public ModShardStrategy() {} @Override - public Integer calcDbShard(String tableName, Object shardValue) { + protected Integer calcDbShard(String tableName, Object shardValue) { Integer mod = getTableIntProperty(tableName, DB_SHARD_MOD); if (mod == null) throw new ClusterRuntimeException(String.format("db shard mod undefined for table '%s'", tableName)); - int modResult = getModResult(mod, shardValue); - int offset = getTableIntProperty(tableName, DB_SHARD_OFFSET, DEFAULT_DB_SHARD_OFFSET); - return modResult + offset; + return getModResult(mod, shardValue); } @Override - public String calcTableShard(String tableName, Object shardValue) { + protected String calcTableShard(String tableName, Object shardValue) { Integer mod = getTableIntProperty(tableName, TABLE_SHARD_MOD); if (mod == null) throw new ClusterRuntimeException(String.format("table shard mod undefined for table '%s'", tableName)); - int modResult = getModResult(mod, shardValue); - int offset = getTableIntProperty(tableName, TABLE_SHARD_OFFSET, DEFAULT_TABLE_SHARD_OFFSET); - return String.valueOf(modResult + offset); + return String.valueOf(getModResult(mod, shardValue)); } @Override - public Set getAllTableShards(String tableName) { + protected Set calcAllTableShards(String tableName) { Integer mod = getTableIntProperty(tableName, TABLE_SHARD_MOD); if (mod == null) throw new ClusterRuntimeException(String.format("table shard mod undefined for table '%s'", tableName)); - int offset = getTableIntProperty(tableName, TABLE_SHARD_OFFSET, DEFAULT_TABLE_SHARD_OFFSET); Set allShards = new HashSet<>(); - for (int i = offset; i < mod + offset; i++) + for (int i = 0; i < mod; i++) allShards.add(String.valueOf(i)); return allShards; } @@ -64,7 +54,7 @@ protected Long getLongValue(Object value) { return ((Number) value).longValue(); if (value instanceof String) return Long.parseLong((String) value); - throw new ClusterRuntimeException(String.format("value [%s] cannot be parsed as a number", value.toString())); + throw new ClusterRuntimeException(String.format("value [%s] cannot be parsed as a valid number", value.toString())); } } diff --git a/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/UserHintStrategy.java b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/UserHintStrategy.java new file mode 100644 index 000000000..b6dabd633 --- /dev/null +++ b/dal-cluster-client/src/main/java/com/ctrip/framework/dal/cluster/client/sharding/strategy/UserHintStrategy.java @@ -0,0 +1,20 @@ +package com.ctrip.framework.dal.cluster.client.sharding.strategy; + +import com.ctrip.framework.dal.cluster.client.sharding.context.DbShardContext; +import com.ctrip.framework.dal.cluster.client.sharding.context.TableShardContext; + +public class UserHintStrategy extends BaseShardStrategy implements ShardStrategy { + + public UserHintStrategy() {} + + @Override + protected Integer calcDbShard(String tableName, DbShardContext context) { + return context.getShardId(); + } + + @Override + protected String calcTableShard(String tableName, TableShardContext context) { + return context.getShardId(); + } + +} diff --git a/dal-cluster-client/src/test/java/com/ctrip/framework/dal/cluster/client/DefaultLocalConfigProviderTest.java b/dal-cluster-client/src/test/java/com/ctrip/framework/dal/cluster/client/DefaultLocalConfigProviderTest.java index 12d3182a4..28a783b77 100644 --- a/dal-cluster-client/src/test/java/com/ctrip/framework/dal/cluster/client/DefaultLocalConfigProviderTest.java +++ b/dal-cluster-client/src/test/java/com/ctrip/framework/dal/cluster/client/DefaultLocalConfigProviderTest.java @@ -3,6 +3,7 @@ import com.ctrip.framework.dal.cluster.client.config.ClusterConfig; import com.ctrip.framework.dal.cluster.client.config.ClusterConfigProvider; import com.ctrip.framework.dal.cluster.client.config.DefaultLocalConfigProvider; +import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,4 +22,9 @@ public void test() { LOGGER.info(String.format("cluster config: %s", config.toString())); } + @Test + public void testParseInt() { + Assert.assertEquals(-1, Integer.parseInt("-1")); + } + }