From 62678a885416b5f0eea9f248238f8d11ca1b4d28 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Thu, 4 Jul 2024 15:53:19 +0800 Subject: [PATCH 1/7] Add hive hook configuration parameters for each hms --- .../mapping/model/MetaStoreMappingFactoryImpl.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java index 4e74cc53..ecba6d2a 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java @@ -108,6 +108,12 @@ private MetaStoreFilterHook loadMetastoreFilterHook(AbstractMetaStore metaStore) conf.set(property.getKey(), property.getValue()); } } + Map metaConfigurationProperties = metaStore.getConfigurationProperties(); + if (metaConfigurationProperties != null) { + for (Map.Entry property : metaConfigurationProperties.entrySet()) { + conf.set(property.getKey(), property.getValue()); + } + } conf.set(HiveConf.ConfVars.METASTORE_FILTER_HOOK.varname, metaStoreFilterHook); try { Class filterHookClass = conf From 5c69dfefc4ec75b8a5a3379edd2da0453066dc5d Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Fri, 5 Jul 2024 19:27:46 +0800 Subject: [PATCH 2/7] Added unit tests for hive hooks that work on a single hms --- .../MetaStoreMappingFactoryImplTest.java | 37 +++++++++++++++++-- .../model/PrefixingMetastoreFilter.java | 7 +++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java index acf5557a..447e8477 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java @@ -15,9 +15,7 @@ */ package com.hotels.bdp.waggledance.mapping.model; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.verify; @@ -25,13 +23,20 @@ import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance; +import java.io.File; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl; +import org.apache.hadoop.hive.metastore.MetaStoreFilterHook; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; import org.apache.thrift.TException; import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; @@ -64,6 +69,7 @@ public class MetaStoreMappingFactoryImplTest { new WaggleDanceConfiguration(), new SplitTrafficMetastoreClientFactory()); private MetaStoreMappingFactoryImpl factory; + public @Rule TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before public void init() { @@ -133,13 +139,36 @@ public void unreachableMetastoreClient() { } } + @Test + public void loadMetastoreFilterHook(){ + + } + @Test public void loadMetastoreFilterHookFromConfig() { AbstractMetaStore federatedMetaStore = newFederatedInstance("fed1", thrift.getThriftConnectionUri()); federatedMetaStore.setHiveMetastoreFilterHook(PrefixingMetastoreFilter.class.getName()); + Map metaStoreConfigurationProperties = new HashMap<>(); + metaStoreConfigurationProperties.put(PrefixingMetastoreFilter.PREFIX_KEY,"prefix-test-"); + federatedMetaStore.setConfigurationProperties(metaStoreConfigurationProperties); + MetaStoreMapping mapping = factory.newInstance(federatedMetaStore); assertThat(mapping, is(notNullValue())); - assertThat(mapping.getMetastoreFilter(), instanceOf(PrefixingMetastoreFilter.class)); + + MetaStoreFilterHook filterHook = mapping.getMetastoreFilter(); + assertThat(filterHook, instanceOf(PrefixingMetastoreFilter.class)); + + Table table = new Table(); + try { + StorageDescriptor sd = new StorageDescriptor(); + File localWarehouseUri = temporaryFolder.newFolder("local-warehouse"); + sd.setLocation(new File(localWarehouseUri,"local_database/local_table").toURI().toString()); + table.setSd(sd); + + String oldLocation=sd.getLocation(); + assertThat(filterHook.filterTable(table).getSd().getLocation(), equalTo("prefix-test-" + oldLocation ) ); + }catch (Exception e){ + } } @Test diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java index b7c12010..82ee3e16 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java @@ -33,9 +33,12 @@ * */ public class PrefixingMetastoreFilter implements MetaStoreFilterHook { - public static final String PREFIX = "prefix-"; + public static final String PREFIX_KEY = "hive.metastore.hooks.prefix"; + public static final String PREFIX_DEFAULT = "prefix-"; + private final String prefix; public PrefixingMetastoreFilter(HiveConf conf) { + prefix = conf.get(PREFIX_KEY,PREFIX_DEFAULT); } @Override @@ -140,7 +143,7 @@ private void setLocationPrefix(Partition partition) { private void setLocationPrefix(StorageDescriptor sd) { String location = sd.getLocation(); - sd.setLocation(PREFIX + location); + sd.setLocation(prefix + location); } } From 61609db1b8276a7bceda9e0f5c764226b01aa1e1 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Mon, 8 Jul 2024 09:06:31 +0800 Subject: [PATCH 3/7] Clean up the excess unit test --- .../mapping/model/MetaStoreMappingFactoryImplTest.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java index 447e8477..7bfc06c4 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java @@ -139,11 +139,6 @@ public void unreachableMetastoreClient() { } } - @Test - public void loadMetastoreFilterHook(){ - - } - @Test public void loadMetastoreFilterHookFromConfig() { AbstractMetaStore federatedMetaStore = newFederatedInstance("fed1", thrift.getThriftConnectionUri()); From ab446906534087e45fa4683d079357fc8207b3a8 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Wed, 10 Jul 2024 10:51:05 +0800 Subject: [PATCH 4/7] Add hive hook custom config unit test --- .../MetaStoreMappingFactoryImplTest.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java index 7bfc06c4..be91d32c 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java @@ -141,6 +141,23 @@ public void unreachableMetastoreClient() { @Test public void loadMetastoreFilterHookFromConfig() { + AbstractMetaStore federatedMetaStore = newFederatedInstance("fed1", thrift.getThriftConnectionUri()); + federatedMetaStore.setHiveMetastoreFilterHook(PrefixingMetastoreFilter.class.getName()); + MetaStoreMapping mapping = factory.newInstance(federatedMetaStore); + assertThat(mapping, is(notNullValue())); + assertThat(mapping.getMetastoreFilter(), instanceOf(PrefixingMetastoreFilter.class)); + } + + @Test + public void loadDefaultMetastoreFilterHook() { + AbstractMetaStore federatedMetaStore = newFederatedInstance("fed1", thrift.getThriftConnectionUri()); + MetaStoreMapping mapping = factory.newInstance(federatedMetaStore); + assertThat(mapping, is(notNullValue())); + assertThat(mapping.getMetastoreFilter(), instanceOf(DefaultMetaStoreFilterHookImpl.class)); + } + + @Test + public void loadMetastoreFilterHookWithCustomConfig() { AbstractMetaStore federatedMetaStore = newFederatedInstance("fed1", thrift.getThriftConnectionUri()); federatedMetaStore.setHiveMetastoreFilterHook(PrefixingMetastoreFilter.class.getName()); Map metaStoreConfigurationProperties = new HashMap<>(); @@ -149,7 +166,6 @@ public void loadMetastoreFilterHookFromConfig() { MetaStoreMapping mapping = factory.newInstance(federatedMetaStore); assertThat(mapping, is(notNullValue())); - MetaStoreFilterHook filterHook = mapping.getMetastoreFilter(); assertThat(filterHook, instanceOf(PrefixingMetastoreFilter.class)); @@ -165,12 +181,4 @@ public void loadMetastoreFilterHookFromConfig() { }catch (Exception e){ } } - - @Test - public void loadDefaultMetastoreFilterHook() { - AbstractMetaStore federatedMetaStore = newFederatedInstance("fed1", thrift.getThriftConnectionUri()); - MetaStoreMapping mapping = factory.newInstance(federatedMetaStore); - assertThat(mapping, is(notNullValue())); - assertThat(mapping.getMetastoreFilter(), instanceOf(DefaultMetaStoreFilterHookImpl.class)); - } } From 0d67f268fc481aae5c02818321f7e1d229e6d85b Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Wed, 10 Jul 2024 11:01:28 +0800 Subject: [PATCH 5/7] Rename variable name --- .../bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java index 82ee3e16..c2afe03d 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java @@ -33,7 +33,7 @@ * */ public class PrefixingMetastoreFilter implements MetaStoreFilterHook { - public static final String PREFIX_KEY = "hive.metastore.hooks.prefix"; + public static final String PREFIX_KEY = "waggledance.hook.prefix"; public static final String PREFIX_DEFAULT = "prefix-"; private final String prefix; From 1ff4846bbc173713a382e190e2e026ad66a18879 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Mon, 29 Jul 2024 15:33:17 +0800 Subject: [PATCH 6/7] Remove try catch and organize imports --- .../mapping/model/MetaStoreMappingFactoryImplTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java index be91d32c..957f8a17 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java @@ -15,7 +15,10 @@ */ package com.hotels.bdp.waggledance.mapping.model; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.verify; @@ -157,7 +160,7 @@ public void loadDefaultMetastoreFilterHook() { } @Test - public void loadMetastoreFilterHookWithCustomConfig() { + public void loadMetastoreFilterHookWithCustomConfig() throws Exception{ AbstractMetaStore federatedMetaStore = newFederatedInstance("fed1", thrift.getThriftConnectionUri()); federatedMetaStore.setHiveMetastoreFilterHook(PrefixingMetastoreFilter.class.getName()); Map metaStoreConfigurationProperties = new HashMap<>(); @@ -170,7 +173,6 @@ public void loadMetastoreFilterHookWithCustomConfig() { assertThat(filterHook, instanceOf(PrefixingMetastoreFilter.class)); Table table = new Table(); - try { StorageDescriptor sd = new StorageDescriptor(); File localWarehouseUri = temporaryFolder.newFolder("local-warehouse"); sd.setLocation(new File(localWarehouseUri,"local_database/local_table").toURI().toString()); @@ -178,7 +180,5 @@ public void loadMetastoreFilterHookWithCustomConfig() { String oldLocation=sd.getLocation(); assertThat(filterHook.filterTable(table).getSd().getLocation(), equalTo("prefix-test-" + oldLocation ) ); - }catch (Exception e){ - } } } From 9508e996aa281df25989ec9b760faece00e43171 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Mon, 29 Jul 2024 15:40:33 +0800 Subject: [PATCH 7/7] Set StorageDescriptor location --- .../model/MetaStoreMappingFactoryImplTest.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java index 957f8a17..f5c24c30 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImplTest.java @@ -26,7 +26,6 @@ import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance; -import java.io.File; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -39,7 +38,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; @@ -72,7 +70,6 @@ public class MetaStoreMappingFactoryImplTest { new WaggleDanceConfiguration(), new SplitTrafficMetastoreClientFactory()); private MetaStoreMappingFactoryImpl factory; - public @Rule TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before public void init() { @@ -173,12 +170,11 @@ public void loadMetastoreFilterHookWithCustomConfig() throws Exception{ assertThat(filterHook, instanceOf(PrefixingMetastoreFilter.class)); Table table = new Table(); - StorageDescriptor sd = new StorageDescriptor(); - File localWarehouseUri = temporaryFolder.newFolder("local-warehouse"); - sd.setLocation(new File(localWarehouseUri,"local_database/local_table").toURI().toString()); - table.setSd(sd); + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation("file:///tmp/local_database/local_table"); + table.setSd(sd); - String oldLocation=sd.getLocation(); - assertThat(filterHook.filterTable(table).getSd().getLocation(), equalTo("prefix-test-" + oldLocation ) ); + String oldLocation=sd.getLocation(); + assertThat(filterHook.filterTable(table).getSd().getLocation(), equalTo("prefix-test-" + oldLocation ) ); } }