Added extensions and Rate limiting options

CHANGELOG.md

@@ -1,5 +1,13 @@
 
-## [3.12.0] - 2023-02-08
+## [3.13.0] - TBD
+### Added
+- Added `waggle-dance-extensions` module. See [extensions README](waggle-dance-extensions/README.md.)
+ - Added support to enable Rate Limiting of the Waggle Dance application based.
+### Changed
+- Changed and added some log message so for better tracking of calls being done.
+- Changed Integration Test WaggleDanceRunner to allow for better reuse.
+
+## [3.12.0] - 2024-02-08
 ### Added
 - Added optional `primary-meta-store.read-only-remote-meta-store-uris` config to allow traffic to be diverted based on calls made. See README.md.
 

README.md

@@ -113,6 +113,8 @@ The table below describes all the available configuration values for Waggle Danc
 | `configuration-properties`        | No         | Map of Hive properties that will be added to the HiveConf used when creating the Thrift clients (they will be shared among all the clients). |
 | `queryFunctionsAcrossAllMetastores` | No | Controls if the Thrift `getAllFunctions` should be fired to all configured metastores or only the primary metastore. The advice is to set this to false. Executing `getAllFunctions` can have an unwanted performance impact when a metastore is slow to respond. The function call is typically only called when a client is initialized and is largely irrelevant. Default is `true` (to be backward compatible) |
 
+Extensions (for instance Rate Limiting) are described here: [waggle-dance-extensions/README.md](waggle-dance-extensions/README.md)
+
 ### Federation
 
 Federation config is by default located in: `$WAGGLE_DANCE_HOME/conf/waggle-dance-federation.yml`.
@@ -611,4 +613,4 @@ The Waggle Dance logo uses the [Beetype Filled font](http://www.1001fonts.com/be
 ## Legal
 This project is available under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0.html).
 
-Copyright 2016-2019 Expedia, Inc.
+Copyright 2016-2024 Expedia, Inc.

pom.xml

@@ -24,6 +24,7 @@
   </scm>
 
   <modules>
+    <module>waggle-dance-extensions</module>
     <module>waggle-dance-core</module>
     <module>waggle-dance-api</module>
     <module>waggle-dance-rest</module>
@@ -275,6 +276,11 @@
         <artifactId>jackson-dataformat-cbor</artifactId>
         <version>${jackson.version}</version>
       </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.dataformat</groupId>
+        <artifactId>jackson-dataformat-yaml</artifactId>
+        <version>${jackson.version}</version>
+      </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-core</artifactId>

waggle-dance-boot/pom.xml

@@ -25,6 +25,11 @@
       <artifactId>waggle-dance-rest</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.hotels</groupId>
+      <artifactId>waggle-dance-extensions</artifactId>
+      <version>${project.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

waggle-dance-core/pom.xml

@@ -158,6 +158,8 @@
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-exec</artifactId>
+      <version>${hive.version}</version>
+      <classifier>core</classifier>
       <exclusions>
         <exclusion>
           <groupId>log4j</groupId>
@@ -167,6 +169,10 @@
           <groupId>org.pentaho</groupId>
           <artifactId>pentaho-aggdesigner-algorithm</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.calcite.avatica</groupId>
+          <artifactId>avatica</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -194,6 +200,10 @@
           <groupId>jdk.tools</groupId>
           <artifactId>jdk.tools</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-exec</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -346,6 +356,12 @@
       <groupId>com.hotels</groupId>
       <artifactId>beeju</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-exec</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>fm.last.commons</groupId>

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/WaggleDance.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIface.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2019 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java

@@ -25,6 +25,8 @@
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.amazonaws.glue.catalog.metastore.AWSCatalogMetastoreClient;
 
@@ -36,7 +38,7 @@
 import com.hotels.hcommon.hive.metastore.conf.HiveConfFactory;
 import com.hotels.hcommon.hive.metastore.util.MetaStoreUriNormaliser;
 
-public class CloseableThriftHiveMetastoreIfaceClientFactory {
+public class CloseableThriftHiveMetastoreIfaceClientFactory implements ThriftClientFactory {
 
   private static final int DEFAULT_CLIENT_FACTORY_RECONNECTION_RETRY = 3;
   private final TunnelingMetaStoreClientFactory tunnelingMetaStoreClientFactory;

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/DefaultMetaStoreClientFactory.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,7 +16,11 @@
 package com.hotels.bdp.waggledance.client;
 
 import java.io.IOException;
-import java.lang.reflect.*;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.UndeclaredThrowableException;
 import java.util.List;
 
 import org.apache.hadoop.hive.conf.HiveConf;

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/ThriftClientFactory.java

@@ -0,0 +1,27 @@
+/**
+ * Copyright (C) 2016-2024 Expedia, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.hotels.bdp.waggledance.client;
+
+import com.hotels.bdp.waggledance.api.model.AbstractMetaStore;
+
+public interface ThriftClientFactory {
+
+  /**
+   * @param metaStore (configuration object)
+   * @return client that will be used to query the metaStore.
+   */
+  public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore);
+}

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/ThriftMetastoreClientManager.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -121,7 +121,7 @@ void open(HiveUgiArgs ugiArgs) {
 
     for (int attempt = 0; !isConnected && (attempt < retries); ++attempt) {
       for (URI store : metastoreUris) {
-        LOG.info("Trying to connect to metastore with URI " + store);
+        LOG.debug("Trying to connect to metastore with URI " + store);
         try {
           transport = new TSocket(store.getHost(), store.getPort(), clientSocketTimeout, connectionTimeout);
           if (useSasl) {

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/tunnelling/TunnelableFactorySupplier.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2019 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/tunnelling/TunnelingMetaStoreClientFactory.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2021 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/conf/WaggleDanceConfiguration.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -145,5 +145,4 @@ public boolean isQueryFunctionsAcrossAllMetastores() {
   public void setQueryFunctionsAcrossAllMetastores(boolean queryFunctionsAcrossAllMetastores) {
     this.queryFunctionsAcrossAllMetastores = queryFunctionsAcrossAllMetastores;
   }
-
 }

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java

@@ -19,12 +19,14 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.context.annotation.Bean;
 
 import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIfaceClientFactory;
 import com.hotels.bdp.waggledance.client.DefaultMetaStoreClientFactory;
 import com.hotels.bdp.waggledance.client.GlueClientFactory;
 import com.hotels.bdp.waggledance.client.SplitTrafficMetastoreClientFactory;
+import com.hotels.bdp.waggledance.client.ThriftClientFactory;
 import com.hotels.bdp.waggledance.client.tunnelling.TunnelingMetaStoreClientFactory;
 import com.hotels.bdp.waggledance.conf.WaggleDanceConfiguration;
 import com.hotels.bdp.waggledance.mapping.model.ASTQueryMapping;
@@ -59,13 +61,21 @@ public SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory() {
     return new SplitTrafficMetastoreClientFactory();
   }
 
+
   @Bean
-  public CloseableThriftHiveMetastoreIfaceClientFactory metaStoreClientFactory(
+  public ThriftClientFactory defaultWaggleDanceClientFactory(
       WaggleDanceConfiguration waggleDanceConfiguration, SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory) {
     return new CloseableThriftHiveMetastoreIfaceClientFactory(new TunnelingMetaStoreClientFactory(),
         new DefaultMetaStoreClientFactory(), new GlueClientFactory(), waggleDanceConfiguration, splitTrafficMetaStoreClientFactory);
   }
 
+  //Only load when no other beans with this name can be found.
+  @ConditionalOnMissingBean
+  @Bean
+  public ThriftClientFactory thriftClientFactory(ThriftClientFactory defaultWaggleDanceClientFactory) {
+    return defaultWaggleDanceClientFactory;
+  }
+
   @Bean
   public QueryMapping queryMapping() {
     return ASTQueryMapping.INSTANCE;

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2022 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -33,7 +33,7 @@
 import com.hotels.bdp.waggledance.api.model.AbstractMetaStore;
 import com.hotels.bdp.waggledance.api.model.DatabaseResolution;
 import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIface;
-import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIfaceClientFactory;
+import com.hotels.bdp.waggledance.client.ThriftClientFactory;
 import com.hotels.bdp.waggledance.conf.WaggleDanceConfiguration;
 import com.hotels.bdp.waggledance.mapping.service.MetaStoreMappingFactory;
 import com.hotels.bdp.waggledance.mapping.service.PrefixNamingStrategy;
@@ -46,24 +46,24 @@ public class MetaStoreMappingFactoryImpl implements MetaStoreMappingFactory {
 
   private final WaggleDanceConfiguration waggleDanceConfiguration;
   private final PrefixNamingStrategy prefixNamingStrategy;
-  private final CloseableThriftHiveMetastoreIfaceClientFactory metaStoreClientFactory;
+  private final ThriftClientFactory thriftClientFactory;
   private final AccessControlHandlerFactory accessControlHandlerFactory;
 
   @Autowired
   public MetaStoreMappingFactoryImpl(
       WaggleDanceConfiguration waggleDanceConfiguration,
       PrefixNamingStrategy prefixNamingStrategy,
-      CloseableThriftHiveMetastoreIfaceClientFactory metaStoreClientFactory,
+      ThriftClientFactory thriftClientFactory,
       AccessControlHandlerFactory accessControlHandlerFactory) {
     this.waggleDanceConfiguration = waggleDanceConfiguration;
     this.prefixNamingStrategy = prefixNamingStrategy;
-    this.metaStoreClientFactory = metaStoreClientFactory;
+    this.thriftClientFactory = thriftClientFactory;
     this.accessControlHandlerFactory = accessControlHandlerFactory;
   }
 
   private CloseableThriftHiveMetastoreIface createClient(AbstractMetaStore metaStore) {
     try {
-      return metaStoreClientFactory.newInstance(metaStore);
+      return thriftClientFactory.newInstance(metaStore);
     } catch (Exception e) {
       LOG.error("Can't create a client for metastore '{}':", metaStore.getName(), e);
       return newUnreachableMetastoreClient(metaStore);

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/ExceptionWrappingHMSHandler.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2019 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,6 +22,8 @@
 import java.lang.reflect.UndeclaredThrowableException;
 import java.util.Arrays;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
 import org.apache.hadoop.hive.metastore.IHMSHandler;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.slf4j.Logger;
@@ -34,10 +36,12 @@ public class ExceptionWrappingHMSHandler implements InvocationHandler {
   private final static Logger LOG = LoggerFactory.getLogger(ExceptionWrappingHMSHandler.class);
 
   private final IHMSHandler baseHandler;
+  private String user = "";
 
   public static IHMSHandler newProxyInstance(IHMSHandler baseHandler) {
-    return (IHMSHandler) Proxy.newProxyInstance(ExceptionWrappingHMSHandler.class.getClassLoader(),
-        new Class[] { IHMSHandler.class }, new ExceptionWrappingHMSHandler(baseHandler));
+    return (IHMSHandler) Proxy
+        .newProxyInstance(ExceptionWrappingHMSHandler.class.getClassLoader(), new Class[] { IHMSHandler.class },
+            new ExceptionWrappingHMSHandler(baseHandler));
   }
 
   public ExceptionWrappingHMSHandler(IHMSHandler baseHandler) {
@@ -46,7 +50,13 @@ public ExceptionWrappingHMSHandler(IHMSHandler baseHandler) {
 
   @Override
   public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+    if (method.getName().equals("set_ugi")) {
+      user = (String) args[0];
+    }
     try {
+      LOG
+          .info("WD Audit:[User:{}, method:{}, source_ip:{}, args:{}]", user, method.getName(),
+              HMSHandler.getThreadLocalIpAddress(), StringUtils.left(Arrays.toString(args), 256));
       return method.invoke(baseHandler, args);
     } catch (InvocationTargetException e) {
       Throwable cause = e.getCause();

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java

@@ -135,9 +135,9 @@ public void create_database(Database database)
   @Override
   @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
   public Database get_database(String name) throws NoSuchObjectException, MetaException, TException {
-    LOG.info("Fetching database {}", name);
+    LOG.debug("Fetching database {}", name);
     DatabaseMapping mapping = databaseMappingService.databaseMapping(name);
-    LOG.info("Mapping is '{}'", mapping.getDatabasePrefix());
+    LOG.debug("Mapping is '{}'", mapping.getDatabasePrefix());
     Database result = mapping.getClient().get_database(mapping.transformInboundDatabaseName(name));
     return mapping.transformOutboundDatabase(mapping.getMetastoreFilter().filterDatabase(result));
   }

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/WaggleDanceServerException.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2021 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/DefaultMetaStoreClientFactoryTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/compatibility/HiveCompatibleThriftHiveMetastoreIfaceFactoryTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2020 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/tunnelling/TunnelableFactorySupplierTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2020 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.