HAWQ-1644. Make delegation token optional in PXF

Make delegation token property optional for PXF when used against secure hadoop. Delegation token is a complementary means for authentication with hadoop along with Kerberos. There is not much value in using the delegation token with masterless and segment only PXF architecture, as each pxf jvm will need to establish the token anyway. Simply authenticating with Keberos should be good enough in such a deployment mode. Co-authored-by: Alexander Denissov <[email protected]> Co-authored-by: Francisco Guerrero <[email protected]> Co-authored-by: Shivram Mani <[email protected]>
huor · Aug 13, 2018 · 4f12950 · 4f12950
1 parent 80142f4
commit 4f12950
Show file tree

Hide file tree

Showing 16 changed files with 115 additions and 109 deletions.
diff --git a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/ParquetResolver.java b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/ParquetResolver.java
@@ -57,7 +57,7 @@ public ParquetResolver(InputData metaData) {
     }
 
     /**
-     * @inheritDoc
+     * {@inheritDoc}
      * @param schema the MessageType instance, which is obtained from the Parquet file footer
      */
     public List<OneField> getFields(OneRow row, MessageType schema) throws Exception

diff --git a/pxf/pxf-service/src/configs/pxf-site.xml b/pxf/pxf-service/src/configs/pxf-site.xml
diff --git a/pxf/pxf-service/src/configs/tomcat/bin/setenv.sh b/pxf/pxf-service/src/configs/tomcat/bin/setenv.sh
@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -28,7 +28,7 @@ JVM_OPTS=""
 PXF_LOGDIR="$CATALINA_BASE/logs"
 PXF_USER_IMPERSONATION=""
 
-PXF_OPTS="-Dpxf.log.dir=$PXF_LOGDIR -Dpxf.service.user.impersonation.enabled=$PXF_USER_IMPERSONATION"
+PXF_OPTS="-Dpxf.log.dir=$PXF_LOGDIR -Dpxf.service.user.impersonation.enabled=$PXF_USER_IMPERSONATION -Dpxf.service.kerberos.keytab=$PXF_KEYTAB -Dpxf.service.kerberos.principal=$PXF_PRINCIPAL"
 JAVA_OPTS="$JVM_OPTS $AGENT_PATHS $JAVA_AGENTS $JAVA_LIBRARY_PATH $PXF_OPTS"
 
 CATALINA_PID="$CATALINA_BASE/logs/catalina.pid"

diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/BridgeResource.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/BridgeResource.java
@@ -96,9 +96,8 @@ public Response read(@Context final ServletContext servletContext,
             LOG.debug("started with parameters: " + params);
         }
 
-        ProtocolData protData = new ProtocolData(params);
-        SecuredHDFS.verifyToken(protData, servletContext);
         Bridge bridge;
+        ProtocolData protData = new ProtocolData(params);
         float sampleRatio = protData.getStatsSampleRatio();
         if (sampleRatio > 0) {
             bridge = new ReadSamplingBridge(protData);

diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/FragmenterResource.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/FragmenterResource.java
@@ -147,7 +147,6 @@ private ProtocolData getProtocolData(final ServletContext servletContext,
         if (protData.getFragmenter() == null) {
             protData.protocolViolation("fragmenter");
         }
-        SecuredHDFS.verifyToken(protData, servletContext);
 
         return protData;
     }

diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
@@ -99,11 +99,10 @@ public Response read(@Context final ServletContext servletContext,
             // Convert headers into a regular map
             Map<String, String> params = convertToCaseInsensitiveMap(headers.getRequestHeaders());
 
-            // Add profile and verify token
+            // Add profile
             ProtocolData protData = new ProtocolData(params, profile.toLowerCase());
 
-            // 0. Verify token
-            SecuredHDFS.verifyToken(protData, servletContext);
+            // Token verification happens at the SecurityServletFilter
 
             // 1. start MetadataFetcher
             MetadataFetcher metadataFetcher = MetadataFetcherFactory.create(protData);

diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/WritableResource.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/WritableResource.java
@@ -117,8 +117,6 @@ public Response stream(@Context final ServletContext servletContext,
 
         ProtocolData protData = new ProtocolData(params);
         protData.setDataSource(path);
-
-        SecuredHDFS.verifyToken(protData, servletContext);
         Bridge bridge = new WriteBridge(protData);
 
         // THREAD-SAFE parameter has precedence

diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/servlet/SecurityServletFilter.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/servlet/SecurityServletFilter.java
@@ -34,9 +34,11 @@
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hawq.pxf.service.SessionId;
 import org.apache.hawq.pxf.service.UGICache;
 import org.apache.hawq.pxf.service.utilities.SecureLogin;
+import org.apache.hawq.pxf.service.utilities.SecuredHDFS;
 
 /**
  * Listener on lifecycle events of our webapp
@@ -48,8 +50,10 @@ public class SecurityServletFilter implements Filter {
     private static final String SEGMENT_ID_HEADER = "X-GP-SEGMENT-ID";
     private static final String TRANSACTION_ID_HEADER = "X-GP-XID";
     private static final String LAST_FRAGMENT_HEADER = "X-GP-LAST-FRAGMENT";
+    private static final String DELEGATION_TOKEN_HEADER = "X-GP-TOKEN";
     private static final String MISSING_HEADER_ERROR = "Header %s is missing in the request";
     private static final String EMPTY_HEADER_ERROR = "Header %s is empty in the request";
+    private FilterConfig config;
     UGICache proxyUGICache;
 
     /**
@@ -59,6 +63,7 @@ public class SecurityServletFilter implements Filter {
      */
     @Override
     public void init(FilterConfig filterConfig) {
+        config = filterConfig;
         proxyUGICache = new UGICache();
     }
 
@@ -76,7 +81,6 @@ public void doFilter(final ServletRequest request, final ServletResponse respons
             throws IOException, ServletException {
 
         if (SecureLogin.isUserImpersonationEnabled()) {
-
             // retrieve user header and make sure header is present and is not empty
             final String gpdbUser = getHeaderValue(request, USER_HEADER, true);
             final String transactionId = getHeaderValue(request, TRANSACTION_ID_HEADER, true);
@@ -85,9 +89,7 @@ public void doFilter(final ServletRequest request, final ServletResponse respons
 
             SessionId session = new SessionId(segmentId, transactionId, gpdbUser);
 
-            // TODO refresh Kerberos token when security is enabled
-
-            // prepare privileged action to run on behalf of proxy user
+            // Prepare privileged action to run on behalf of proxy user
             PrivilegedExceptionAction<Boolean> action = new PrivilegedExceptionAction<Boolean>() {
                 @Override
                 public Boolean run() throws IOException, ServletException {
@@ -102,12 +104,18 @@ public Boolean run() throws IOException, ServletException {
             if (LOG.isDebugEnabled()) {
                 LOG.debug("Retrieving proxy user for session: " + session);
             }
+
+            // Refresh Kerberos token when security is enabled
+            String tokenString = getHeaderValue(request, DELEGATION_TOKEN_HEADER, false);
+            SecuredHDFS.verifyToken(tokenString, config.getServletContext());
+
             try {
                 // Retrieve proxy user UGI from the UGI of the logged in user
-                // and execute the servlet chain as that user
-                proxyUGICache
-                        .getUserGroupInformation(session)
-                        .doAs(action);
+                UserGroupInformation proxyUserGroupInformation = proxyUGICache
+                        .getUserGroupInformation(session);
+
+                // Execute the servlet chain as that user
+                proxyUserGroupInformation.doAs(action);
             } catch (UndeclaredThrowableException ute) {
                 // unwrap the real exception thrown by the action
                 throw new ServletException(ute.getCause());

diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
@@ -8,9 +8,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -371,7 +371,7 @@ private void parseSecurityProperties() {
 
         /* Kerberos token information */
         if (UserGroupInformation.isSecurityEnabled()) {
-            this.token = getProperty("TOKEN");
+            this.token = getOptionalProperty("TOKEN");
         }
     }
 

diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/SecureLogin.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/SecureLogin.java
@@ -19,8 +19,10 @@
  * under the License.
  */
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.SecurityUtil;
@@ -45,24 +47,53 @@ public class SecureLogin {
     private static final Log LOG = LogFactory.getLog(SecureLogin.class);
 
     private static final String PROPERTY_KEY_USER_IMPERSONATION = "pxf.service.user.impersonation.enabled";
-
-    private static final String CONFIG_KEY_SERVICE_KEYTAB = "pxf.service.kerberos.keytab";
     private static final String CONFIG_KEY_SERVICE_PRINCIPAL = "pxf.service.kerberos.principal";
+    private static final String CONFIG_KEY_SERVICE_KEYTAB = "pxf.service.kerberos.keytab";
 
     /**
      * Establishes Login Context for the PXF service principal using Kerberos keytab.
      */
     public static void login() {
         try {
+            boolean isUserImpersonationEnabled = isUserImpersonationEnabled();
+            LOG.info("User impersonation is " + (isUserImpersonationEnabled ? "enabled" : "disabled"));
+
+            if (!UserGroupInformation.isSecurityEnabled()) {
+                LOG.info("Kerberos Security is not enabled");
+                return;
+            }
+
+            LOG.info("Kerberos Security is enabled");
+
+            if (!isUserImpersonationEnabled) {
+                throw new RuntimeException("User Impersonation is required when Kerberos Security is enabled. " +
+                        "Set PXF_USER_IMPERSONATION=true in $PXF_HOME/conf/pxf-env.sh");
+            }
+
+            String principal = System.getProperty(CONFIG_KEY_SERVICE_PRINCIPAL);
+            String keytabFilename = System.getProperty(CONFIG_KEY_SERVICE_KEYTAB);
+
+            if (StringUtils.isEmpty(principal)) {
+                throw new RuntimeException("Kerberos Security requires a valid principal.");
+            }
+
+            if (StringUtils.isEmpty(keytabFilename)) {
+                throw new RuntimeException("Kerberos Security requires a valid keytab file name.");
+            }
+
             Configuration config = new Configuration();
-            config.addResource("pxf-site.xml");
+            config.set(CONFIG_KEY_SERVICE_PRINCIPAL, principal);
+            config.set(CONFIG_KEY_SERVICE_KEYTAB, keytabFilename);
 
-            SecurityUtil.login(config, CONFIG_KEY_SERVICE_KEYTAB, CONFIG_KEY_SERVICE_PRINCIPAL);
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Kerberos principal: " + config.get(CONFIG_KEY_SERVICE_PRINCIPAL));
+                LOG.debug("Kerberos keytab: " + config.get(CONFIG_KEY_SERVICE_KEYTAB));
+            }
 
-            LOG.info("User impersonation is " + (isUserImpersonationEnabled() ? "enabled" : "disabled"));
+            SecurityUtil.login(config, CONFIG_KEY_SERVICE_KEYTAB, CONFIG_KEY_SERVICE_PRINCIPAL);
 
         } catch (Exception e) {
-            LOG.error("PXF service login failed");
+            LOG.error("PXF service login failed: " + e.getMessage());
             throw new RuntimeException(e);
         }
     }
@@ -73,6 +104,6 @@ public static void login() {
      * @return true if user impersonation is enabled, false otherwise
      */
     public static boolean isUserImpersonationEnabled() {
-        return System.getProperty(PROPERTY_KEY_USER_IMPERSONATION, "").equalsIgnoreCase("true") ? true : false;
+        return StringUtils.equalsIgnoreCase(System.getProperty(PROPERTY_KEY_USER_IMPERSONATION, ""), "true");
     }
 }
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/SecuredHDFS.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/SecuredHDFS.java
@@ -45,28 +45,29 @@ public class SecuredHDFS {
      *
      * All token properties will be deserialized from string to a Token object
      *
-     * @param protData input parameters
+     * @param tokenString (optional) the delegation token
      * @param context servlet context which contains the NN address
      *
      * @throws SecurityException Thrown when authentication fails
      */
-    public static void verifyToken(ProtocolData protData, ServletContext context) {
+    public static void verifyToken(String tokenString, ServletContext context) {
         try {
             if (UserGroupInformation.isSecurityEnabled()) {
                 /*
-                 * HAWQ-1215: The verify token method validates that the token sent from
+                 * The verify token method validates that the token sent from
                  * Hawq to PXF is valid. However, this token is for a user other than
                  * 'pxf'. The following line ensures that before attempting any secure communication
                  * PXF tries to relogin in the case that its own ticket is about to expire
                  * #reloginFromKeytab is a no-op if the ticket is not near expiring
                  */
                 UserGroupInformation.getLoginUser().reloginFromKeytab();
-                Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
-                String tokenString = protData.getToken();
-                token.decodeFromUrlString(tokenString);
+                if (tokenString != null) {
+                    Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
+                    token.decodeFromUrlString(tokenString);
 
-                verifyToken(token.getIdentifier(), token.getPassword(),
-                        token.getKind(), token.getService(), context);
+                    verifyToken(token.getIdentifier(), token.getPassword(),
+                            token.getKind(), token.getService(), context);
+                }
             }
         } catch (IOException e) {
             throw new SecurityException("Failed to verify delegation token "

diff --git a/pxf/pxf-service/src/scripts/pxf-env.sh b/pxf/pxf-service/src/scripts/pxf-env.sh
@@ -44,6 +44,12 @@ export PXF_PORT=${PXF_PORT:-@pxfPortNum@}
 # Memory
 export PXF_JVM_OPTS="-Xmx2g -Xms1g"
 
+# Kerberos
+# Path to keytab file owned by pxf service with permissions 0400
+export PXF_KEYTAB="/etc/security/keytab/pxf.service.keytab"
+# Kerberos principal pxf service should use. _HOST is replaced automatically with hostnames FQDN
+export PXF_PRINCIPAL="pxf/[email protected]"
+
 # Hadoop Distribution Type (optional), supported values:
 # <empty> - for auto discovery of HDP, CDH or tarball based client installation
 # HDP     - for HDP Hadoop client installation

diff --git a/pxf/pxf-service/src/scripts/pxf-service b/pxf/pxf-service/src/scripts/pxf-service
@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -110,9 +110,9 @@ function deployWebapp()
 function waitForTomcat()
 {
 	attempts=0
-	max_attempts=$1 # number of attempts to connect 
+	max_attempts=$1 # number of attempts to connect
 	sleep_time=1 # sleep 1 second between attempts
-	
+
 	# wait until tomcat is up:
 	echo Checking if tomcat is up and running...
 	until [[ "`curl --silent --connect-timeout 1 -I http://localhost:${instance_port} | grep 'Coyote'`" != "" ]];
@@ -136,16 +136,16 @@ function waitForTomcat()
 function checkWebapp()
 {
 	waitForTomcat $1 || return 1
-	
+
 	echo Checking if PXF webapp is up and running...
 	curlResponse=$(${curl} -s "http://localhost:${instance_port}/pxf/v0")
 	expectedResponse="Wrong version v0, supported version is v[0-9]+"
-	
+
 	if [[ ${curlResponse} =~ $expectedResponse ]]; then
 		echo PXF webapp is listening on port ${instance_port}
 		return 0
 	fi
-	
+
 	echo ERROR: PXF webapp is inaccessible but tomcat is up. Check logs for more information
 	return 1
 }
@@ -210,6 +210,8 @@ function configureWebapp()
     catalinaEnv=${instance_root}/${instance_name}/bin/setenv.sh
     sed -i -e "s|JVM_OPTS=.*$|JVM_OPTS=\"${PXF_JVM_OPTS}\"|g" ${catalinaEnv}
     sed -i -e "s|-Dpxf.log.dir=[^[:space:]^\"]*|-Dpxf.log.dir=${PXF_LOGDIR} |g" ${catalinaEnv}
+    sed -i -e "s|-Dpxf.service.kerberos.keytab=[^[:space:]^\"]*|-Dpxf.service.kerberos.keytab=${PXF_KEYTAB} |g" ${catalinaEnv}
+    sed -i -e "s|-Dpxf.service.kerberos.principal=[^[:space:]^\"]*|-Dpxf.service.kerberos.principal=${PXF_PRINCIPAL} |g" ${catalinaEnv}
     sed -i -e "s|^[[:blank:]]*PXF_USER_IMPERSONATION=.*$|PXF_USER_IMPERSONATION=\"${PXF_USER_IMPERSONATION}\"|g" ${catalinaEnv}
     sed -i -e "s|^[[:blank:]]*CATALINA_PID=.*$|CATALINA_PID=${PXF_RUNDIR}/catalina.pid|g" ${catalinaEnv}
     sed -i -e "s|^[[:blank:]]*CATALINA_OUT=.*$|CATALINA_OUT=${PXF_LOGDIR}/catalina.out|g" ${catalinaEnv}
@@ -386,7 +388,7 @@ function validateParameters()
     return 0
 }
 
-# 
+#
 # doStart handles start command
 # command is executed as the user $pxf_user
 #