KEYCLOAK-8349 KEYCLOAK-8659 Use TLS for all tests in the suite

.travis.yml

@@ -17,7 +17,6 @@ env:
     - TESTS=old
     - TESTS=crossdc-server
     - TESTS=crossdc-adapter
-    - TESTS=ssl
 
 jdk:
   - oraclejdk8

adapters/oidc/adapter-core/src/main/java/org/keycloak/adapters/KeycloakDeployment.java

@@ -282,6 +282,13 @@ public void setSslRequired(SslRequired sslRequired) {
         this.sslRequired = sslRequired;
     }
 
+    public boolean isSSLEnabled() {
+        if (SslRequired.NONE == sslRequired) {
+            return false;
+        }
+        return true;
+    }
+
     public int getConfidentialPort() {
         return confidentialPort;
     }

adapters/oidc/adapter-core/src/main/java/org/keycloak/adapters/KeycloakDeploymentBuilder.java

@@ -130,7 +130,7 @@ protected KeycloakDeployment internalBuild(AdapterConfig adapterConfig) {
         if (realmKeyPem == null && adapterConfig.isBearerOnly() && adapterConfig.getAuthServerUrl() == null) {
             throw new IllegalArgumentException("For bearer auth, you must set the realm-public-key or auth-server-url");
         }
-        if (realmKeyPem == null || !deployment.isBearerOnly() || deployment.isEnableBasicAuth() || deployment.isRegisterNodeAtStartup() || deployment.getRegisterNodePeriod() != -1) {
+        if (realmKeyPem == null || !deployment.isBearerOnly() || deployment.isSSLEnabled() || deployment.isEnableBasicAuth() || deployment.isRegisterNodeAtStartup() || deployment.getRegisterNodePeriod() != -1) {
             deployment.setClient(new HttpClientBuilder().build(adapterConfig));
         }
         if (adapterConfig.getAuthServerUrl() == null && (!deployment.isBearerOnly() || realmKeyPem == null)) {

core/src/main/java/org/keycloak/util/JsonSerialization.java

@@ -41,7 +41,9 @@ public class JsonSerialization {
     public static final ObjectMapper sysPropertiesAwareMapper = new ObjectMapper(new SystemPropertiesJsonParserFactory());
 
     static {
+        mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
         mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+        prettyMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
         prettyMapper.enable(SerializationFeature.INDENT_OUTPUT);
         prettyMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
     }

integration/admin-client/src/main/java/org/keycloak/admin/client/Keycloak.java

@@ -27,7 +27,9 @@
 import org.keycloak.admin.client.resource.ServerInfoResource;
 import org.keycloak.admin.client.token.TokenManager;
 
+import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
 
 import java.net.URI;
 
@@ -49,10 +51,10 @@ public class Keycloak implements AutoCloseable {
     private final String authToken;
     private final ResteasyWebTarget target;
     private final ResteasyClient client;
-    
+
     Keycloak(String serverUrl, String realm, String username, String password, String clientId, String clientSecret, String grantType, ResteasyClient resteasyClient, String authtoken) {
         config = new Config(serverUrl, realm, username, password, clientId, clientSecret, grantType);
-        client = resteasyClient != null ? resteasyClient : new ResteasyClientBuilder().connectionPoolSize(10).build();
+        client = resteasyClient != null ? resteasyClient : newRestEasyClient(null, null, false);
         authToken = authtoken;
         tokenManager = authtoken == null ? new TokenManager(config, client) : null;
 
@@ -64,33 +66,53 @@ private BearerAuthFilter newAuthFilter() {
         return authToken != null ? new BearerAuthFilter(authToken) : new BearerAuthFilter(tokenManager);
     }
 
-    public static Keycloak getInstance(String serverUrl, String realm, String username, String password, String clientId, String clientSecret, SSLContext sslContext) {
-        return getInstance(serverUrl, realm, username, password, clientId, clientSecret, sslContext, null);
-    }
-
-    public static Keycloak getInstance(String serverUrl, String realm, String username, String password, String clientId, String clientSecret, SSLContext sslContext, ResteasyJackson2Provider customJacksonProvider) {
+    private static ResteasyClient newRestEasyClient(ResteasyJackson2Provider customJacksonProvider, SSLContext sslContext, boolean disableTrustManager) {
         ResteasyClientBuilder clientBuilder = new ResteasyClientBuilder()
-                .sslContext(sslContext)
-                .hostnameVerification(ResteasyClientBuilder.HostnameVerificationPolicy.WILDCARD)
-                .connectionPoolSize(10);
+              .sslContext(sslContext)
+              .connectionPoolSize(10);
+
+        if (disableTrustManager) {
+            // Disable PKIX path validation errors when running tests using SSL
+            clientBuilder.disableTrustManager().hostnameVerification(ResteasyClientBuilder.HostnameVerificationPolicy.ANY);
+        }
 
         if (customJacksonProvider != null) {
             clientBuilder.register(customJacksonProvider, 100);
         }
 
-        return new Keycloak(serverUrl, realm, username, password, clientId, clientSecret, PASSWORD, clientBuilder.build(), null);
+        return clientBuilder.build();
+    }
+
+    public static Keycloak getInstance(String serverUrl, String realm, String username, String password, String clientId, String clientSecret, SSLContext sslContext, ResteasyJackson2Provider customJacksonProvider, boolean disableTrustManager, String authToken) {
+        return new Keycloak(serverUrl, realm, username, password, clientId, clientSecret, PASSWORD, newRestEasyClient(customJacksonProvider, sslContext, disableTrustManager), authToken);
     }
 
     public static Keycloak getInstance(String serverUrl, String realm, String username, String password, String clientId, String clientSecret) {
-        return new Keycloak(serverUrl, realm, username, password, clientId, clientSecret, PASSWORD, null, null);
+        return getInstance(serverUrl, realm, username, password, clientId, clientSecret, null, null, false, null);
+    }
+
+    public static Keycloak getInstance(String serverUrl, String realm, String username, String password, String clientId, String clientSecret, SSLContext sslContext) {
+        return getInstance(serverUrl, realm, username, password, clientId, clientSecret, sslContext, null, false, null);
+    }
+
+    public static Keycloak getInstance(String serverUrl, String realm, String username, String password, String clientId, String clientSecret, SSLContext sslContext, ResteasyJackson2Provider customJacksonProvider) {
+        return getInstance(serverUrl, realm, username, password, clientId, clientSecret, sslContext, null, false, null);
     }
 
     public static Keycloak getInstance(String serverUrl, String realm, String username, String password, String clientId) {
-        return new Keycloak(serverUrl, realm, username, password, clientId, null, PASSWORD, null, null);
+        return getInstance(serverUrl, realm, username, password, clientId, null, null, null, false, null);
+    }
+
+    public static Keycloak getInstance(String serverUrl, String realm, String username, String password, String clientId, SSLContext sslContext) {
+        return getInstance(serverUrl, realm, username, password, clientId, null, sslContext, null, false, null);
     }
 
     public static Keycloak getInstance(String serverUrl, String realm, String clientId, String authToken) {
-        return new Keycloak(serverUrl, realm, null, null, clientId, null, PASSWORD, null, authToken);
+        return getInstance(serverUrl, realm, null, null, clientId, null, null, null, false, authToken);
+    }
+
+    public static Keycloak getInstance(String serverUrl, String realm, String clientId, String authToken, SSLContext sllSslContext) {
+        return getInstance(serverUrl, realm, null, null, clientId, null, sllSslContext, null, false, authToken);
     }
 
     public RealmsResource realms() {

integration/client-cli/admin-cli/src/main/java/org/keycloak/client/admin/cli/commands/AbstractAuthOptionsCmd.java

@@ -89,6 +89,9 @@ public abstract class AbstractAuthOptionsCmd extends AbstractGlobalOptionsCmd {
     @Option(name = "trustpass", description = "Truststore password (prompted for if not specified and --truststore is used)")
     String trustPass;
 
+    @Option(name = "insecure", description = "Turns off TLS validation", hasValue = false)
+    boolean insecure;
+
     @Option(name = "token", description = "Token to use for invocations.  With this option set, every other authentication option is ignored")
     String externalToken;
 
@@ -178,6 +181,10 @@ protected void setupTruststore(ConfigData configData, CommandInvocation invocati
                 throw new RuntimeException("Failed to load truststore: " + truststore, e);
             }
         }
+
+        if (insecure) {
+            HttpUtil.setSkipCertificateValidation();
+        }
     }
 
     protected ConfigData ensureAuthInfo(ConfigData config, CommandInvocation commandInvocation) {

integration/client-cli/admin-cli/src/main/java/org/keycloak/client/admin/cli/util/HttpUtil.java

@@ -30,9 +30,11 @@
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
 import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.ssl.SSLContexts;
 import org.keycloak.client.admin.cli.httpcomponents.HttpDelete;
 import org.keycloak.client.admin.cli.operations.LocalSearch;
@@ -53,6 +55,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.keycloak.common.util.ObjectUtil.capitalize;
 
@@ -68,6 +71,7 @@ public class HttpUtil {
 
     private static HttpClient httpClient;
     private static SSLConnectionSocketFactory sslsf;
+    private static final AtomicBoolean tlsWarningEmitted = new AtomicBoolean();
 
     public static InputStream doGet(String url, String acceptType, String authorization) {
         try {
@@ -257,11 +261,29 @@ public static void setTruststore(File file, String password) throws CertificateE
         }
         SSLContext theContext = SSLContexts.custom()
                 .useProtocol("TLS")
-                .loadTrustMaterial(file, password == null ? null : password.toCharArray())
+                .loadTrustMaterial(file, password == null ? null : password.toCharArray(), TrustSelfSignedStrategy.INSTANCE)
                 .build();
         sslsf = new SSLConnectionSocketFactory(theContext);
     }
 
+    public static void setSkipCertificateValidation() {
+        if (!tlsWarningEmitted.getAndSet(true)) {
+            // Since this is a static util, it may happen that TLS is setup many times in one command
+            // invocation (e.g. when a command requires logging in). However, we would like to
+            // prevent this warning from appearing multiple times. That's why we need to guard it with a boolean.
+            System.err.println("The server is configured to use TLS but there is no truststore specified.");
+            System.err.println("The tool will skip certificate validation. This is highly discouraged for production use cases");
+        }
+
+        SSLContextBuilder builder = new SSLContextBuilder();
+        try {
+            builder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
+            sslsf = new SSLConnectionSocketFactory(builder.build());
+        } catch (Exception e) {
+            throw new RuntimeException("Failed setting up TLS", e);
+        }
+    }
+
     public static String extractIdFromLocation(String location) {
         int last = location.lastIndexOf("/");
         if (last != -1) {

integration/client-cli/client-registration-cli/src/main/java/org/keycloak/client/registration/cli/commands/AbstractAuthOptionsCmd.java

@@ -25,7 +25,6 @@ public abstract class AbstractAuthOptionsCmd extends AbstractGlobalOptionsCmd {
 
     static final String DEFAULT_CLIENT = "admin-cli";
 
-
     @Option(name = "config", description = "Path to the config file (~/.keycloak/kcreg.config by default)", hasValue = true)
     protected String config;
 
@@ -68,6 +67,9 @@ public abstract class AbstractAuthOptionsCmd extends AbstractGlobalOptionsCmd {
     @Option(name = "trustpass", description = "Truststore password (prompted for if not specified and --truststore is used)", hasValue = true)
     protected String trustPass;
 
+    @Option(name = "insecure", description = "Turns off TLS validation", hasValue = false)
+    protected boolean insecure;
+
     @Option(shortName = 't', name = "token", description = "Initial / Registration access token to use)", hasValue = true)
     protected String token;
 
@@ -90,6 +92,7 @@ protected void initFromParent(AbstractAuthOptionsCmd parent) {
         trustStore = parent.trustStore;
         trustPass = parent.trustPass;
         token = parent.token;
+        insecure = parent.insecure;
     }
 
     protected void applyDefaultOptionValues() {
@@ -152,6 +155,10 @@ protected void setupTruststore(ConfigData configData, CommandInvocation invocati
                 throw new RuntimeException("Failed to load truststore: " + truststore, e);
             }
         }
+
+        if (insecure) {
+            HttpUtil.setSkipCertificateValidation();
+        }
     }
 
     protected ConfigData ensureAuthInfo(ConfigData config, CommandInvocation commandInvocation) {

integration/client-cli/client-registration-cli/src/main/java/org/keycloak/client/registration/cli/util/HttpUtil.java

@@ -28,6 +28,8 @@
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.HttpClientBuilder;
@@ -46,6 +48,7 @@
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateException;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * @author <a href="mailto:[email protected]">Marko Strukelj</a>
@@ -59,6 +62,7 @@ public class HttpUtil {
 
     private static HttpClient httpClient;
     private static SSLConnectionSocketFactory sslsf;
+    private static final AtomicBoolean tlsWarningEmitted = new AtomicBoolean();
 
     public static InputStream doGet(String url, String acceptType, String authorization) {
         try {
@@ -181,8 +185,26 @@ public static void setTruststore(File file, String password) throws CertificateE
         }
         SSLContext theContext = SSLContexts.custom()
                 .useProtocol("TLS")
-                .loadTrustMaterial(file, password == null ? null : password.toCharArray())
+                .loadTrustMaterial(file, password == null ? null : password.toCharArray(), TrustSelfSignedStrategy.INSTANCE)
                 .build();
         sslsf = new SSLConnectionSocketFactory(theContext);
     }
+
+    public static void setSkipCertificateValidation() {
+        if (!tlsWarningEmitted.getAndSet(true)) {
+            // Since this is a static util, it may happen that TLS is setup many times in one command
+            // invocation (e.g. when a command requires logging in). However, we would like to
+            // prevent this warning from appearing multiple times. That's why we need to guard it with a boolean.
+            System.err.println("The server is configured to use TLS but there is no truststore specified.");
+            System.err.println("The tool will skip certificate validation. This is highly discouraged for production use cases");
+        }
+
+        SSLContextBuilder builder = new SSLContextBuilder();
+        try {
+            builder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
+            sslsf = new SSLConnectionSocketFactory(builder.build());
+        } catch (Exception e) {
+            throw new RuntimeException("Failed setting up TLS", e);
+        }
+    }
 }

testsuite/integration-arquillian/HOW-TO-RUN.md

@@ -41,6 +41,10 @@ and adapter are all in the same JVM and you can debug them easily. If it is not
 
 
     -Dmaven.surefire.debug=true
+
+Or slightly longer version (that allows you to specify debugging port as well as wait till you attach the debugger):
+
+    -Dmaven.surefire.debug="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5006 -Xnoagent -Djava.compiler=NONE"
 
 
 and you will be able to attach remote debugger to the test. Unfortunately server and adapter are running in different JVMs, so this won't help to debug those. 
@@ -486,27 +490,17 @@ To run the X.509 client certificate authentication tests:
 	  -Dbrowser=phantomjs \
 	  "-Dtest=*.x509.*"
 
-## Run Mutual TLS Client Certificate Bound Access Tokens tests
+## Disabling TLS (SSL) in the tests
 
-To run the Mutual TLS Client Certificate Bound Access Tokens tests:
+All tests are executed with TLS by default. In order to disable it, you need to switch the `auth.server.ssl.required` property off.
+Here's an example:
 
     mvn -f testsuite/integration-arquillian/pom.xml \
           clean install \
-      -Pauth-server-wildfly \
-      -Dauth.server.ssl.required \
-      -Dbrowser=phantomjs \
-      -Dtest=org.keycloak.testsuite.hok.HoKTest
-
-## Run Mutual TLS for the Client tests
+      -Dauth.server.ssl.required=false
 
-To run the Mutual TLS test for the client:
-
-    mvn -f testsuite/integration-arquillian/pom.xml \
-          clean install \
-      -Pauth-server-wildfly \
-      -Dauth.server.ssl.required \
-      -Dbrowser=phantomjs \
-      -Dtest=org.keycloak.testsuite.client.MutualTLSClientTest
+NOTE: You can also do it ad-hoc from your IDE, however some tests (like AuthZ or JS Adapter tests) require rebuilt test applications.
+so please make sure you rebuild all `testsuite/integration-arquillian` child modules.
 
 ## Cluster tests
 

testsuite/integration-arquillian/servers/app-server/jboss/common/cli/add-secured-deployments.cli

@@ -3,17 +3,19 @@ embed-server --server-config=standalone.xml
 /subsystem=keycloak/secure-deployment=customer-portal-subsystem.war/:add( \
     realm=demo, \
     resource=customer-portal-subsystem, \
-    auth-server-url=${auth.server.actual.protocol:http}://localhost:${auth.server.actual.http.port:8180}/auth, \
+    auth-server-url=${auth.server.actual.protocol:https}://localhost:${auth.server.actual.https.port:8543}/auth, \
     ssl-required=EXTERNAL, \
+    disable-trust-manager=true, \
     realm-public-key=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCrVrCuTtArbgaZzL1hvh0xtL5mc7o0NqPVnYXkLvgcwiC3BjLGw1tGEGoJaXDuSaRllobm53JBhjx33UNv+5z/UMG4kytBWxheNVKnL6GgqlNabMaFfPLPCF8kAgKnsi79NMo+n6KnSY8YeUmec/p2vjO2NjsSAVcWEQMVhJ31LwIDAQAB \
 )
 /subsystem=keycloak/secure-deployment=customer-portal-subsystem.war/credential=secret/:add(value=password)
 
 /subsystem=keycloak/secure-deployment=product-portal-subsystem.war/:add( \
     realm=demo, \
     resource=product-portal-subsystem, \
-    auth-server-url=${auth.server.actual.protocol:http}://localhost:${auth.server.actual.http.port:8180}/auth, \
+    auth-server-url=${auth.server.actual.protocol:https}://localhost:${auth.server.actual.https.port:8543}/auth, \
     ssl-required=EXTERNAL, \
+    disable-trust-manager=true, \
     realm-public-key=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCrVrCuTtArbgaZzL1hvh0xtL5mc7o0NqPVnYXkLvgcwiC3BjLGw1tGEGoJaXDuSaRllobm53JBhjx33UNv+5z/UMG4kytBWxheNVKnL6GgqlNabMaFfPLPCF8kAgKnsi79NMo+n6KnSY8YeUmec/p2vjO2NjsSAVcWEQMVhJ31LwIDAQAB \
 )
 /subsystem=keycloak/secure-deployment=product-portal-subsystem.war/credential=secret/:add(value=password)