Add https support

.docker/certs/ca.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDJzCCAg+gAwIBAgIUAS2uTuXHj86guZ1HFeHh/0HA8ZEwDQYJKoZIhvcNAQEL
+BQAwIzEhMB8GA1UEAwwYY2xpY2tob3VzZXNlcnZlci50ZXN0IENBMB4XDTI0MTEw
+NjIwNTc0MloXDTM0MTEwNDIwNTc0MlowIzEhMB8GA1UEAwwYY2xpY2tob3VzZXNl
+cnZlci50ZXN0IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1pLT
+Lavh2ScoiVVVr2ygZwbilo3zJ2H3aRM7C1yaN8kuNe7T7gsVAbYg8+UE6tFtiRtg
+Xmsw2Hk3nlS73Lr8kfr/vwfGkmImnEYEyiWU6MYYq5ZVW1f7Iy3XKU2LfGwT45Rh
+c0fmBklu0Jhuyjo01R7zvcpuw+wq4a5wW/qf86aR2kLpNnScIjhTh4Xf5ndbOGO1
+L/bcRqQqmzFcCNLcQc+xyVVwUONwTCFGH+M/HhMAOCakUZ9a2hSWRTp1fzqbgZhu
++mvSSogtsyhMN9+YOJvaAvwP+fO5nZ/TTbTA60q887A/kUOy1hILrsfZvYk2cEsC
+N8ELiGYVLq7gdvC57QIDAQABo1MwUTAdBgNVHQ4EFgQUuv4mhc/eLXxs2fY7M1qy
+DBM9m6swHwYDVR0jBBgwFoAUuv4mhc/eLXxs2fY7M1qyDBM9m6swDwYDVR0TAQH/
+BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEA0AZOQB80+PCDM2HQIZFqgdDQybKj
+wV8je+9I3py858+lX5YEqfIEMeh6qkmNBIrPgTAsBDBnBUUEdO60ft7AjZCI07jG
+e1T66Ia/GX1f2gb0vLFm5OakdmeS8PCbxi39h0JbM8BJ0B12KE1hKbE5vwXfTudj
+AZjXIVe0jTSzybxi06CvXIPJjfnrObXQTOM1A/AuPEc9utUHwniVxYLcF9LPicbU
+K9oWelQ8RC0ejI4X2DLxfr2n9krqv5QZUMOIxKqAaB5kfzrUTc9C4uY0uKwxaofH
+cFG5CaPMMjh6vikB3b0RXhHpP7i2W3Qy61gUOTNqydpRrO3SOgeYHhtMYA==
+-----END CERTIFICATE-----

.docker/certs/server.crt

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyjCCAbICFCKq9CquubJSpLtCXcTWIp4vX9AgMA0GCSqGSIb3DQEBCwUAMCMx
+ITAfBgNVBAMMGGNsaWNraG91c2VzZXJ2ZXIudGVzdCBDQTAeFw0yNDExMDYyMDU3
+NDJaFw0zNDExMDQyMDU3NDJaMCAxHjAcBgNVBAMMFWNsaWNraG91c2VzZXJ2ZXIu
+dGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKYRSFVa9ZWsIRI1
+ponZPLZ+5J3GLRbqSE+B1wXiiU10fJTopbfRr6uqam63lInqW3PQ2cw0LAgRzF4K
+F9nydFguEkVXerVTWIfSSBDOh9p6l+gF8+rPlo3rzAnloO3WA6f6dtfqd6McwBk6
+mPpXmX4lftrduJS8JN8qESKZJ/vLkSG1tR+XZT9JsRakuJi3gmTzrQY+QdPGingz
+60GzLtUIEd+mY7hfKKd4heinowmyW72qqqbhYKK62d8Ox1Cx1igmfSlsCEcJxo03
+RJXzLhSz2h//EfGjDXZjJJEsRjHuHaPSYhJF/XEw82mwM2KZQR+LTfyU2PgXOQIO
+4XG/jC0CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAU+rnHlTp18kKaqXS2jN2qBY0
+jLryZKIAp/LBX8Viz9gfb+SIibavjU3491yWBdkeN8aF0gJpusUv4aOt4tmesX5j
+rhi2Z6nEl275QkO50ALJmAifjELtpju8nKQbQSj45rzEq4o+mdC8+mWTVvnW7rWz
+TM22EKhLNQ0n49epUvb8mxY2eddUhtLfHq8nOzmZUr8QtFq6ix9tqeBxdb3HNOcJ
+ZqM5HqXI5EjJMOqt0WdaOGXrO4RF/JjVh/feznJ/WSJAtYew5/FeCSATug7nmTsl
+IbsHTF989zBAJQYR3eiKQhiVsnUZlrjRSXiNnPZkRHVgaqt06iSvFCsBvkTUlw==
+-----END CERTIFICATE-----

.docker/certs/server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCmEUhVWvWVrCES
+NaaJ2Ty2fuSdxi0W6khPgdcF4olNdHyU6KW30a+rqmput5SJ6ltz0NnMNCwIEcxe
+ChfZ8nRYLhJFV3q1U1iH0kgQzofaepfoBfPqz5aN68wJ5aDt1gOn+nbX6nejHMAZ
+Opj6V5l+JX7a3biUvCTfKhEimSf7y5EhtbUfl2U/SbEWpLiYt4Jk860GPkHTxop4
+M+tBsy7VCBHfpmO4XyineIXop6MJslu9qqqm4WCiutnfDsdQsdYoJn0pbAhHCcaN
+N0SV8y4Us9of/xHxow12YySRLEYx7h2j0mISRf1xMPNpsDNimUEfi038lNj4FzkC
+DuFxv4wtAgMBAAECggEAG7pOAAZwc92qJe60UFbnxrbYOCmzNJzIlxHgU5i4ASk+
+deSEavVFZh3NqI/Zas+pFmTcQLbDVVOXcmowAlFzsJd8sHxDPkxcv0zSgI/jagzH
+GVQJHn8rraWdy1c+cAeoU59DwZhiAmeWAYBhs4b113SuQh/8JkVlyKcu9j5pSSa2
+qYcDopblbdMFpz5/ouzeSnDw5+tlwQ6J0HCZ0M9O17w8vtLwh2HmNqBLbMZzXZ4A
+XvV6xYNU9uh2GLsXuNBKPbAqk5swKOIZi1j118BZa1TcWlD4zrrgs//Mh5aepGBO
+nkm06WBV6/KG1TI+6Kb3KLCZb+pKwJSYvQDGy9XeAQKBgQDCGa+8+8qzfV2i69ZH
+hPQBHyPo+DfWsEC900pVDO+inhT97G1iw6nYqfCuiyTQo5Cj+1X7iXXJjlE3TgJm
+HhjAlW6OI2t0kvA8v07nsW3XkgxdF0praLS4SjMVrCCiu/waU/hvi8PA8c1HGGi+
+nU9e513QYyhZCS/pq+Ty+DOxAQKBgQDbBwBiwp2ugBzwk/amGpJP1+5ztTXiCIeX
+lbUqCpRqNX9F5aM1eF2LNtkmQ2qyxG2uhN0nS5mA30vFa7GbrPogIvBzKfFIN8LH
+kYNorcWimTNT7MCyMUz0ltJ6ONTLklMgM9oDTDbZwJesdt+CdYm1rvK7JrlxKYj2
+R/ZxnOpvLQKBgQCpsWtC/Z8pbOEAHs8nl8T7dg4fD2NB/rWbP64szTJ0KWCQ3rKD
+Niy3j1ssPdnhzh4WPxXc40BnBlVa1IikXoulUv9kpPK85/nAIkEJrsUpQjYs7L/T
+R6c3tycSO3oMy+yL1/3LB5YMiVm+Xki03TMW9Y+/L/GGZdHNqMpw4S3QAQKBgEY1
+qFoAKKuXRa+YvqfgPKZ7gWR14h/ivBRLWRdw7MJfA71UL00kmFfX7SPm2H0VmlMn
+I/1ErqAw9AkEzJoqiMNkb8960m1j8nwmmQklKwVENr/gVKTKSPDep/ml3hCjhyH7
+fRcxhyOmKPiP4nXfV5V1zzlO4IL13YGU79rbggrBAoGAD8Z0Hd87caomIwcBI8Zr
+ZcKhWhyfAPyTX6BKjdknwfWllap+xRJ7Peg68ZZv7Of6QZfpRzSUn0sccCr1NpB8
+pArn21raU6pPP2MZxPTtVKzj7Vicq/HWM+u0egLqre04EicoRIO3SG++wsDWgtHi
+qDZzWZ12JxPc45kUCt/Txcw=
+-----END PRIVATE KEY-----

.docker/docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3'
+services:
+  clickhouse:
+      # https://docs.docker.com/compose/compose-file/#variable-substitution
+      image: "clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-22.3}"
+      volumes:
+        - ./entrypoint.sh:/custom-entrypoint.sh
+        - ./server-config.xml:/etc/clickhouse-server/config.d/server-config.xml
+        - ./certs/server.crt:/tmp/certs/server.crt
+        - ./certs/server.key:/tmp/certs/server.key
+        - ./certs/ca.crt:/tmp/certs/ca.crt
+      expose:
+        - "8123"
+        - "8447"
+      ports:
+        - "8123:8123"
+        - "8447:8447"
+        - "9000:9000"
+      entrypoint:
+        - /custom-entrypoint.sh
+      hostname: clickhouseserver.test

.docker/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+CERTS_DIR=/etc/clickhouse-server/certs/
+mkdir -p $CERTS_DIR
+# Copy cert files to $CERTS_DIR and apply required rights so as not to affect the original files
+cp /tmp/certs/* $CERTS_DIR
+chown clickhouse:clickhouse $CERTS_DIR*
+chmod 644 $CERTS_DIR*
+/entrypoint.sh

.docker/generate-certs.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Script is used to (re)generate self-signed certificates needed to run the tests
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CERTS_DIR=$SCRIPT_DIR/certs
+rm -rf $CERTS_DIR
+mkdir -p $CERTS_DIR
+cd $CERTS_DIR
+openssl genrsa -out ca.key 2048
+openssl req -x509 -subj "/CN=clickhouseserver.test CA" -nodes -key ca.key -days 3650 -out ca.crt
+openssl req -newkey rsa:2048 -nodes -subj "/CN=clickhouseserver.test" -keyout server.key -out server.csr
+openssl x509 -req -in server.csr -out server.crt -CA ca.crt -CAkey ca.key -days 3650 -copy_extensions copy
+openssl req -newkey rsa:2048 -nodes -subj "/CN=clickhouseserver.test" -keyout client.key -out client.csr
+openssl x509 -req -in client.csr -out client.crt -CA ca.crt -CAkey ca.key -days 3650 -copy_extensions copy
+
+openssl pkcs12 -export -in client.crt -inkey client.key -out keystore.p12 -name client -CAfile ca.crt -caname 'clickhouseserver.test CA' -password pass:password
+
+keytool -importkeystore -deststorepass password -destkeypass password -destkeystore keystore.jks -deststoretype JKS -srckeystore keystore.p12 -srcstoretype PKCS12 -srcstorepass password -alias client -noprompt
+
+keytool -importcert -alias ca -file ca.crt -keystore keystore.jks -storepass password -noprompt
+rm ca.key client.key client.csr client.crt server.csr keystore.p12 

.docker/server-config.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!--
+  NOTE: User and query level settings are set up in "users.xml" file.
+  If you have accidentally specified user-level settings here, server won't start.
+  You can either move the settings to the right place inside "users.xml" file
+   or add <skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings> here.
+-->
+<yandex>
+
+    <!-- HTTP API with TLS (HTTPS).
+         You have to configure certificate to enable this interface.
+         See the openSSL section below.
+    -->
+    <https_port>8447</https_port>
+
+    <!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
+    <openSSL>
+        <server> <!-- Used for https server AND secure tcp port -->
+            <!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
+            <certificateFile>/etc/clickhouse-server/certs/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/certs/server.key</privateKeyFile>
+            <caConfig>/etc/clickhouse-server/certs/ca.crt</caConfig>
+            <!-- dhparams are optional. You can delete the <dhParamsFile> element.
+                 To generate dhparams, use the following command:
+                  openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096
+                 Only file format with BEGIN DH PARAMETERS is supported.
+              -->
+            <dhParamsFile remove="remove">/etc/clickhouse-server/dhparam.pem</dhParamsFile>
+            <verificationMode>relaxed</verificationMode>
+            <cacheSessions>true</cacheSessions>
+            <preferServerCiphers>true</preferServerCiphers>
+            <loadDefaultCAFile>false</loadDefaultCAFile>
+        </server>
+    </openSSL>
+</yandex>

.docker/update-hosts.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Add 'clickhouseserver.test' to '/etc/hosts' file if it is not already there
+if ! grep -q '127.0.0.1[[:space:]]*clickhouseserver.test'  /etc/hosts; then 
+  echo '127.0.0.1   clickhouseserver.test' | tee -a /etc/hosts  
+fi

.github/workflows/ci.yml

@@ -33,12 +33,15 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Update '/etc/hosts' file
+        run: ./.docker/update-hosts.sh
+
       - name: Docker Compose Action
         uses: hoverkraft-tech/[email protected]
         env:
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
         with:
-          compose-file: './docker-compose.yml'
+          compose-file: './.docker/docker-compose.yml'
           down-flags: '--volumes'
 
       - name: Setup Java (temurin@17)

.gitignore

@@ -15,4 +15,10 @@ credentials.sbt
 # Scala-IDE specific
 .scala_dependencies
 .worksheet
-.idea
+.idea
+# vscode specific
+.lh
+.vscode
+.bloop
+.metals
+metals.sbt

client/src/main/scala/com/crobox/clickhouse/ClickhouseClient.scala

@@ -2,6 +2,7 @@ package com.crobox.clickhouse
 
 import org.apache.pekko.NotUsed
 import org.apache.pekko.actor.ActorSystem
+import org.apache.pekko.http.scaladsl.HttpsConnectionContext
 import org.apache.pekko.http.scaladsl.model._
 import org.apache.pekko.stream.scaladsl.{Framing, Source}
 import org.apache.pekko.util.ByteString
@@ -20,7 +21,8 @@ import scala.concurrent.{Await, ExecutionContext, Future}
  * @author Sjoerd Mulder
  * @since 31-03-17
  */
-class ClickhouseClient(configuration: Option[Config] = None)
+class ClickhouseClient(configuration: Option[Config] = None, 
+                        override val customConnectionContext: Option[HttpsConnectionContext] = None)
     extends ClickHouseExecutor
     with ClickhouseResponseParser
     with ClickhouseQueryBuilder {

client/src/main/scala/com/crobox/clickhouse/internal/ClickHouseExecutor.scala

@@ -1,7 +1,7 @@
 package com.crobox.clickhouse.internal
 
 import org.apache.pekko.actor.{ActorSystem, Terminated}
-import org.apache.pekko.http.scaladsl.Http
+import org.apache.pekko.http.scaladsl.{Http, HttpsConnectionContext}
 import org.apache.pekko.http.scaladsl.model._
 import org.apache.pekko.http.scaladsl.settings.{ClientConnectionSettings, ConnectionPoolSettings}
 import org.apache.pekko.stream._
@@ -23,6 +23,7 @@ private[clickhouse] trait ClickHouseExecutor extends LazyLogging {
   protected implicit val executionContext: ExecutionContext
   protected val hostBalancer: HostBalancer
   protected val config: Config
+  protected val customConnectionContext: Option[HttpsConnectionContext]
 
   lazy val (progressQueue, progressSource) = {
     val builtSource = QueryProgress.queryProgressStream.run()
@@ -35,7 +36,8 @@ private[clickhouse] trait ClickHouseExecutor extends LazyLogging {
       ClientConnectionSettings(system).withTransport(new StreamingProgressClickhouseTransport(progressQueue))
     )
   private lazy val http              = Http()
-  private lazy val pool              = http.superPool[Promise[HttpResponse]](settings = superPoolSettings)
+  private lazy val connectionContext = customConnectionContext.getOrElse(http.defaultClientHttpsContext)
+  private lazy val pool              = http.superPool[Promise[HttpResponse]](connectionContext = connectionContext, settings = superPoolSettings)
   private lazy val bufferSize: Int   = config.getInt("buffer-size")
   private lazy val queryRetries: Int = config.getInt("retries")
 
@@ -90,7 +92,7 @@ private[clickhouse] trait ClickHouseExecutor extends LazyLogging {
         case QueueOfferResult.Failure(e)  => Future.failed(e)
       }
     } else {
-      http.singleRequest(request)
+      http.singleRequest(request, connectionContext = connectionContext)
     }
   }
 

client/src/test/scala/com/crobox/clickhouse/ClickhouseClientTest.scala

@@ -5,6 +5,13 @@ import com.crobox.clickhouse.internal.QuerySettings
 import com.crobox.clickhouse.internal.progress.QueryProgress.{Progress, QueryAccepted, QueryFinished, QueryProgress}
 import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
 import org.apache.pekko.http.scaladsl.model.headers.HttpEncodings.gzip
+import org.apache.pekko.http.scaladsl.ConnectionContext
+import java.security.KeyStore
+import javax.net.ssl.SSLContext
+import javax.net.ssl.TrustManagerFactory
+import java.io.FileInputStream
+import javax.net.ssl.KeyManagerFactory
+import java.security.SecureRandom
 
 /**
  * @author Sjoerd Mulder
@@ -29,6 +36,43 @@ class ClickhouseClientTest extends ClickhouseClientAsyncSpec {
       )
   }
 
+  it should "support SSL certs" in {
+    def createConnectionContext() = {
+      val keyStoreResource = "../.docker/certs/keystore.jks"
+      val password = "password"
+
+      val keyStore = KeyStore.getInstance("JKS")
+      val in = new FileInputStream(keyStoreResource)
+      keyStore.load(in, password.toCharArray)
+
+      val keyManagerFactory = KeyManagerFactory.getInstance("SunX509")
+      keyManagerFactory.init(keyStore, password.toCharArray)
+      val trustManagerFactory = TrustManagerFactory.getInstance("SunX509")
+      trustManagerFactory.init(keyStore)
+      val context = SSLContext.getInstance("TLS")
+      context.init(keyManagerFactory.getKeyManagers, trustManagerFactory.getTrustManagers, new SecureRandom())
+
+      val connectionContext = ConnectionContext.httpsClient(context)
+      connectionContext
+    }
+
+    new ClickhouseClient(
+      Some(config
+      .withValue("crobox.clickhouse.client.connection.port", ConfigValueFactory.fromAnyRef(8447))
+      .withValue("crobox.clickhouse.client.connection.host", ConfigValueFactory.fromAnyRef("https://clickhouseserver.test"))),
+      customConnectionContext = Some(createConnectionContext()))
+      .query("select 1 + 2")
+      .map { f =>
+        f.trim.toInt should be(3)
+      }
+      .flatMap(
+        _ =>
+          client.query("select currentDatabase()").map { f =>
+            f.trim should be("default")
+        }
+      )
+  }
+
   it should "support response compression" in {
     val client: ClickhouseClient = new ClickhouseClient(
       Some(config.resolveWith(ConfigFactory.parseString("crobox.clickhouse.client.http-compression = true")))

client/src/test/scala/com/crobox/clickhouse/internal/ClickhouseExecutorTest.scala

@@ -1,5 +1,6 @@
 package com.crobox.clickhouse.internal
 import org.apache.pekko.actor.ActorSystem
+import org.apache.pekko.http.scaladsl.HttpsConnectionContext
 import org.apache.pekko.http.scaladsl.model.{HttpResponse, Uri}
 import org.apache.pekko.stream.scaladsl.{Sink, SourceQueue}
 import org.apache.pekko.stream.{Materializer, StreamTcpException}
@@ -19,6 +20,7 @@ class ClickhouseExecutorTest extends ClickhouseClientAsyncSpec {
   private var response: Uri => Future[String] = _
   private lazy val executor = {
     new ClickHouseExecutor with ClickhouseResponseParser with ClickhouseQueryBuilder {
+      override protected val customConnectionContext: Option[HttpsConnectionContext] = None
       override protected implicit val system: ActorSystem                = self.system
       override protected implicit val executionContext: ExecutionContext = system.dispatcher
       override protected val config: Config                              = self.config.getConfig("crobox.clickhouse.client")