From 0939968bbfd6e9f7208989491f5321853626fdd9 Mon Sep 17 00:00:00 2001 From: Francesco Date: Thu, 10 Aug 2023 10:21:11 +0200 Subject: [PATCH 1/5] Prepping for 1.7.1 release --- sql-client/Dockerfile | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/sql-client/Dockerfile b/sql-client/Dockerfile index eaee310..5460fbb 100644 --- a/sql-client/Dockerfile +++ b/sql-client/Dockerfile @@ -21,20 +21,24 @@ #SQL CLI - inspired by https://github.com/wuchong/flink-sql-demo/tree/v1.11-EN/sql-client ############################################################################### -FROM flink:1.16.0-scala_2.12-java11 +FROM flink:1.17.1-scala_2.12-java11 # Create CLI lib folder -COPY ./sql-client/bin/* /opt/sql-client/ +COPY sql-client/bin/* /opt/sql-client/ RUN mkdir -p /opt/sql-client/lib # Download connector libraries -RUN wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch7/3.0.0-1.16/flink-sql-connector-elasticsearch7-3.0.0-1.16.jar; \ - wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.16.0/flink-sql-connector-kafka-1.16.0.jar; \ - wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/1.16.0/flink-connector-jdbc-1.16.0.jar; \ - wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/1.16.0/flink-json-1.16.0.jar; \ - wget -P /opt/sql-client/lib/ https://jdbc.postgresql.org/download/postgresql-42.5.0.jar; \ - wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/1.16.0/flink-sql-avro-confluent-registry-1.16.0.jar; \ - wget -P /opt/sql-client/lib/ https://github.com/knaufk/flink-faker/releases/download/v0.5.1/flink-faker-0.5.1.jar; +RUN wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch7/1.16.2/flink-sql-connector-elasticsearch7-1.16.2.jar; \ + wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.17.1/flink-sql-connector-kafka-1.17.1.jar; \ + wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/1.16.2/flink-connector-jdbc-1.16.2.jar; \ + wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/1.7.1/flink-json-1.7.1.jar; \ + wget -P /opt/sql-client/lib/ https://jdbc.postgresql.org/download/postgresql-42.6.0.jar; \ + wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/1.17.1/flink-sql-avro-confluent-registry-1.17.1.jar; \ + wget -P /opt/sql-client/lib/ https://github.com/knaufk/flink-faker/releases/download/v0.5.1/flink-faker-0.5.1.jar; \ + wget -P /opt/sql-client/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-table-common/1.17.1/flink-table-common-1.17.1.jar; \ + wget -P /opt/sql-client/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-table-api-scala_2.12/1.17.1/flink-table-api-scala_2.12-1.17.1.jar; \ + wget -P /opt/sql-client/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-table-api-scala-bridge_2.12/1.17.1/flink-table-api-scala-bridge_2.12-1.17.1.jar; \ + wget -P /opt/sql-client/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-table-planner_2.12/1.17.1/flink-table-planner_2.12-1.17.1.jar; # Copy configuration COPY sql-client/conf/* /opt/flink/conf/ @@ -43,4 +47,4 @@ WORKDIR /opt/sql-client ENV SQL_CLIENT_HOME /opt/sql-client COPY sql-client/docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] +ENTRYPOINT ["/docker-entrypoint.sh"] \ No newline at end of file From 7ccd13d9cf8925569fb81d51daa82adaa6bdcaa9 Mon Sep 17 00:00:00 2001 From: Francesco Date: Thu, 10 Aug 2023 12:05:54 +0200 Subject: [PATCH 2/5] Start of 1.7.1 support --- docker-compose.yml | 4 ++-- sql-client/Dockerfile | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e3c8fdf..cfd598c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: - ./settings/:/settings jobmanager: - image: flink:1.16.0-scala_2.12-java11 + image: flink:1.17.1-scala_2.12-java11 ports: - "8081:8081" command: jobmanager @@ -26,7 +26,7 @@ services: - ./data/:/data taskmanager: - image: flink:1.16.0-scala_2.12-java11 + image: flink:1.17.1-scala_2.12-java11 depends_on: - jobmanager command: taskmanager diff --git a/sql-client/Dockerfile b/sql-client/Dockerfile index 5460fbb..c6bb8b5 100644 --- a/sql-client/Dockerfile +++ b/sql-client/Dockerfile @@ -28,17 +28,13 @@ COPY sql-client/bin/* /opt/sql-client/ RUN mkdir -p /opt/sql-client/lib # Download connector libraries -RUN wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch7/1.16.2/flink-sql-connector-elasticsearch7-1.16.2.jar; \ +RUN wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch7/3.0.1-1.17/flink-sql-connector-elasticsearch7-3.0.1-1.17.jar; \ wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.17.1/flink-sql-connector-kafka-1.17.1.jar; \ - wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/1.16.2/flink-connector-jdbc-1.16.2.jar; \ + wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/3.1.1-1.17/flink-connector-jdbc-3.1.1-1.17.jar; \ wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/1.7.1/flink-json-1.7.1.jar; \ - wget -P /opt/sql-client/lib/ https://jdbc.postgresql.org/download/postgresql-42.6.0.jar; \ + wget -P /opt/sql-client/lib/ https://jdbc.postgresql.org/download/postgresql-42.5.0.jar; \ wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/1.17.1/flink-sql-avro-confluent-registry-1.17.1.jar; \ - wget -P /opt/sql-client/lib/ https://github.com/knaufk/flink-faker/releases/download/v0.5.1/flink-faker-0.5.1.jar; \ - wget -P /opt/sql-client/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-table-common/1.17.1/flink-table-common-1.17.1.jar; \ - wget -P /opt/sql-client/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-table-api-scala_2.12/1.17.1/flink-table-api-scala_2.12-1.17.1.jar; \ - wget -P /opt/sql-client/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-table-api-scala-bridge_2.12/1.17.1/flink-table-api-scala-bridge_2.12-1.17.1.jar; \ - wget -P /opt/sql-client/lib/ https://repo1.maven.org/maven2/org/apache/flink/flink-table-planner_2.12/1.17.1/flink-table-planner_2.12-1.17.1.jar; + wget -P /opt/sql-client/lib/ https://github.com/knaufk/flink-faker/releases/download/v0.5.1/flink-faker-0.5.1.jar; # Copy configuration COPY sql-client/conf/* /opt/flink/conf/ From 445b2e5e31bfb52566814b93120a7aa30ef213ca Mon Sep 17 00:00:00 2001 From: Francesco Date: Thu, 10 Aug 2023 12:15:31 +0200 Subject: [PATCH 3/5] Changed README --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f97d4c5..657bce7 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,18 @@ # SQL CLI for Apache Flink® on Docker® -This docker-compose provides an Apache Flink® SQL CLI image updated to the 1.16.0 version. It's inspired by [this](https://github.com/wuchong/flink-sql-demo/tree/v1.11-EN/sql-client). +This docker-compose provides an Apache Flink® SQL CLI image updated to the 1.17.1 version. It's inspired by [this](https://github.com/wuchong/flink-sql-demo/tree/v1.11-EN/sql-client). -It makes use of the `flink:1.16.0-scala_2.12` images and of the `ftisiot/flink-sql-client:1.16.0` which is based on the same `flink:1.16.0-scala_2.12` image. +It makes use of the `flink:1.17.1-scala_2.12-java11` images. The `sql-client` service maps a `~/kafkacerts/` folder to `/certs` which can be used to create and pass files like Keystores when SSL authentication is needed (e.g. with Apache Kafka®). Includes the SQL connectors to: -* [Elasticsearch® 7](https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch7/1.16.0/flink-sql-connector-elasticsearch7-1.16.0.jar) -* [Apache Kafka®](https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.16.0/flink-sql-connector-kafka-1.16.0.jar) -* [AVRO](https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/1.16.0/flink-sql-avro-confluent-registry-1.16.0.jar) -* [JDBC](https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/1.16.0/flink-connector-jdbc-1.16.0.jar) +* [Elasticsearch® 7](ttps://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch7/3.0.1-1.17/flink-sql-connector-elasticsearch7-3.0.1-1.17.jar) +* [Apache Kafka®](/opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.17.1/flink-sql-connector-kafka-1.17.1.jar) +* [AVRO](https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/1.17.1/flink-sql-avro-confluent-registry-1.17.1.jar) +* [JDBC](https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/3.1.1-1.17/flink-connector-jdbc-3.1.1-1.17.jar) * [PostgreSQL 42.5.0](https://jdbc.postgresql.org/download/postgresql-42.5.0.jar) * [Flink Faker](https://github.com/knaufk/flink-faker/releases/download/v0.5.0/flink-faker-0.5.1.jar) allowing to [generate fake data](https://github.com/knaufk/flink-faker) @@ -20,7 +20,7 @@ Includes the SQL connectors to: You need both [docker](https://docs.docker.com/engine/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed. -Clone the current repository, navigate to the `flink-sql-cli` folder, then execute +Clone the current repository, navigate to the `sql-cli-for-apache-flink-docker` folder, then execute ``` docker-compose up -d @@ -38,7 +38,7 @@ The result should be similar to the below Name Command State Ports ----------------------------------------------------------------------------------------------- flink_jobmanager_1 /docker-entrypoint.sh jobm ... Up 6123/tcp, 0.0.0.0:8081->8081/tcp -flink_sql-client_1 /docker-entrypoint.sh Up 6123/tcp, 8081/tcp +sql-client /docker-entrypoint.sh Up 6123/tcp, 8081/tcp flink_taskmanager_1 /docker-entrypoint.sh task ... Up 6123/tcp, 8081/tcp ``` From 00b518aa6939bc52c9cf8a8a053fa6b0137685c6 Mon Sep 17 00:00:00 2001 From: Francesco Tisiot Date: Thu, 10 Aug 2023 15:33:44 +0200 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Sergey Nuyanzin --- sql-client/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql-client/Dockerfile b/sql-client/Dockerfile index c6bb8b5..7a096ed 100644 --- a/sql-client/Dockerfile +++ b/sql-client/Dockerfile @@ -31,10 +31,10 @@ RUN mkdir -p /opt/sql-client/lib RUN wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch7/3.0.1-1.17/flink-sql-connector-elasticsearch7-3.0.1-1.17.jar; \ wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.17.1/flink-sql-connector-kafka-1.17.1.jar; \ wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/3.1.1-1.17/flink-connector-jdbc-3.1.1-1.17.jar; \ - wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/1.7.1/flink-json-1.7.1.jar; \ - wget -P /opt/sql-client/lib/ https://jdbc.postgresql.org/download/postgresql-42.5.0.jar; \ + wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/1.17.1/flink-json-1.17.1.jar; \ + wget -P /opt/sql-client/lib/ https://jdbc.postgresql.org/download/postgresql-42.5.4.jar; \ wget -P /opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/1.17.1/flink-sql-avro-confluent-registry-1.17.1.jar; \ - wget -P /opt/sql-client/lib/ https://github.com/knaufk/flink-faker/releases/download/v0.5.1/flink-faker-0.5.1.jar; + wget -P /opt/sql-client/lib/ https://github.com/knaufk/flink-faker/releases/download/v0.5.3/flink-faker-0.5.3.jar; # Copy configuration COPY sql-client/conf/* /opt/flink/conf/ From 6f14ff88fcbe8bcc9d7b8940c70145014ab6a846 Mon Sep 17 00:00:00 2001 From: Francesco Date: Thu, 10 Aug 2023 15:40:35 +0200 Subject: [PATCH 5/5] fixed link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 657bce7..ec21eb3 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ It makes use of the `flink:1.17.1-scala_2.12-java11` images. The `sql-client` service maps a `~/kafkacerts/` folder to `/certs` which can be used to create and pass files like Keystores when SSL authentication is needed (e.g. with Apache Kafka®). Includes the SQL connectors to: -* [Elasticsearch® 7](ttps://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch7/3.0.1-1.17/flink-sql-connector-elasticsearch7-3.0.1-1.17.jar) +* [Elasticsearch® 7](https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-elasticsearch7/3.0.1-1.17/flink-sql-connector-elasticsearch7-3.0.1-1.17.jar) * [Apache Kafka®](/opt/sql-client/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.17.1/flink-sql-connector-kafka-1.17.1.jar) * [AVRO](https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/1.17.1/flink-sql-avro-confluent-registry-1.17.1.jar) * [JDBC](https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/3.1.1-1.17/flink-connector-jdbc-3.1.1-1.17.jar)