[DOP-15547] Update Docker images to latest versions

MobileTeleSystems · Apr 26, 2024 · 27ec89e · 27ec89e
1 parent 96ce940
commit 27ec89e
Show file tree

Hide file tree

Showing 46 changed files with 78 additions and 335 deletions.
diff --git a/.env.dependencies b/.env.dependencies
@@ -13,13 +13,6 @@ KAFKA_CFG_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://:9092,EXTERNAL_PLAINTEXT_ANON
 KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://kafka:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://localhost:9093,INTERNAL_PLAINTEXT_SASL://kafka:9094,EXTERNAL_PLAINTEXT_SASL://localhost:9095
 KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,EXTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,INTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT,EXTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT
 KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
-# old config names for <1.1.1
-KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
-KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL_PLAINTEXT_ANONYMOUS
-KAFKA_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://:9093,INTERNAL_PLAINTEXT_SASL://:9094,EXTERNAL_PLAINTEXT_SASL://:9095
-KAFKA_ADVERTISED_LISTENERS=INTERNAL_PLAINTEXT_ANONYMOUS://kafka:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://localhost:9093,INTERNAL_PLAINTEXT_SASL://kafka:9094,EXTERNAL_PLAINTEXT_SASL://localhost:9095
-KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,EXTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,INTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT,EXTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT
-KAFKA_SASL_ENABLED_MECHANISMS=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
 
 # Mongo
 MONGO_INITDB_ROOT_USERNAME=onetl

diff --git a/.github/workflows/data/clickhouse/matrix.yml b/.github/workflows/data/clickhouse/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -22,15 +22,15 @@ latest: &latest
 matrix:
   small:
     - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 23.6.1-alpine
+      clickhouse-version: 24.3.2.23-alpine
       <<: *max
   full:
   # the lowest supported Clickhouse version by JDBC driver
     - clickhouse-image: yandex/clickhouse-server
       clickhouse-version: '20.7'
       <<: *min
     - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 23.6.1-alpine
+      clickhouse-version: 24.3.2.23-alpine
       <<: *max
   nightly:
     - clickhouse-image: yandex/clickhouse-server

diff --git a/.github/workflows/data/core/matrix.yml b/.github/workflows/data/core/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/core/tracked.txt b/.github/workflows/data/core/tracked.txt
@@ -2,5 +2,7 @@ onetl/hooks/**
 onetl/plugins/**
 onetl/impl/**
 onetl/hwm/**
+onetl/_util/**
 onetl/_internal.py
 onetl/log.py
+.github/workflows/data/core/**
diff --git a/.github/workflows/data/ftp/matrix.yml b/.github/workflows/data/ftp/matrix.yml
@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/ftps image has only latest tag
+    # chonjay21/ftps image has only latest tag
     - ftp-version: latest
       <<: *max
   full:

diff --git a/.github/workflows/data/ftps/matrix.yml b/.github/workflows/data/ftps/matrix.yml
@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/ftps image has only latest tag
+    # chonjay21/ftps image has only latest tag
     - ftps-version: latest
       <<: *max
   full:

diff --git a/.github/workflows/data/greenplum/matrix.yml b/.github/workflows/data/greenplum/matrix.yml
@@ -28,14 +28,14 @@ matrix:
       package-version: 2.3.1
       <<: *max
   full:
-    - greenplum-version: 6.25.3
+    - greenplum-version: 6.23.1
       package-version: 2.2.0
       <<: *min
     - greenplum-version: 7.0.0
       package-version: 2.3.1
       <<: *max
   nightly:
-    - greenplum-version: 6.25.3
+    - greenplum-version: 6.23.1
       package-version: 2.2.0
       <<: *min
     - greenplum-version: 7.0.0

diff --git a/.github/workflows/data/hdfs/matrix.yml b/.github/workflows/data/hdfs/matrix.yml
@@ -8,7 +8,7 @@ min: &min
 
 max: &max
   hadoop-version: hadoop3-hdfs
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/hive/matrix.yml b/.github/workflows/data/hive/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/kafka/matrix.yml b/.github/workflows/data/kafka/matrix.yml
@@ -1,16 +1,18 @@
 min: &min
-  # kafka_version: 0.10.2-1-r3
-  kafka-version: 3.5.1
+  # Headers are supported only since 2.x.
+  # Images before 3.2.3 are not creating kafka_jaas.conf properly, and failing to start
+  # https://github.com/bitnami/containers/blob/9db9064668365cac89bff58259f63eb78bb97e79/bitnami/kafka/README.md?plain=1#L933
+  kafka-version: 3.2.3
   pydantic-version: 1
   spark-version: 2.4.8
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
-  kafka-version: 3.5.1
+  kafka-version: 3.7.0
   pydantic-version: 2
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/local-fs/matrix.yml b/.github/workflows/data/local-fs/matrix.yml
@@ -20,6 +20,7 @@ min_excel: &min_excel
   os: ubuntu-latest
 
 max: &max
+  # Excel package currently has no release for 3.5.1
   spark-version: 3.5.0
   pydantic-version: 2
   python-version: '3.12'

diff --git a/.github/workflows/data/mongodb/matrix.yml b/.github/workflows/data/mongodb/matrix.yml
@@ -7,7 +7,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.4.2
+  spark-version: 3.4.3
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -22,12 +22,12 @@ latest: &latest
 
 matrix:
   small:
-    - mongodb-version: 6.0.7
+    - mongodb-version: 7.0.8
       <<: *max
   full:
     - mongodb-version: 4.0.0
       <<: *min
-    - mongodb-version: 6.0.7
+    - mongodb-version: 7.0.8
       <<: *max
   nightly:
     - mongodb-version: 4.0.0

diff --git a/.github/workflows/data/mssql/matrix.yml b/.github/workflows/data/mssql/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/mysql/matrix.yml b/.github/workflows/data/mysql/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,17 +21,17 @@ latest: &latest
 
 matrix:
   small:
-    - mysql-version: 8.0.33
+    - mysql-version: 8.3.0
       <<: *max
   full:
-  # Min supported version by JDBC driver is 5.7
-    - mysql-version: 5.7.42
+    # Min supported version by JDBC driver is 5.7
+    - mysql-version: 5.7.6
       <<: *min
-  # Max supported version by JDBC driver is 8.0
-    - mysql-version: 8.0.33
+    # Max supported version by JDBC driver is 8.3
+    - mysql-version: 8.3.0
       <<: *max
   nightly:
-    - mysql-version: 5.7.42
+    - mysql-version: 5.7.6
       <<: *min
     - mysql-version: latest
       <<: *latest
diff --git a/.github/workflows/data/oracle/matrix.yml b/.github/workflows/data/oracle/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/postgres/matrix.yml b/.github/workflows/data/postgres/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,13 +21,13 @@ latest: &latest
 
 matrix:
   small:
-    - postgres-version: 15.2-alpine
+    - postgres-version: 16.2-alpine
       <<: *max
   full:
-  # Min supported version by JDBC driver is 8.4, but it is too ancient to be used by anyone in real life
+    # Min supported version by JDBC driver is 8.4, but it is too ancient to be used by anyone in real life
     - postgres-version: 9.4.26-alpine
       <<: *min
-    - postgres-version: 15.2-alpine
+    - postgres-version: 16.2-alpine
       <<: *max
   nightly:
     - postgres-version: 9.4.26-alpine

diff --git a/.github/workflows/data/s3/matrix.yml b/.github/workflows/data/s3/matrix.yml
@@ -9,8 +9,8 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  minio-version: 2023.7.18
-  spark-version: 3.5.0
+  minio-version: 2024.4.18
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/samba/matrix.yml b/.github/workflows/data/samba/matrix.yml
@@ -15,6 +15,7 @@ latest: &latest
 
 matrix:
   small:
+    # elswork/samba image versions does not correlate with smbd version, it is always 4.x
     - server-version: latest
       <<: *max
   full:

diff --git a/.github/workflows/data/sftp/matrix.yml b/.github/workflows/data/sftp/matrix.yml
@@ -15,13 +15,13 @@ latest: &latest
 
 matrix:
   small:
-    - openssh-version: 9.3_p1-r3-ls120
+    - openssh-version: 9.6_p1-r0-ls154
       <<: *max
   full:
-  # prior image versions does not accept incoming connections, seems like a bug
+    # prior image versions does not accept incoming connections, seems like a bug
     - openssh-version: 8.1_p1-r0-ls5
       <<: *min
-    - openssh-version: 9.3_p1-r3-ls120
+    - openssh-version: 9.6_p1-r0-ls154
       <<: *max
   nightly:
     - openssh-version: 8.1_p1-r0-ls5

diff --git a/.github/workflows/data/teradata/matrix.yml b/.github/workflows/data/teradata/matrix.yml
@@ -1,5 +1,5 @@
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/webdav/matrix.yml b/.github/workflows/data/webdav/matrix.yml
@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/webdav image has only latest tag
+    # chonjay21/webdav image has only latest tag
     - webdav-version: latest
       <<: *max
   full:

diff --git a/.github/workflows/test-clickhouse.yml b/.github/workflows/test-clickhouse.yml
@@ -83,10 +83,6 @@ jobs:
         run: |
           pip install -I -r requirements/core.txt -r requirements/tests/base.txt -r requirements/tests/clickhouse.txt -r requirements/tests/spark-${{ inputs.spark-version }}.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
 
-      - name: Wait for Clickhouse to be ready
-        run: |
-          ./docker/wait-for-it.sh -h localhost -p 8123 -t 60
-
       - name: Run tests
         run: |
           mkdir reports/ || echo "Directory exists"

diff --git a/.github/workflows/test-ftp.yml b/.github/workflows/test-ftp.yml
@@ -50,20 +50,15 @@ jobs:
         run: |
           pip install -I -r requirements/core.txt -r requirements/ftp.txt -r requirements/tests/base.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
 
-    # Replace with Github Actions' services after https://github.com/chonjay21/docker-ftp/pull/3
-    # Cannot use services because we need to mount config file from the repo, but services start before checkout.
-    # See https://github.com/orgs/community/discussions/25792
+      # Replace with Github Actions' services after https://github.com/chonjay21/docker-ftp/pull/3
+      # Cannot use services because we need to mount config file from the repo, but services start before checkout.
+      # See https://github.com/orgs/community/discussions/25792
       - name: Start FTP
         run: |
           docker compose down -v --remove-orphans
-          docker compose up -d ftp
+          docker compose up -d ftp --wait --wait --wait-timeout 200
         env:
           FTP_IMAGE: chonjay21/ftps:${{ inputs.ftp-version }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftp${{ inputs.ftp-version }}
-
-      - name: Wait for FTP to be ready
-        run: |
-          ./docker/wait-for-it.sh -h localhost -p 2121 -t 60
 
       - name: Run tests
         run: |
@@ -76,8 +71,6 @@ jobs:
         if: always()
         run: |
           docker compose down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftp${{ inputs.ftp-version }}
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-ftps.yml b/.github/workflows/test-ftps.yml
@@ -50,20 +50,15 @@ jobs:
         run: |
           pip install -I -r requirements/core.txt -r requirements/ftp.txt -r requirements/tests/base.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
 
-    # Replace with Github Actions' services after https://github.com/chonjay21/docker-ftps/pull/3
-    # Cannot use services because we need to mount config file from the repo, but services start before checkout.
-    # See https://github.com/orgs/community/discussions/25792
+      # Replace with Github Actions' services after https://github.com/chonjay21/docker-ftps/pull/3
+      # Cannot use services because we need to mount config file from the repo, but services start before checkout.
+      # See https://github.com/orgs/community/discussions/25792
       - name: Start FTPS
         run: |
           docker compose down -v --remove-orphans
-          docker compose up -d ftps
+          docker compose up -d ftps --wait --wait --wait-timeout 200
         env:
           FTPS_IMAGE: chonjay21/ftps:${{ inputs.ftps-version }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftps${{ inputs.ftps-version }}
-
-      - name: Wait for FTPS to be ready
-        run: |
-          ./docker/wait-for-it.sh -h localhost -p 2122 -t 60
 
       - name: Run tests
         run: |
@@ -76,8 +71,6 @@ jobs:
         if: always()
         run: |
           docker compose down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftps${{ inputs.ftps-version }}
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,6 +15,7 @@ latest: &latest @@
     matrix:
       small:
+        # elswork/samba image versions does not correlate with smbd version, it is always 4.x
         - server-version: latest
           <<: *max
       full:
@@ Expand Down @@