[DOP-15547] Update Docker images to latest versions

MobileTeleSystems · Apr 26, 2024 · ed12abe · ed12abe
1 parent 96ce940
commit ed12abe
Show file tree

Hide file tree

Showing 44 changed files with 72 additions and 318 deletions.
diff --git a/.github/workflows/data/clickhouse/matrix.yml b/.github/workflows/data/clickhouse/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -22,15 +22,15 @@ latest: &latest
 matrix:
   small:
     - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 23.6.1-alpine
+      clickhouse-version: 24.3.2.23-alpine
       <<: *max
   full:
   # the lowest supported Clickhouse version by JDBC driver
     - clickhouse-image: yandex/clickhouse-server
       clickhouse-version: '20.7'
       <<: *min
     - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 23.6.1-alpine
+      clickhouse-version: 24.3.2.23-alpine
       <<: *max
   nightly:
     - clickhouse-image: yandex/clickhouse-server

diff --git a/.github/workflows/data/core/matrix.yml b/.github/workflows/data/core/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/core/tracked.txt b/.github/workflows/data/core/tracked.txt
@@ -2,5 +2,7 @@ onetl/hooks/**
 onetl/plugins/**
 onetl/impl/**
 onetl/hwm/**
+onetl/_util/**
 onetl/_internal.py
 onetl/log.py
+.github/workflows/data/core/**
diff --git a/.github/workflows/data/ftp/matrix.yml b/.github/workflows/data/ftp/matrix.yml
@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/ftps image has only latest tag
+    # chonjay21/ftps image has only latest tag
     - ftp-version: latest
       <<: *max
   full:

diff --git a/.github/workflows/data/ftps/matrix.yml b/.github/workflows/data/ftps/matrix.yml
@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/ftps image has only latest tag
+    # chonjay21/ftps image has only latest tag
     - ftps-version: latest
       <<: *max
   full:

diff --git a/.github/workflows/data/greenplum/matrix.yml b/.github/workflows/data/greenplum/matrix.yml
@@ -28,14 +28,14 @@ matrix:
       package-version: 2.3.1
       <<: *max
   full:
-    - greenplum-version: 6.25.3
+    - greenplum-version: 6.23.1
       package-version: 2.2.0
       <<: *min
     - greenplum-version: 7.0.0
       package-version: 2.3.1
       <<: *max
   nightly:
-    - greenplum-version: 6.25.3
+    - greenplum-version: 6.23.1
       package-version: 2.2.0
       <<: *min
     - greenplum-version: 7.0.0

diff --git a/.github/workflows/data/hdfs/matrix.yml b/.github/workflows/data/hdfs/matrix.yml
@@ -8,7 +8,7 @@ min: &min
 
 max: &max
   hadoop-version: hadoop3-hdfs
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/hive/matrix.yml b/.github/workflows/data/hive/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/kafka/matrix.yml b/.github/workflows/data/kafka/matrix.yml
@@ -8,9 +8,9 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  kafka-version: 3.5.1
+  kafka-version: 3.7.0
   pydantic-version: 2
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/local-fs/matrix.yml b/.github/workflows/data/local-fs/matrix.yml
@@ -20,6 +20,7 @@ min_excel: &min_excel
   os: ubuntu-latest
 
 max: &max
+  # Excel package currently has no release for 3.5.1
   spark-version: 3.5.0
   pydantic-version: 2
   python-version: '3.12'

diff --git a/.github/workflows/data/mongodb/matrix.yml b/.github/workflows/data/mongodb/matrix.yml
@@ -7,7 +7,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.4.2
+  spark-version: 3.4.3
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -22,12 +22,12 @@ latest: &latest
 
 matrix:
   small:
-    - mongodb-version: 6.0.7
+    - mongodb-version: 7.0.8
       <<: *max
   full:
     - mongodb-version: 4.0.0
       <<: *min
-    - mongodb-version: 6.0.7
+    - mongodb-version: 7.0.8
       <<: *max
   nightly:
     - mongodb-version: 4.0.0

diff --git a/.github/workflows/data/mssql/matrix.yml b/.github/workflows/data/mssql/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/mysql/matrix.yml b/.github/workflows/data/mysql/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,17 +21,17 @@ latest: &latest
 
 matrix:
   small:
-    - mysql-version: 8.0.33
+    - mysql-version: 8.3.0
       <<: *max
   full:
-  # Min supported version by JDBC driver is 5.7
-    - mysql-version: 5.7.42
+    # Min supported version by JDBC driver is 5.7
+    - mysql-version: 5.7.6
       <<: *min
-  # Max supported version by JDBC driver is 8.0
-    - mysql-version: 8.0.33
+    # Max supported version by JDBC driver is 8.3
+    - mysql-version: 8.3.0
       <<: *max
   nightly:
-    - mysql-version: 5.7.42
+    - mysql-version: 5.7.6
       <<: *min
     - mysql-version: latest
       <<: *latest
diff --git a/.github/workflows/data/oracle/matrix.yml b/.github/workflows/data/oracle/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/postgres/matrix.yml b/.github/workflows/data/postgres/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,13 +21,13 @@ latest: &latest
 
 matrix:
   small:
-    - postgres-version: 15.2-alpine
+    - postgres-version: 16.2-alpine
       <<: *max
   full:
-  # Min supported version by JDBC driver is 8.4, but it is too ancient to be used by anyone in real life
+    # Min supported version by JDBC driver is 8.4, but it is too ancient to be used by anyone in real life
     - postgres-version: 9.4.26-alpine
       <<: *min
-    - postgres-version: 15.2-alpine
+    - postgres-version: 16.2-alpine
       <<: *max
   nightly:
     - postgres-version: 9.4.26-alpine

diff --git a/.github/workflows/data/s3/matrix.yml b/.github/workflows/data/s3/matrix.yml
@@ -9,8 +9,8 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  minio-version: 2023.7.18
-  spark-version: 3.5.0
+  minio-version: 2024.4.18
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/sftp/matrix.yml b/.github/workflows/data/sftp/matrix.yml
@@ -15,13 +15,13 @@ latest: &latest
 
 matrix:
   small:
-    - openssh-version: 9.3_p1-r3-ls120
+    - openssh-version: 9.6_p1-r0-ls154
       <<: *max
   full:
-  # prior image versions does not accept incoming connections, seems like a bug
+    # prior image versions does not accept incoming connections, seems like a bug
     - openssh-version: 8.1_p1-r0-ls5
       <<: *min
-    - openssh-version: 9.3_p1-r3-ls120
+    - openssh-version: 9.6_p1-r0-ls154
       <<: *max
   nightly:
     - openssh-version: 8.1_p1-r0-ls5

diff --git a/.github/workflows/data/teradata/matrix.yml b/.github/workflows/data/teradata/matrix.yml
@@ -1,5 +1,5 @@
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/webdav/matrix.yml b/.github/workflows/data/webdav/matrix.yml
@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/webdav image has only latest tag
+    # chonjay21/webdav image has only latest tag
     - webdav-version: latest
       <<: *max
   full:

diff --git a/.github/workflows/test-clickhouse.yml b/.github/workflows/test-clickhouse.yml
@@ -83,10 +83,6 @@ jobs:
         run: |
           pip install -I -r requirements/core.txt -r requirements/tests/base.txt -r requirements/tests/clickhouse.txt -r requirements/tests/spark-${{ inputs.spark-version }}.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
 
-      - name: Wait for Clickhouse to be ready
-        run: |
-          ./docker/wait-for-it.sh -h localhost -p 8123 -t 60
-
       - name: Run tests
         run: |
           mkdir reports/ || echo "Directory exists"

diff --git a/.github/workflows/test-ftp.yml b/.github/workflows/test-ftp.yml
@@ -50,20 +50,15 @@ jobs:
         run: |
           pip install -I -r requirements/core.txt -r requirements/ftp.txt -r requirements/tests/base.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
 
-    # Replace with Github Actions' services after https://github.com/chonjay21/docker-ftp/pull/3
-    # Cannot use services because we need to mount config file from the repo, but services start before checkout.
-    # See https://github.com/orgs/community/discussions/25792
+      # Replace with Github Actions' services after https://github.com/chonjay21/docker-ftp/pull/3
+      # Cannot use services because we need to mount config file from the repo, but services start before checkout.
+      # See https://github.com/orgs/community/discussions/25792
       - name: Start FTP
         run: |
           docker compose down -v --remove-orphans
-          docker compose up -d ftp
+          docker compose up -d ftp --wait --wait --wait-timeout 200
         env:
           FTP_IMAGE: chonjay21/ftps:${{ inputs.ftp-version }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftp${{ inputs.ftp-version }}
-
-      - name: Wait for FTP to be ready
-        run: |
-          ./docker/wait-for-it.sh -h localhost -p 2121 -t 60
 
       - name: Run tests
         run: |
@@ -76,8 +71,6 @@ jobs:
         if: always()
         run: |
           docker compose down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftp${{ inputs.ftp-version }}
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-ftps.yml b/.github/workflows/test-ftps.yml
@@ -50,20 +50,15 @@ jobs:
         run: |
           pip install -I -r requirements/core.txt -r requirements/ftp.txt -r requirements/tests/base.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
 
-    # Replace with Github Actions' services after https://github.com/chonjay21/docker-ftps/pull/3
-    # Cannot use services because we need to mount config file from the repo, but services start before checkout.
-    # See https://github.com/orgs/community/discussions/25792
+      # Replace with Github Actions' services after https://github.com/chonjay21/docker-ftps/pull/3
+      # Cannot use services because we need to mount config file from the repo, but services start before checkout.
+      # See https://github.com/orgs/community/discussions/25792
       - name: Start FTPS
         run: |
           docker compose down -v --remove-orphans
-          docker compose up -d ftps
+          docker compose up -d ftps --wait --wait --wait-timeout 200
         env:
           FTPS_IMAGE: chonjay21/ftps:${{ inputs.ftps-version }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftps${{ inputs.ftps-version }}
-
-      - name: Wait for FTPS to be ready
-        run: |
-          ./docker/wait-for-it.sh -h localhost -p 2122 -t 60
 
       - name: Run tests
         run: |
@@ -76,8 +71,6 @@ jobs:
         if: always()
         run: |
           docker compose down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftps${{ inputs.ftps-version }}
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-hdfs.yml b/.github/workflows/test-hdfs.yml
@@ -70,8 +70,8 @@ jobs:
         run: |
           pip install -I -r requirements/core.txt -r requirements/kerberos.txt -r requirements/hdfs.txt -r requirements/tests/base.txt -r requirements/tests/spark-${{ inputs.spark-version }}.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
 
-    # Cannot use services because we need to mount config file from the repo, but services start before checkout.
-    # See https://github.com/orgs/community/discussions/25792
+      # Cannot use services because we need to mount config file from the repo, but services start before checkout.
+      # See https://github.com/orgs/community/discussions/25792
       - name: Start HDFS
         run: |
           docker compose down -v --remove-orphans
@@ -81,11 +81,6 @@ jobs:
           wait $wait_pid
         env:
           HDFS_IMAGE: mtsrus/hadoop:${{ inputs.hadoop-version }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-hadoop${{ inputs.hadoop-version }}
-
-      - name: Wait for HDFS to be ready
-        run: |
-          ./docker/wait-for-it.sh -h localhost -p 9870 -t 60
 
       - name: Run tests
         run: |
@@ -99,8 +94,6 @@ jobs:
         if: always()
         run: |
           docker compose down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-hadoop${{ inputs.hadoop-version }}
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-kafka.yml b/.github/workflows/test-kafka.yml
@@ -116,11 +116,6 @@ jobs:
         run: |
           pip install -I -r requirements/core.txt -r requirements/tests/base.txt -r requirements/tests/kafka.txt -r requirements/tests/spark-${{ inputs.spark-version }}.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
 
-      - name: Wait for Kafka to be ready
-        run: |
-          ./docker/wait-for-it.sh -h localhost -p 9093 -t 60
-          ./docker/wait-for-it.sh -h localhost -p 9095 -t 60
-
       - name: Run tests
         run: |
           mkdir reports/ || echo "Directory exists"

diff --git a/.github/workflows/test-mongodb.yml b/.github/workflows/test-mongodb.yml
@@ -81,10 +81,6 @@ jobs:
         run: |
           pip install -I -r requirements/core.txt -r requirements/tests/base.txt -r requirements/tests/mongodb.txt -r requirements/tests/spark-${{ inputs.spark-version }}.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
 
-      - name: Wait for MongoDB to be ready
-        run: |
-          ./docker/wait-for-it.sh -h localhost -p 27017 -t 60
-
       - name: Run tests
         run: |
           mkdir reports/ || echo "Directory exists"