[DOP-15547] Update Docker images to latest versions

MobileTeleSystems · Apr 26, 2024 · 2798ce4 · 2798ce4
1 parent 5fde37a
commit 2798ce4
Show file tree

Hide file tree

Showing 35 changed files with 67 additions and 74 deletions.
diff --git a/.github/workflows/data/clickhouse/matrix.yml b/.github/workflows/data/clickhouse/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -22,15 +22,15 @@ latest: &latest
 matrix:
   small:
     - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 23.6.1-alpine
+      clickhouse-version: 24.3.2.23-alpine
       <<: *max
   full:
   # the lowest supported Clickhouse version by JDBC driver
     - clickhouse-image: yandex/clickhouse-server
       clickhouse-version: '20.7'
       <<: *min
     - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 23.6.1-alpine
+      clickhouse-version: 24.3.2.23-alpine
       <<: *max
   nightly:
     - clickhouse-image: yandex/clickhouse-server

diff --git a/.github/workflows/data/core/matrix.yml b/.github/workflows/data/core/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/core/tracked.txt b/.github/workflows/data/core/tracked.txt
@@ -2,5 +2,7 @@ onetl/hooks/**
 onetl/plugins/**
 onetl/impl/**
 onetl/hwm/**
+onetl/_util/**
 onetl/_internal.py
 onetl/log.py
+.github/workflows/data/core/**
diff --git a/.github/workflows/data/ftp/matrix.yml b/.github/workflows/data/ftp/matrix.yml
@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/ftps image has only latest tag
+    # chonjay21/ftps image has only latest tag
     - ftp-version: latest
       <<: *max
   full:

diff --git a/.github/workflows/data/ftps/matrix.yml b/.github/workflows/data/ftps/matrix.yml
@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/ftps image has only latest tag
+    # chonjay21/ftps image has only latest tag
     - ftps-version: latest
       <<: *max
   full:

diff --git a/.github/workflows/data/greenplum/matrix.yml b/.github/workflows/data/greenplum/matrix.yml
@@ -28,14 +28,14 @@ matrix:
       package-version: 2.3.1
       <<: *max
   full:
-    - greenplum-version: 6.25.3
+    - greenplum-version: 6.23.1
       package-version: 2.2.0
       <<: *min
     - greenplum-version: 7.0.0
       package-version: 2.3.1
       <<: *max
   nightly:
-    - greenplum-version: 6.25.3
+    - greenplum-version: 6.23.1
       package-version: 2.2.0
       <<: *min
     - greenplum-version: 7.0.0

diff --git a/.github/workflows/data/hdfs/matrix.yml b/.github/workflows/data/hdfs/matrix.yml
@@ -8,7 +8,7 @@ min: &min
 
 max: &max
   hadoop-version: hadoop3-hdfs
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/hive/matrix.yml b/.github/workflows/data/hive/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/kafka/matrix.yml b/.github/workflows/data/kafka/matrix.yml
@@ -1,16 +1,15 @@
 min: &min
-  # kafka_version: 0.10.2-1-r3
-  kafka-version: 3.5.1
+  kafka-version: 1.0.0-r0
   pydantic-version: 1
   spark-version: 2.4.8
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
-  kafka-version: 3.5.1
+  kafka-version: 3.7.0
   pydantic-version: 2
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/local-fs/matrix.yml b/.github/workflows/data/local-fs/matrix.yml
@@ -20,6 +20,7 @@ min_excel: &min_excel
   os: ubuntu-latest
 
 max: &max
+  # Excel package currently has no release for 3.5.1
   spark-version: 3.5.0
   pydantic-version: 2
   python-version: '3.12'

diff --git a/.github/workflows/data/mongodb/matrix.yml b/.github/workflows/data/mongodb/matrix.yml
@@ -7,7 +7,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.4.2
+  spark-version: 3.4.3
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -22,12 +22,12 @@ latest: &latest
 
 matrix:
   small:
-    - mongodb-version: 6.0.7
+    - mongodb-version: 7.0.8
       <<: *max
   full:
     - mongodb-version: 4.0.0
       <<: *min
-    - mongodb-version: 6.0.7
+    - mongodb-version: 7.0.8
       <<: *max
   nightly:
     - mongodb-version: 4.0.0

diff --git a/.github/workflows/data/mssql/matrix.yml b/.github/workflows/data/mssql/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/mysql/matrix.yml b/.github/workflows/data/mysql/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,17 +21,17 @@ latest: &latest
 
 matrix:
   small:
-    - mysql-version: 8.0.33
+    - mysql-version: 8.3.0
       <<: *max
   full:
-  # Min supported version by JDBC driver is 5.7
-    - mysql-version: 5.7.42
+    # Min supported version by JDBC driver is 5.7
+    - mysql-version: 5.7.6
       <<: *min
-  # Max supported version by JDBC driver is 8.0
-    - mysql-version: 8.0.33
+    # Max supported version by JDBC driver is 8.3
+    - mysql-version: 8.3.0
       <<: *max
   nightly:
-    - mysql-version: 5.7.42
+    - mysql-version: 5.7.6
       <<: *min
     - mysql-version: latest
       <<: *latest
diff --git a/.github/workflows/data/oracle/matrix.yml b/.github/workflows/data/oracle/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/postgres/matrix.yml b/.github/workflows/data/postgres/matrix.yml
@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,13 +21,13 @@ latest: &latest
 
 matrix:
   small:
-    - postgres-version: 15.2-alpine
+    - postgres-version: 16.2-alpine
       <<: *max
   full:
-  # Min supported version by JDBC driver is 8.4, but it is too ancient to be used by anyone in real life
+    # Min supported version by JDBC driver is 8.4, but it is too ancient to be used by anyone in real life
     - postgres-version: 9.4.26-alpine
       <<: *min
-    - postgres-version: 15.2-alpine
+    - postgres-version: 16.2-alpine
       <<: *max
   nightly:
     - postgres-version: 9.4.26-alpine

diff --git a/.github/workflows/data/s3/matrix.yml b/.github/workflows/data/s3/matrix.yml
@@ -9,8 +9,8 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  minio-version: 2023.7.18
-  spark-version: 3.5.0
+  minio-version: 2024.4.18
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/samba/matrix.yml b/.github/workflows/data/samba/matrix.yml
@@ -15,15 +15,15 @@ latest: &latest
 
 matrix:
   small:
-    - server-version: latest
+    - server-version: 4.15.7
       <<: *max
   full:
-    - server-version: latest
+    - server-version: 1.0.0
       <<: *min
-    - server-version: latest
+    - server-version: 4.15.7
       <<: *max
   nightly:
-    - server-version: latest
+    - server-version: 1.0.0
       <<: *min
     - server-version: latest
       <<: *latest
diff --git a/.github/workflows/data/sftp/matrix.yml b/.github/workflows/data/sftp/matrix.yml
@@ -15,13 +15,13 @@ latest: &latest
 
 matrix:
   small:
-    - openssh-version: 9.3_p1-r3-ls120
+    - openssh-version: 9.6_p1-r0-ls154
       <<: *max
   full:
-  # prior image versions does not accept incoming connections, seems like a bug
+    # prior image versions does not accept incoming connections, seems like a bug
     - openssh-version: 8.1_p1-r0-ls5
       <<: *min
-    - openssh-version: 9.3_p1-r3-ls120
+    - openssh-version: 9.6_p1-r0-ls154
       <<: *max
   nightly:
     - openssh-version: 8.1_p1-r0-ls5

diff --git a/.github/workflows/data/teradata/matrix.yml b/.github/workflows/data/teradata/matrix.yml
@@ -1,5 +1,5 @@
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/webdav/matrix.yml b/.github/workflows/data/webdav/matrix.yml
@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/webdav image has only latest tag
+    # chonjay21/webdav image has only latest tag
     - webdav-version: latest
       <<: *max
   full:

diff --git a/.github/workflows/test-ftp.yml b/.github/workflows/test-ftp.yml
@@ -59,7 +59,6 @@ jobs:
           docker compose up -d ftp
         env:
           FTP_IMAGE: chonjay21/ftps:${{ inputs.ftp-version }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftp${{ inputs.ftp-version }}
 
       - name: Wait for FTP to be ready
         run: |
@@ -76,8 +75,6 @@ jobs:
         if: always()
         run: |
           docker compose down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftp${{ inputs.ftp-version }}
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-ftps.yml b/.github/workflows/test-ftps.yml
@@ -59,7 +59,6 @@ jobs:
           docker compose up -d ftps
         env:
           FTPS_IMAGE: chonjay21/ftps:${{ inputs.ftps-version }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftps${{ inputs.ftps-version }}
 
       - name: Wait for FTPS to be ready
         run: |
@@ -76,8 +75,6 @@ jobs:
         if: always()
         run: |
           docker compose down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-ftps${{ inputs.ftps-version }}
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-hdfs.yml b/.github/workflows/test-hdfs.yml
@@ -81,7 +81,6 @@ jobs:
           wait $wait_pid
         env:
           HDFS_IMAGE: mtsrus/hadoop:${{ inputs.hadoop-version }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-hadoop${{ inputs.hadoop-version }}
 
       - name: Wait for HDFS to be ready
         run: |
@@ -99,8 +98,6 @@ jobs:
         if: always()
         run: |
           docker compose down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-hadoop${{ inputs.hadoop-version }}
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-samba.yml b/.github/workflows/test-samba.yml
@@ -57,7 +57,6 @@ jobs:
           docker compose up -d samba
         env:
           SAMBA_IMAGE: elswork/samba:${{ inputs.server-version }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-samba${{ inputs.server-version }}
 
       - name: Wait for Samba to be ready
         run: |
@@ -74,8 +73,6 @@ jobs:
         if: always()
         run: |
           docker compose down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-samba${{ inputs.server-version }}
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test-webdav.yml b/.github/workflows/test-webdav.yml
@@ -59,7 +59,6 @@ jobs:
           docker compose up -d webdav
         env:
           WEBDAV_IMAGE: chonjay21/webdav:${{ inputs.webdav-version }}
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-webdav${{ inputs.webdav-version }}
 
       - name: Wait for WebDAV to be ready
         run: |
@@ -76,8 +75,6 @@ jobs:
         if: always()
         run: |
           docker compose down -v --remove-orphans
-        env:
-          COMPOSE_PROJECT_NAME: ${{ github.run_id }}-webdav${{ inputs.webdav-version }}
 
       - name: Upload coverage results
         uses: actions/upload-artifact@v4

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -11,7 +11,7 @@ Limitations
 
 We should keep close to these items during development:
 
-* Some companies still use old Spark versions, like 2.3.0. So it is required to keep compatibility if possible, e.g. adding branches for different Spark versions.
+* Some companies still use old Spark versions, like 2.3.1. So it is required to keep compatibility if possible, e.g. adding branches for different Spark versions.
 * Different users uses onETL in different ways - some uses only DB connectors, some only files. Connector-specific dependencies should be optional.
 * Instead of creating classes with a lot of different options, prefer splitting them into smaller classes, e.g. options class, context manager, etc, and using composition.
 
@@ -71,7 +71,7 @@ Create virtualenv and install dependencies:
         -r requirements/tests/postgres.txt \
         -r requirements/tests/oracle.txt \
         -r requirements/tests/pydantic-2.txt \
-        -r requirements/tests/spark-3.5.0.txt
+        -r requirements/tests/spark-3.5.1.txt
 
     # TODO: remove after https://github.com/zqmillet/sphinx-plantuml/pull/4
     pip install sphinx-plantuml --no-deps