[DOP-15547] Update Docker images to latest versions

.github/workflows/data/clickhouse/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -22,15 +22,15 @@ latest: &latest
 matrix:
   small:
     - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 23.6.1-alpine
+      clickhouse-version: 24.3.2.23-alpine
       <<: *max
   full:
   # the lowest supported Clickhouse version by JDBC driver
     - clickhouse-image: yandex/clickhouse-server
       clickhouse-version: '20.7'
       <<: *min
     - clickhouse-image: clickhouse/clickhouse-server
-      clickhouse-version: 23.6.1-alpine
+      clickhouse-version: 24.3.2.23-alpine
       <<: *max
   nightly:
     - clickhouse-image: yandex/clickhouse-server

.github/workflows/data/core/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/ftp/matrix.yml

@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/ftps image has only latest tag
+    # chonjay21/ftps image has only latest tag
     - ftp-version: latest
       <<: *max
   full:

.github/workflows/data/ftps/matrix.yml

@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/ftps image has only latest tag
+    # chonjay21/ftps image has only latest tag
     - ftps-version: latest
       <<: *max
   full:

.github/workflows/data/greenplum/matrix.yml

@@ -28,14 +28,14 @@ matrix:
       package-version: 2.3.1
       <<: *max
   full:
-    - greenplum-version: 6.25.3
+    - greenplum-version: 6.23.1
       package-version: 2.2.0
       <<: *min
     - greenplum-version: 7.0.0
       package-version: 2.3.1
       <<: *max
   nightly:
-    - greenplum-version: 6.25.3
+    - greenplum-version: 6.23.1
       package-version: 2.2.0
       <<: *min
     - greenplum-version: 7.0.0

.github/workflows/data/hdfs/matrix.yml

@@ -8,7 +8,7 @@ min: &min
 
 max: &max
   hadoop-version: hadoop3-hdfs
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/hive/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/kafka/matrix.yml

@@ -1,16 +1,15 @@
 min: &min
-  # kafka_version: 0.10.2-1-r3
-  kafka-version: 3.5.1
+  kafka_version: 1.0.0-r0
   pydantic-version: 1
   spark-version: 2.4.8
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
-  kafka-version: 3.5.1
+  kafka-version: 3.7.0
   pydantic-version: 2
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

.github/workflows/data/local-fs/matrix.yml

@@ -20,6 +20,7 @@ min_excel: &min_excel
   os: ubuntu-latest
 
 max: &max
+  # Excel package currently has no release for 3.5.1
   spark-version: 3.5.0
   pydantic-version: 2
   python-version: '3.12'

.github/workflows/data/mongodb/matrix.yml

@@ -7,7 +7,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.4.2
+  spark-version: 3.4.3
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -22,12 +22,12 @@ latest: &latest
 
 matrix:
   small:
-    - mongodb-version: 6.0.7
+    - mongodb-version: 7.0.8
       <<: *max
   full:
     - mongodb-version: 4.0.0
       <<: *min
-    - mongodb-version: 6.0.7
+    - mongodb-version: 7.0.8
       <<: *max
   nightly:
     - mongodb-version: 4.0.0

.github/workflows/data/mssql/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/mysql/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,17 +21,17 @@ latest: &latest
 
 matrix:
   small:
-    - mysql-version: 8.0.33
+    - mysql-version: 8.3.0
       <<: *max
   full:
-  # Min supported version by JDBC driver is 5.7
-    - mysql-version: 5.7.42
+    # Min supported version by JDBC driver is 5.7
+    - mysql-version: 5.7.6
       <<: *min
-  # Max supported version by JDBC driver is 8.0
-    - mysql-version: 8.0.33
+    # Max supported version by JDBC driver is 8.3
+    - mysql-version: 8.3.0
       <<: *max
   nightly:
-    - mysql-version: 5.7.42
+    - mysql-version: 5.7.6
       <<: *min
     - mysql-version: latest
       <<: *latest

.github/workflows/data/oracle/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/postgres/matrix.yml

@@ -6,7 +6,7 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
@@ -21,13 +21,13 @@ latest: &latest
 
 matrix:
   small:
-    - postgres-version: 15.2-alpine
+    - postgres-version: 16.2-alpine
       <<: *max
   full:
-  # Min supported version by JDBC driver is 8.4, but it is too ancient to be used by anyone in real life
+    # Min supported version by JDBC driver is 8.4, but it is too ancient to be used by anyone in real life
     - postgres-version: 9.4.26-alpine
       <<: *min
-    - postgres-version: 15.2-alpine
+    - postgres-version: 16.2-alpine
       <<: *max
   nightly:
     - postgres-version: 9.4.26-alpine

.github/workflows/data/s3/matrix.yml

@@ -9,8 +9,8 @@ min: &min
   os: ubuntu-latest
 
 max: &max
-  minio-version: 2023.7.18
-  spark-version: 3.5.0
+  minio-version: 2024.4.18
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/samba/matrix.yml

@@ -15,15 +15,15 @@ latest: &latest
 
 matrix:
   small:
-    - server-version: latest
+    - server-version: 4.15.7
       <<: *max
   full:
-    - server-version: latest
+    - server-version: 1.0.0
       <<: *min
-    - server-version: latest
+    - server-version: 4.15.7
       <<: *max
   nightly:
-    - server-version: latest
+    - server-version: 1.0.0
       <<: *min
     - server-version: latest
       <<: *latest

.github/workflows/data/sftp/matrix.yml

@@ -15,13 +15,13 @@ latest: &latest
 
 matrix:
   small:
-    - openssh-version: 9.3_p1-r3-ls120
+    - openssh-version: 9.6_p1-r0-ls154
       <<: *max
   full:
-  # prior image versions does not accept incoming connections, seems like a bug
+    # prior image versions does not accept incoming connections, seems like a bug
     - openssh-version: 8.1_p1-r0-ls5
       <<: *min
-    - openssh-version: 9.3_p1-r3-ls120
+    - openssh-version: 9.6_p1-r0-ls154
       <<: *max
   nightly:
     - openssh-version: 8.1_p1-r0-ls5

.github/workflows/data/teradata/matrix.yml

@@ -1,5 +1,5 @@
 max: &max
-  spark-version: 3.5.0
+  spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20

.github/workflows/data/webdav/matrix.yml

@@ -15,7 +15,7 @@ latest: &latest
 
 matrix:
   small:
-  # chonjay21/webdav image has only latest tag
+    # chonjay21/webdav image has only latest tag
     - webdav-version: latest
       <<: *max
   full:

CONTRIBUTING.rst

@@ -11,7 +11,7 @@ Limitations
 
 We should keep close to these items during development:
 
-* Some companies still use old Spark versions, like 2.3.0. So it is required to keep compatibility if possible, e.g. adding branches for different Spark versions.
+* Some companies still use old Spark versions, like 2.3.1. So it is required to keep compatibility if possible, e.g. adding branches for different Spark versions.
 * Different users uses onETL in different ways - some uses only DB connectors, some only files. Connector-specific dependencies should be optional.
 * Instead of creating classes with a lot of different options, prefer splitting them into smaller classes, e.g. options class, context manager, etc, and using composition.
 
@@ -71,7 +71,7 @@ Create virtualenv and install dependencies:
         -r requirements/tests/postgres.txt \
         -r requirements/tests/oracle.txt \
         -r requirements/tests/pydantic-2.txt \
-        -r requirements/tests/spark-3.5.0.txt
+        -r requirements/tests/spark-3.5.1.txt
 
     # TODO: remove after https://github.com/zqmillet/sphinx-plantuml/pull/4
     pip install sphinx-plantuml --no-deps

README.rst

@@ -187,17 +187,17 @@ Compatibility matrix
 +--------------------------------------------------------------+-------------+-------------+-------+
 | Spark                                                        | Python      | Java        | Scala |
 +==============================================================+=============+=============+=======+
-| `2.3.x <https://spark.apache.org/docs/2.3.0/#downloading>`_  | 3.7 only    | 8 only      | 2.11  |
+| `2.3.x <https://spark.apache.org/docs/2.3.1/#downloading>`_  | 3.7 only    | 8 only      | 2.11  |
 +--------------------------------------------------------------+-------------+-------------+-------+
 | `2.4.x <https://spark.apache.org/docs/2.4.8/#downloading>`_  | 3.7 only    | 8 only      | 2.11  |
 +--------------------------------------------------------------+-------------+-------------+-------+
 | `3.2.x <https://spark.apache.org/docs/3.2.4/#downloading>`_  | 3.7 - 3.10  | 8u201 - 11  | 2.12  |
 +--------------------------------------------------------------+-------------+-------------+-------+
-| `3.3.x <https://spark.apache.org/docs/3.3.3/#downloading>`_  | 3.7 - 3.10  | 8u201 - 17  | 2.12  |
+| `3.3.x <https://spark.apache.org/docs/3.3.4/#downloading>`_  | 3.7 - 3.10  | 8u201 - 17  | 2.12  |
 +--------------------------------------------------------------+-------------+-------------+-------+
-| `3.4.x <https://spark.apache.org/docs/3.4.1/#downloading>`_  | 3.7 - 3.12  | 8u362 - 20  | 2.12  |
+| `3.4.x <https://spark.apache.org/docs/3.4.3/#downloading>`_  | 3.7 - 3.12  | 8u362 - 20  | 2.12  |
 +--------------------------------------------------------------+-------------+-------------+-------+
-| `3.5.x <https://spark.apache.org/docs/3.5.0/#downloading>`_  | 3.8 - 3.12  | 8u371 - 20  | 2.12  |
+| `3.5.x <https://spark.apache.org/docs/3.5.1/#downloading>`_  | 3.8 - 3.12  | 8u371 - 20  | 2.12  |
 +--------------------------------------------------------------+-------------+-------------+-------+
 
 .. _pyspark-install:
@@ -212,7 +212,7 @@ or install PySpark explicitly:
 
 .. code:: bash
 
-    pip install onetl pyspark==3.5.0  # install a specific PySpark version
+    pip install onetl pyspark==3.5.1  # install a specific PySpark version
 
 or inject PySpark to ``sys.path`` in some other way BEFORE creating a class instance.
 **Otherwise connection object cannot be created.**
@@ -553,7 +553,7 @@ Read files directly from S3 path, convert them to dataframe, transform it and th
     setup_logging()
 
     # Initialize new SparkSession with Hadoop AWS libraries and Postgres driver loaded
-    maven_packages = SparkS3.get_packages(spark_version="3.5.0") + Postgres.get_packages()
+    maven_packages = SparkS3.get_packages(spark_version="3.5.1") + Postgres.get_packages()
     spark = (
         SparkSession.builder.appName("spark_app_onetl_demo")
         .config("spark.jars.packages", ",".join(maven_packages))

docker-compose.yml

@@ -9,7 +9,7 @@ services:
       context: .
       target: base
       args:
-        SPARK_VERSION: 3.5.0
+        SPARK_VERSION: 3.5.1
     env_file: .env.docker
     volumes:
       - ./:/app/

docker/Dockerfile

@@ -45,7 +45,7 @@ COPY --chown=onetl:onetl ./run_tests.sh ./pytest_runner.sh ./combine_coverage.sh
 COPY --chown=onetl:onetl ./docker/wait-for-it.sh /app/docker/wait-for-it.sh
 RUN chmod +x /app/run_tests.sh /app/pytest_runner.sh /app/combine_coverage.sh /app/docker/wait-for-it.sh
 
-ARG SPARK_VERSION=3.5.0
+ARG SPARK_VERSION=3.5.1
 # Spark is heavy, and version change is quite rare
 COPY --chown=onetl:onetl ./requirements/tests/spark-${SPARK_VERSION}.txt /app/requirements/tests/
 RUN pip install -r /app/requirements/tests/spark-${SPARK_VERSION}.txt

requirements/tests/spark-3.3.3.txt → requirements/tests/spark-3.3.4.txt

@@ -1,5 +1,5 @@
 numpy>=1.16,<1.24
 pandas>=1.0,<2
 pyarrow>=1.0
-pyspark==3.3.3
+pyspark==3.3.4
 sqlalchemy<2.0

requirements/tests/spark-3.4.2.txt → requirements/tests/spark-3.4.3.txt

@@ -1,5 +1,5 @@
 numpy>=1.16
 pandas>=1.0
 pyarrow>=1.0
-pyspark==3.4.2
+pyspark==3.4.3
 sqlalchemy

requirements/tests/spark-3.5.1.txt

@@ -0,0 +1,5 @@
+numpy>=1.16
+pandas>=1.0
+pyarrow>=1.0
+pyspark==3.5.1
+sqlalchemy

tests/tests_integration/tests_db_connection_integration/test_clickhouse_integration.py

@@ -250,7 +250,7 @@ def table_finalizer():
     assert not clickhouse.fetch(f"SELECT * FROM {temp_table}{suffix}").count()
 
 
-@pytest.mark.xfail(reason="Clickhouse 20.7 doesn't support functions")
+@pytest.mark.xfail(reason="CREATE FUNCTION is not supported in Clickhouse < 21.20")
 @pytest.mark.parametrize("suffix", ["", ";"])
 def test_clickhouse_connection_execute_function(
     request,