Update versions of core languages and libraries for the 0.8 series (#508

)

* Change supported Python versions in setup.cfg

* Change supported Python versions Python runtime README

* Change supported Python versions main project README

* Remove explicit version reference in requirements_plugins.txt

* Update comments referring to old Python versions

* Update Python config for read the docs

* Update references to min version in RST docs

* Bump Apache Arrow version for Python
Using 18.1.0 since 19.0.0 has a bug (19.0.1 patch release coming soon)

* Bump Apache Arrow version for Java

* Bump Python versions in CI

* Force initialization of Arrow memory subsystem in data service startup

* Update CI build matrix to match Pandas versions to supported Python versions

* Change constraints on NumPy versions

* Update versions of Protobuf and gRPC in Python

* Make Pandas an optional dependency

* Update dulwich

* Align NumPy and Pandas versions in CI build matrix

* CI updates for E2E integration test

* Update core Java dependencies to match Python

* Revert uses reference for child workflows

* Remove reference to Protobuf internal classes used to sanitize error messages in REST API translator

* Try to use Python 3.13 for GCP integration

* Update constraints info in the runtime README

.github/workflows/build.yml

@@ -10,7 +10,7 @@ on:
 
 env:
   JAVA_DISTRIBUTION: "zulu"
-  PYTHON_VERSION: "3.12"
+  PYTHON_VERSION: "3.13"
   NODE_VERSION: "22"
 
 
@@ -79,42 +79,46 @@ jobs:
 
           # Latest supported versions on all 3 major platforms
           - { PLATFORM: "windows",
-              PYTHON_VERSION: "3.12",
-              PANDAS_DEPENDENCY: "pandas ~= 2.1.0",
-              PYSPARK_DEPENDENCY: "pyspark ~= 3.5.0" }
+              PYTHON_VERSION: "3.13",
+              PANDAS_DEPENDENCY: "'pandas ~= 2.2.0'",
+              PYSPARK_DEPENDENCY: "'pyspark ~= 3.5.0'" }
 
           - { PLATFORM: "macos",
-              PYTHON_VERSION: "3.12",
-              PANDAS_DEPENDENCY: "pandas ~= 2.1.0",
-              PYSPARK_DEPENDENCY: "pyspark ~= 3.5.0" }
+              PYTHON_VERSION: "3.13",
+              PANDAS_DEPENDENCY: "'pandas ~= 2.2.0'",
+              PYSPARK_DEPENDENCY: "'pyspark ~= 3.5.0'" }
+
+          - { PLATFORM: "ubuntu",
+              PYTHON_VERSION: "3.13",
+              PANDAS_DEPENDENCY: "'pandas ~= 2.2.0'",
+              PYSPARK_DEPENDENCY: "'pyspark ~= 3.5.0'" }
 
+          # Python 3.12 requires Pandas 2.1, Pandas 2.0 only has binaries for 3.11
           - { PLATFORM: "ubuntu",
               PYTHON_VERSION: "3.12",
-              PANDAS_DEPENDENCY: "pandas ~= 2.1.0",
-              PYSPARK_DEPENDENCY: "pyspark ~= 3.5.0" }
+              PANDAS_DEPENDENCY: "'pandas ~= 2.1.0'",
+              PYSPARK_DEPENDENCY: "'pyspark ~= 3.4.0'" }
 
-          # Intermediate supported versions, force testing against .0 for Pandas 2.0
+          # Force testing on the first release of the Pandas 2 series, requires Python 3.11
+          # Note NumPy 2.0 only works with Pandas 2.1 and later
           - { PLATFORM: "ubuntu",
               PYTHON_VERSION: "3.11",
-              PANDAS_DEPENDENCY: "pandas == 2.0.0",
-              PYSPARK_DEPENDENCY: "pyspark ~= 3.4.0" }
+              PANDAS_DEPENDENCY: "'pandas == 2.0.0' 'numpy < 2.0.0'",
+              PYSPARK_DEPENDENCY: "'pyspark ~= 3.3.0'" }
 
+          # Test the latest release in the Pandas 1.X series
+          # This could also work with Python 3.11 but we want to test every supported Python version
           - { PLATFORM: "ubuntu",
               PYTHON_VERSION: "3.10",
-              PANDAS_DEPENDENCY: "pandas ~= 1.5.0",
-              PYSPARK_DEPENDENCY: "pyspark ~= 3.3.0" }
-
-          - { PLATFORM: "ubuntu",
-              PYTHON_VERSION: "3.9",
-              PANDAS_DEPENDENCY: "pandas ~= 1.3.0",
-              PYSPARK_DEPENDENCY: "pyspark ~= 3.1.0" }
+              PANDAS_DEPENDENCY: "'pandas ~= 1.5.0' 'numpy < 2.0.0'",
+              PYSPARK_DEPENDENCY: "'pyspark ~= 3.1.0'" }
 
           # Oldest supported versions, force testing against .0 for Pandas and PySpark
           # If those don't work due to bugs, we need to update README for the supported versions
           - { PLATFORM: "ubuntu",
-              PYTHON_VERSION: "3.8",
-              PANDAS_DEPENDENCY: "pandas == 1.2.0",
-              PYSPARK_DEPENDENCY: "pyspark == 3.0.0" }
+              PYTHON_VERSION: "3.9",
+              PANDAS_DEPENDENCY: "'pandas == 1.2.0' 'numpy < 2.0.0'",
+              PYSPARK_DEPENDENCY: "'pyspark == 3.0.0'" }
 
     # E.g. platform = "windows" -> build image = "windows-latest"
     runs-on: ${{ matrix.environment.PLATFORM }}-latest
@@ -136,8 +140,8 @@ jobs:
       - name: Install dependencies
         run: |
           cd tracdap-runtime/python
-          pip install "${{ matrix.environment.PANDAS_DEPENDENCY }}"
-          pip install "${{ matrix.environment.PYSPARK_DEPENDENCY }}"
+          pip install ${{ matrix.environment.PANDAS_DEPENDENCY }}
+          pip install ${{ matrix.environment.PYSPARK_DEPENDENCY }}
           pip install -r requirements.txt
 
       - name: Protoc code generation

.github/workflows/compliance.yml

@@ -24,7 +24,7 @@ on:
 env:
   JAVA_VERSION: "21"
   JAVA_DISTRIBUTION: "zulu"
-  PYTHON_VERSION: "3.12"
+  PYTHON_VERSION: "3.13"
   NODE_VERSION: "22"
 
 

.github/workflows/integration-cloud.yaml

@@ -30,7 +30,7 @@ on:
 env:
   JAVA_VERSION: "21"
   JAVA_DISTRIBUTION: "zulu"
-  PYTHON_VERSION: "3.12"
+  PYTHON_VERSION: "3.13"
   NODE_VERSION: "22"
 
 

.github/workflows/integration-sql.yaml

@@ -24,7 +24,7 @@ on:
 env:
   JAVA_VERSION: "21"
   JAVA_DISTRIBUTION: "zulu"
-  PYTHON_VERSION: "3.12"
+  PYTHON_VERSION: "3.13"
   NODE_VERSION: "22"
 
 

.github/workflows/integration.yml

@@ -18,7 +18,7 @@ on:
 env:
   JAVA_VERSION: "21"
   JAVA_DISTRIBUTION: "zulu"
-  PYTHON_VERSION: "3.12"
+  PYTHON_VERSION: "3.13"
   # Node 20 breaks relative imports used in examples for web API tests
   NODE_VERSION: "18"
 
@@ -310,7 +310,6 @@ jobs:
               TRAC_CONFIG_FILE: '.github/config/int-storage-gcs.yaml',
               TRAC_SECRET_KEY: short-lived-secret,
               PYTHON_TESTS: int_storage_gcp*.py,
-              PYTHON_VERSION: "3.11",
               JAVA_TESTS: int-storage }
 
           - { SERVICE: storage,

.github/workflows/packaging.yml

@@ -26,7 +26,7 @@ on:
 env:
   JAVA_VERSION: "21"
   JAVA_DISTRIBUTION: "zulu"
-  PYTHON_VERSION: "3.12"
+  PYTHON_VERSION: "3.13"
   NODE_VERSION: "22"
 
 

.readthedocs.yaml

@@ -18,12 +18,11 @@
 # RTD schema version
 version: 2
 
-# Use latest stable image and Python version
-# May 2023: Older versions (Python 3.8) have dependency issues with liburl3 and OpenSSL
+# Use latest stable image and Python version (updated Jan 25)
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    python: "3.11"
+    python: "3.13"
 
 # Use requirements from the dev (build tools) folder
 python:

README.md

@@ -64,7 +64,7 @@ For more information see the
 
 With TRAC D.A.P. you can build and run production-ready models right on your desktop!
 All you need is an IDE, Python  and the tracdap-runtime Python package.
-TRAC D.A.P. requires Python 3.8 or later.
+TRAC D.A.P. requires Python 3.9 or later.
 
 The [modelling tutorial](https://tracdap.finos.org/en/stable/modelling/tutorial/hello_world.html)
 shows you how to get set up and write your first models. You can write models locally using

dev/codegen/protoc-ctrl.py

@@ -67,7 +67,8 @@ def _copytree(src, dst):
 
     _log.info(f"Copying {src} -> {dst}")
 
-    # In shutil.copytree, dir_exists_ok is only available from Python 3.8, but we need Python 3.7
+    # This could be replaced with shutil since minimum Python version is 3.9 as of Jan 2025
+    # In shutil.copytree, dir_exists_ok is available from Python 3.8
     # Codegen is part of the core build tools so needs to match supported Python versions of the TRAC runtime
 
     src_dir = pathlib.Path(src)

doc/deployment/sandbox.rst

@@ -41,10 +41,10 @@ so long as it is version 11 or higher.
 
 **Python**
 
-TRAC requires Python 3.8 or higher, which can be downloaded and installed from
+TRAC requires Python 3.9 or higher, which can be downloaded and installed from
 `python.org <https://www.python.org/downloads/>`_.
 If your company restricts software deployment, it is fine to use the Python package they provide
-so long as it is version 3.8 or higher.
+so long as it is version 3.9 or higher.
 
 **TRAC Services**
 

gradle/versions.gradle

@@ -26,16 +26,16 @@ ext {
     license_check_plugin_version = "2.8"
 
     // Core platform technologies
-    netty_version = '4.1.115.Final'
-    guava_version = '33.3.0-jre'
-    proto_version = '4.28.2'
-    grpc_version = '1.68.0'
-    gapi_version = '2.44.0'
+    netty_version = '4.1.117.Final'
+    guava_version = '33.4.0-jre'
+    proto_version = '4.29.3'
+    grpc_version = '1.70.0'
+    gapi_version = '2.51.0'
 
     // Data technologies
-    arrow_version = '16.1.0'
-    jackson_version = '2.17.2'
-    jackson_databind_version = '2.17.2'
+    arrow_version = '18.1.0'
+    jackson_version = '2.18.2'
+    jackson_databind_version = '2.18.2'
 
     // Util libraries
     slf4j_version = '2.0.16'

tracdap-libs/tracdap-lib-test/src/main/java/org/finos/tracdap/test/helpers/PlatformTest.java

@@ -504,7 +504,7 @@ void prepareDataAndExecutor() throws Exception {
 
                 // Include optional packages needed for end-to-end testing
                 var pipPB = new ProcessBuilder();
-                pipPB.command(pythonExe, "-m", "pip", "install", tracRtWhl.get().toString(), "polars");
+                pipPB.command(pythonExe, "-m", "pip", "install", tracRtWhl.get().toString(), "pandas", "polars");
                 pipPB.environment().put(VENV_ENV_VAR, venvPath.toString());
 
                 var pipP = pipPB.start();

tracdap-runtime/python/README.md

@@ -16,12 +16,13 @@ Documentation for the TRAC platform is available on our website at
 
 The TRAC runtime for Python has these requirements:
 
-* Python: 3.8 up to 3.12
+* Python: 3.9 up to 3.13
 * Pandas: 1.2 up to 2.2
 * PySpark 3.0 up to 3.5 (optional)
 
-3rd party libraries may impose additional constraints on supported versions of Python, Pandas or PySpark.
-As of February 2024, the Python libraries for GCP do not yet support Python 3.12.
+3rd party libraries may impose additional constraints on supported versions of key libraries.
+For example, Pandas 1.5 is not available for Python 3.12 or 3.13, while NumPy 2.0 is only
+compatible with Pandas 2.1 and later.
 
 ## Installing the runtime
 

tracdap-runtime/python/requirements.txt

@@ -8,23 +8,23 @@
 
 # Core protobuf support for metadata / config classes
 # This should match the Java Protobuf version, if possible
-protobuf == 5.28.2
+protobuf == 5.29.3
 
 # Support for the gRPC runtime API server
 # Optional in the distributed package but required for development
-grpcio == 1.66.1
-grpcio-status == 1.66.1
-grpcio-tools == 1.66.1
+grpcio == 1.70.0
+grpcio-status == 1.70.0
+grpcio-tools == 1.70.0
 
 # Core data framework is based on Arrow
 # This should always match the Java Arrow version
-pyarrow == 16.1.0
+pyarrow == 18.1.0
 
 # PyYAML is used to load config supplied in YAML format
 pyyaml == 6.0.2
 
 # Python implementation of Git, for model loading
-dulwich == 0.22.1
+dulwich == 0.22.7
 
 # Requests, used for downloading model packages
 requests == 2.32.3
@@ -45,7 +45,7 @@ requests == 2.32.3
 # NumPy 2.0 is not supported yet, it breaks the ABI and other packages need time to catch up
 
 pandas >= 1.2.0, < 2.3.0
-numpy < 2.0.0
+numpy >= 1.20, < 2.3.0
 
 # Polars is an optional dependency - support the 1.x series
 

tracdap-runtime/python/requirements_plugins.txt

@@ -6,7 +6,6 @@
 # Dependencies in the dist package are controlled as optional features
 
 # Some of these libraries do not support the oldest versions of Python
-# So e.g. to use the Azure libraries, Python 3.8 is required
 # We can still support older Python versions, so long as those plugins are not being used
 
 # To control dependencies installed during CI, the BEGIN_PLUGIN and END_PLUGIN markers are used

tracdap-runtime/python/setup.cfg

@@ -64,26 +64,27 @@ package_dir =
 
 # Support a range of Python versions
 # (These versions are tested in CI)
-python_requires = >= 3.8, < 3.13
+python_requires = >= 3.9, < 3.14
 
 install_requires =
-    protobuf == 5.28.2
-    pyarrow == 16.1.0
+    protobuf == 5.29.3
+    pyarrow == 18.1.0
     pyyaml == 6.0.2
-    dulwich == 0.22.1
+    dulwich == 0.22.7
     requests == 2.32.3
 
-    # Support a range of Pandas versions (These versions are tested in CI)
-    # Do not allow NumPy 2.0 yet, it breaks the ABI, other packages need time to catch up
-    pandas >= 1.2.0, < 2.3.0
-    numpy < 2.0.0
 
 [options.extras_require]
 
 grpc =
     # Enable the gRPC runtime API server
-    grpcio == 1.66.1
-    grpcio-status == 1.66.1
+    grpcio == 1.70.0
+    grpcio-status == 1.70.0
+
+pandas =
+    # Support a range of Pandas / NumPy versions (These versions are tested in CI)
+    pandas >= 1.2.0, < 2.3.0
+    numpy >= 1.20, < 2.3.0
 
 polars =
     # Support all 1.x releases of Polars

tracdap-services/tracdap-gateway/src/main/java/org/finos/tracdap/gateway/proxy/rest/RestApiTranslator.java

@@ -25,7 +25,6 @@
 import org.finos.tracdap.common.exception.ETracInternal;
 import org.finos.tracdap.common.exception.EUnexpected;
 
-import com.google.gson.stream.MalformedJsonException;
 import com.google.protobuf.Descriptors;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.Message;
@@ -132,22 +131,14 @@ private void decodeRestBody(ByteBuf bodyBuffer, Message.Builder body) {
         }
         catch (InvalidProtocolBufferException e) {
 
-            // TODO: error handling
-
-            // Validation failures will go back to users (API users, i.e. application developers)
-            // Strip out GSON class name from the error message for readability
-            var detailMessage = e.getLocalizedMessage();
-            var classNamePrefix = MalformedJsonException.class.getName() + ": ";
-
-            if (detailMessage.startsWith(classNamePrefix))
-                detailMessage = detailMessage.substring(classNamePrefix.length());
+            var errorDetails = sanitizeErrorMessage(e.getMessage());
 
             var message = String.format(
                     "Invalid JSON input for type [%s]: %s",
                     body.getDescriptorForType().getName(),
-                    detailMessage);
+                    errorDetails);
 
-            log.warn(message);
+            log.error(message);
             throw new EInputValidation(message, e);
         }
         catch (IOException e) {
@@ -435,4 +426,17 @@ private TRequest.Builder extractInt(
     private Function<URI, String> preparePathSegmentExtractor(int pathSegmentIndex) {
         return uri -> extractPathSegment(pathSegmentIndex, uri);
     }
+
+    private String sanitizeErrorMessage(String message) {
+
+        // Some error messages from JSON parsing include the exception class name
+        // Clean these up to make errors more readable for users
+
+        if (message.contains(GSON_ERROR_PREFIX))
+            return message.replace(GSON_ERROR_PREFIX, "");
+        else
+            return message;
+    }
+
+    private static final String GSON_ERROR_PREFIX = "com.google.gson.stream.MalformedJsonException: ";
 }