From 1aae65289e3e7c27374aa2c08ebfe8b7008e50da Mon Sep 17 00:00:00 2001
From: Julien Cuquemelle <j.cuquemelle@criteo.com>
Date: Wed, 20 Nov 2024 19:02:44 +0100
Subject: [PATCH] Support python3.11

---
 .github/workflows/main.yml        | 14 +++++++++-----
 examples/spark-with-S3/Dockerfile | 21 +++++++++++----------
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 1f47a30..ba472a3 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -33,7 +33,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: [3.8, 3.9, 3.11]
 
     steps:
     - uses: actions/checkout@v2
@@ -53,22 +53,26 @@ jobs:
   standalone_spark3_with_S3:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        python-version: [ 3.9, 3.11 ]
+
     steps:
     - uses: actions/checkout@v2
     
     - name: Build spark-docker
-      run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=3.2.2 --build-arg PYTHON_VERSION=3.9.15
+      run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=3.2.2 --build-arg PYTHON_VERSION=${{ matrix.python-version }}
     
     - name: Build the docker-compose stack
       run: |
-        export PYTHON_VERSION=3.9
+        export PYTHON_VERSION=${{ matrix.python-version }}
         docker compose -f ./examples/spark-with-S3/docker-compose.yml up -d
       
     - name: Check running containers
       run: docker ps -a
 
     - name: Run spark Job
-      run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.9 3.2.2
+      run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python${{ matrix.python-version }} 3.2.2
 
   # hadoop_hdfs:
   #   runs-on: ubuntu-latest
@@ -107,7 +111,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.9]
+        python-version: [3.9, 3.11]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/examples/spark-with-S3/Dockerfile b/examples/spark-with-S3/Dockerfile
index 98bb6e8..5383c9c 100644
--- a/examples/spark-with-S3/Dockerfile
+++ b/examples/spark-with-S3/Dockerfile
@@ -7,16 +7,17 @@ ARG SPARK_INPUT_VERSION=3.2.2
 ARG PYTHON_VERSION=3.9.15
 
 # Python
-RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
-  libreadline-dev libsqlite3-dev wget curl unzip llvm libncurses5-dev libncursesw5-dev \
-  xz-utils tk-dev libffi-dev liblzma-dev
-RUN curl https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz --output /usr/python.tgz
-RUN tar xf /usr/python.tgz -C /usr && rm /usr/python.tgz
-RUN cd /usr/Python-${PYTHON_VERSION} \ 
- && ./configure --enable-shared \
- && make -j8 \
- && make altinstall \
- && cd -
+RUN apt-get install -y make build-essential libssl-dev zlib1g-dev \
+                    libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev \
+                    libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl git
+RUN curl https://pyenv.run | bash
+
+ENV PATH=/root/.pyenv/bin:$PATH
+RUN eval "$(pyenv init -)"
+RUN eval "$(pyenv virtualenv-init -)"
+
+RUN pyenv install ${PYTHON_VERSION}
+RUN pyenv global ${PYTHON_VERSION}
 
 # Java
 RUN apt-get install -y openjdk-11-jre \