Build Lithops images for every push to main (#422)

.github/workflows/.lithops_config_aws.template

@@ -0,0 +1,17 @@
+lithops:
+  backend: aws_lambda
+  storage: aws_s3
+  storage_bucket: lithops-tom-data
+  log_level: WARN
+
+aws:
+  access_key_id:
+  secret_access_key:
+
+aws_lambda:
+  region_name: eu-west-1
+  execution_role: arn:aws:iam::111560892610:role/lithops-execution-role
+
+aws_s3:
+  region_name: eu-west-1
+  storage_bucket: lithops-tom-data

.github/workflows/.lithops_config_gcp.template

@@ -0,0 +1,12 @@
+lithops:
+  backend: gcp_functions
+  storage: gcp_storage
+  log_level: WARN
+
+gcp:
+  region: europe-west2
+  credentials_path:
+
+gcp_storage:
+  region: europe-west2
+  storage_bucket: tom-lithops-data

.github/workflows/lithops-images.yml

@@ -0,0 +1,86 @@
+name: Lithops images
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: ${{ matrix.lithops-backend }} ${{ matrix.os }} py${{ matrix.python_version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            python-version: "3.11"
+            requirements-file: ci/requirements-lithops-aws.txt
+            lithops-config: .github/workflows/.lithops_config_aws
+            lithops-backend: aws_lambda
+            lithops-build-file: ci/docker/Dockerfile_aws_lambda_main
+            lithops-memory: 2000
+          - os: ubuntu-latest
+            python-version: "3.11"
+            requirements-file: ci/requirements-lithops-gcp.txt
+            lithops-config: .github/workflows/.lithops_config_gcp
+            lithops-backend: gcp_functions
+            lithops-build-file: ci/requirements-lithops-gcp.txt
+            lithops-memory: 2048
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Copy lithops configuration templates
+        run: |
+          cp $GITHUB_WORKSPACE/.github/workflows/.lithops_config_aws.template $GITHUB_WORKSPACE/.github/workflows/.lithops_config_aws
+          cp $GITHUB_WORKSPACE/.github/workflows/.lithops_config_gcp.template $GITHUB_WORKSPACE/.github/workflows/.lithops_config_gcp
+
+      - name: Google auth
+        id: 'auth'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+          create_credentials_file: true
+
+      - name: Configure lithops AWS
+        uses: microsoft/variable-substitution@v1
+        with:
+          files: ${{ github.workspace }}/.github/workflows/.lithops_config_aws
+        env:
+          aws.access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws.secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Configure lithops GCP
+        uses: microsoft/variable-substitution@v1
+        with:
+          files: ${{ github.workspace }}/.github/workflows/.lithops_config_gcp
+        env:
+          gcp.credentials_path: ${{ steps.auth.outputs.credentials_file_path }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r ${{ matrix.requirements-file }}
+
+      - name: Build and deploy images
+        env:
+          LITHOPS_CONFIG_FILE: ${{ github.workspace }}/${{ matrix.lithops-config }}
+        run: |
+          lithops runtime build -b ${{ matrix.lithops-backend }} -f ${{ matrix.lithops-build-file }} cubed-runtime-main
+          lithops runtime deploy -b ${{ matrix.lithops-backend }} --memory ${{ matrix.lithops-memory }} cubed-runtime-main

ci/docker/Dockerfile_aws_lambda_main

@@ -0,0 +1,76 @@
+# Python 3.6
+#FROM python:3.6-slim-buster
+
+# Python 3.7
+#FROM python:3.7-slim-buster
+
+# Python 3.8
+#FROM python:3.8-slim-buster
+
+# Python 3.9
+#FROM python:3.9-slim-buster
+
+# Python 3.10
+#FROM python:3.10-slim-buster
+
+# Python 3.11
+FROM python:3.11-slim-buster
+
+
+RUN apt-get update \
+    # Install aws-lambda-cpp build dependencies
+    && apt-get install -y \
+      g++ \
+      make \
+      cmake \
+      unzip \
+      git \
+    # cleanup package lists, they are not used anymore in this image
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-cache search linux-headers-generic
+
+ARG FUNCTION_DIR="/function"
+
+# Copy function code
+RUN mkdir -p ${FUNCTION_DIR}
+
+# Update pip
+# NB botocore/boto3 are pinned due to https://github.com/boto/boto3/issues/3648
+#    using versions from https://github.com/aio-libs/aiobotocore/blob/72b8dd5d7d4ef2f1a49a0ae0c37b47e5280e2070/setup.py
+#    due to s3fs dependency
+RUN pip install --upgrade --ignore-installed pip wheel six setuptools \
+    && pip install --upgrade --no-cache-dir --ignore-installed \
+        awslambdaric \
+        botocore==1.29.76 \
+        boto3==1.26.76 \
+        redis \
+        httplib2 \
+        requests \
+        numpy \
+        scipy \
+        pandas \
+        pika \
+        kafka-python \
+        cloudpickle \
+        ps-mem \
+        tblib
+
+# Set working directory to function root directory
+WORKDIR ${FUNCTION_DIR}
+
+# Add Lithops
+COPY lithops_lambda.zip ${FUNCTION_DIR}
+RUN unzip lithops_lambda.zip \
+    && rm lithops_lambda.zip \
+    && mkdir handler \
+    && touch handler/__init__.py \
+    && mv entry_point.py handler/
+
+# Put your dependencies here, using RUN pip install... or RUN apt install...
+
+RUN pip install \
+        -e 'git+https://github.com/cubed-dev/cubed.git#egg=cubed' \
+        s3fs
+
+ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ]
+CMD [ "handler.entry_point.lambda_handler" ]

ci/requirements-lithops-aws.txt

@@ -0,0 +1,4 @@
+git+https://github.com/cubed-dev/cubed.git#egg=cubed
+lithops[aws]
+s3fs
+rich

ci/requirements-lithops-gcp.txt

@@ -0,0 +1,4 @@
+git+https://github.com/cubed-dev/cubed.git#egg=cubed
+lithops[gcp]
+gcsfs
+rich