python packaged server tests (FF-3494) (#88)

* python packaged server tests (FF-3494)

* python does not support bandits or dynamic types

* run package test on PR

.github/workflows/test-sdk-packages.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+  pull_request:
   workflow_dispatch:
 
 jobs:
@@ -18,3 +19,15 @@ jobs:
       sdkName: 'eppo/php-sdk'
       sdkRelayDir: 'php-sdk-relay'
     secrets: inherit
+
+  test-python-sdk:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ['linux']
+    uses: ./.github/workflows/test-server-sdk.yml
+    with:
+      platform: ${{ matrix.platform }}
+      sdkName: 'eppo/python-sdk'
+      sdkRelayDir: 'python-sdk-relay'
+    secrets: inherit

.github/workflows/test-server-sdk.yml

@@ -49,7 +49,9 @@ jobs:
 
 
     - name: "Checkout"
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.ref }}
 
     # Set up docker (macos runners)
     - id: setup-docker

package-testing/php-sdk-relay/docker-run.sh

@@ -12,7 +12,7 @@ docker remove php-relay
 
 docker build . -t Eppo-exp/php-sdk-relay:$VERSION
 
-docker run  -p $SDK_RELAY_PORT:$SDK_RELAY_PORT \
+docker run -p $SDK_RELAY_PORT:$SDK_RELAY_PORT \
   --add-host host.docker.internal:host-gateway \
   -e SDK_REF \
   -e EPPO_BASE_URL \

package-testing/python-sdk-relay/.gitignore

@@ -0,0 +1,3 @@
+bin
+lib
+venv

package-testing/python-sdk-relay/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.12
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+# Copy the source code
+COPY src/ ./src/
+
+CMD ["python", "/app/src/server.py"]

package-testing/python-sdk-relay/README.md

@@ -0,0 +1,17 @@
+# Python Testing Server
+
+Post test case files to this server and check the results against what's expected.
+
+## Running locally with Docker
+
+Build the docker image:
+
+```shell
+docker build -t Eppo-exp/python-sdk-relay .
+```
+
+Run the docker container:
+
+```shell
+./docker-run.sh
+```

package-testing/python-sdk-relay/docker-run.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Default is to use the latest build
+VERSION="${1:-latest}"
+
+echo "Starting deployment with version: $VERSION"
+
+if [ -e .env ]; then
+  echo "Loading environment variables from .env file"
+  source .env
+fi
+
+echo "Stopping existing container..."
+docker stop python-relay
+echo "Removing existing container..."
+docker remove python-relay
+
+echo "Building new image..."
+docker build . -t Eppo-exp/python-sdk-relay:$VERSION
+
+echo "Starting new container..."
+docker run -p $SDK_RELAY_PORT:$SDK_RELAY_PORT \
+  --add-host host.docker.internal:host-gateway \
+  -e SDK_REF \
+  -e EPPO_BASE_URL \
+  -e SDK_RELAY_PORT \
+  --name python-relay \
+  --rm \
+  -t Eppo-exp/python-sdk-relay:$VERSION;

package-testing/python-sdk-relay/requirements.txt

@@ -0,0 +1,2 @@
+flask
+eppo-server-sdk==4.1.0

package-testing/python-sdk-relay/src/server.py

@@ -0,0 +1,169 @@
+import eppo_client
+import eppo_client.bandit
+
+from flask import Flask, request, jsonify
+from os import environ
+from dataclasses import dataclass
+from eppo_client.config import Config, AssignmentLogger
+
+app = Flask(__name__)
+
+
+class LocalAssignmentLogger(AssignmentLogger):
+    def log_assignment(self, assignment):
+        print(f"Assignment: {assignment}")
+
+
+@dataclass
+class AssignmentRequest:
+    flag: str
+    subject_key: str
+    subject_attributes: dict
+    assignment_type: str
+    default_value: any
+
+
+@app.route('/', methods=['GET'])
+def health_check():
+    return "OK"
+
+@app.route('/sdk/reset', methods=['POST'])
+def reset_sdk():
+    initialize_client_and_wait()
+
+    return "Reset complete"
+
+@app.route('/sdk/details', methods=['GET'])
+def get_sdk_details():
+    return jsonify({
+        "sdkName": "python-sdk", 
+        "sdkVersion": "4.1.0",
+        "supportsBandits": False,
+        "supportsDynamicTyping": False
+    })
+
+@app.route('/flags/v1/assignment', methods=['POST'])
+def handle_assignment():
+    data = request.json
+    request_obj = AssignmentRequest(
+        flag=data['flag'],
+        subject_key=data['subjectKey'],
+        subject_attributes=data['subjectAttributes'],
+        assignment_type=data['assignmentType'],
+        default_value=data['defaultValue']
+    )
+    print(f"Request object: {request_obj}")
+
+    client = eppo_client.get_instance()
+
+    try:
+        match request_obj.assignment_type:
+            case 'BOOLEAN':
+                result = client.get_boolean_assignment(
+                    request_obj.flag, 
+                    request_obj.subject_key, 
+                    request_obj.subject_attributes, 
+                    bool(request_obj.default_value)
+                )
+            case 'INTEGER':
+                result = client.get_integer_assignment(
+                    request_obj.flag, 
+                    request_obj.subject_key, 
+                    request_obj.subject_attributes, 
+                    int(request_obj.default_value)
+                )
+            case 'STRING':
+                result = client.get_string_assignment(
+                    request_obj.flag, 
+                    request_obj.subject_key, 
+                    request_obj.subject_attributes, 
+                    request_obj.default_value
+                )
+            case 'NUMERIC':
+                result = client.get_numeric_assignment(
+                    request_obj.flag, 
+                    request_obj.subject_key, 
+                    request_obj.subject_attributes, 
+                    float(request_obj.default_value)
+                )
+            case 'JSON':
+                result = client.get_json_assignment(
+                    request_obj.flag, 
+                    request_obj.subject_key, 
+                    request_obj.subject_attributes, 
+                    request_obj.default_value
+                )
+
+        response = {
+            "result": result,
+            "assignmentLog": [],
+            "banditLog": [],
+            "error": None
+        }
+        print(f"response: {response}")
+        return jsonify(response)
+    except Exception as e:
+        print(f"Error processing assignment: {str(e)}")
+        response = {
+            "result": None,
+            "assignmentLog": [],
+            "banditLog": [],
+            "error": str(e)
+        }
+        return jsonify(response)
+
+@dataclass
+class BanditActionRequest:
+    flag: str
+    subject_key: str
+    subject_attributes: dict
+    actions: list
+    default_value: any
+
+
+@app.route('/bandits/v1/action', methods=['POST'])
+def handle_bandit():
+    data = request.json
+    request_obj = BanditActionRequest(
+        flag=data['flag'],
+        subject_key=data['subjectKey'],
+        subject_attributes=data['subjectAttributes'],
+        default_value=data['defaultValue'],
+        actions=data['actions']
+    )
+    print(f"Request object: {request_obj}")
+
+    # TODO: Implement bandit logic
+    return jsonify({
+        "result": "action",
+        "assignmentLog": [],
+        "banditLog": [],
+        "error": None
+    })
+
+def initialize_client_and_wait():
+    print("Initializing client")
+    api_key = environ.get('EPPO_API_KEY', 'NOKEYSPECIFIED')
+    base_url = environ.get('EPPO_BASE_URL', 'http://localhost:5000/api')
+
+    client_config = Config(
+        api_key=api_key,
+        base_url=base_url,
+        assignment_logger=LocalAssignmentLogger()
+    )
+    eppo_client.init(client_config)
+    client = eppo_client.get_instance()
+    client.wait_for_initialization()
+    print("Client initialized")
+
+if __name__ == "__main__":
+    initialize_client_and_wait()
+
+    port = int(environ.get('SDK_RELAY_PORT', 7001))
+    host = environ.get('SDK_RELAY_HOST', '0.0.0.0')
+    print(f"Starting server on {host}:{port}")
+    app.run(
+        host=host,
+        port=port,
+        debug=True  # Add debug mode
+    )

package-testing/sdk-test-runner/README.md

@@ -91,7 +91,7 @@ The following env variable can be set when running the `test-sdk.sh` script
 
 The following components are required to use the the package test runner with a new SDK
 
-1. An **SDK relay server**. This is a REST server running at `localhost:4000` resonding to the [Asssignment and Bandit Request API](#sdk-relay-server)
+1. An **SDK relay server**. This is a REST server running at `localhost:4000` responding to the [Asssignment and Bandit Request API](#sdk-relay-server)
    1. OR, an **SDK relay client**. This is a client application that connects to the SDK test runner via `socket.io` and responses to [Assignment requests](#sdk-relay-client)
 2. Launch Script:
    1. A `build-and-run-<platform>.sh` file which fully configures the environment then initiates a [build and run of the relay server application](#build-and-runsh) **using the specified version of the SDK package**. <platform> is one of `linux`, `macos`, or `windows`.
@@ -201,13 +201,11 @@ Any non-empty response
 
 ##### SDK Details
 
-`POST /sdk/details`
+`GET /sdk/details`
 
 If possible, the SDK relay server should respond with the `sdkName` and `sdkVersion` in use. This may not be directly possible with all SDKs.
 If the SDK does not support Bandits or dynamic typing, the test runner will skip the related test cases if the corresponding values are `false`.
 
-`GET /sdk/details`
-
 ```ts
 // Expected response data:
 type SDKDetailsResponse = {

package-testing/sdk-test-runner/test-sdk.sh

@@ -38,7 +38,7 @@ function wait_for_url() {
 
   while [[ $attempt -le $max_attempts ]]; do
     curl --silent --output /dev/null --fail "$url" && { return 1; }
-    echo "Waiting attempt number ${attempt}"
+    echo "Attempt number ${attempt}; waiting for $url"
     sleep 5
     ((attempt++))
   done
@@ -102,6 +102,16 @@ else
     cp ../scenarios.json test-data/
 fi
 
+echo "EPPO_API_HOST $EPPO_API_HOST"
+echo "EPPO_API_PORT=$EPPO_API_PORT"
+echo "EPPO_BASE_URL=$EPPO_BASE_URL"
+
+echo "SDK_RELAY_HOST=$SDK_RELAY_HOST"
+echo "SDK_RELAY_PORT=$SDK_RELAY_PORT"
+
+echo "EPPO_SCENARIO_FILE=$EPPO_SCENARIO_FILE"
+echo "EPPO_TEST_DATA_PATH=$EPPO_TEST_DATA_PATH"
+
 
 case "$command" in
     server)
@@ -120,7 +130,7 @@ case "$command" in
           --name eppo-api \
           -t Eppo-exp/testing-api:latest
 
-        echo_yellow "    ... Waiting to verify server is up"
+        echo_yellow "    ... Waiting to verify Test API server is up"
 
         wait_for_url http://${EPPO_API_HOST}:${EPPO_API_PORT} 
         if [[ $? -eq 0 ]]; then
@@ -159,7 +169,7 @@ case "$command" in
         popd
 
 
-        echo_yellow "    ... Waiting to verify server is up"
+        echo_yellow "    ... Waiting to verify SDK Relay server is up"
         wait_for_url http://${SDK_RELAY_HOST}:${SDK_RELAY_PORT} 
         if [[ $? -eq 0 ]]; then
           exit_with_message "    ... SDK Relay server failed to start"