feat: 248 add load test to GitHub actions (#256)

Added a github action to run the load tests periodically. Added load test scripts.

.github/workflows/api-qa.yml

@@ -2,7 +2,7 @@
 name: Deploy Feeds API - QA
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   workflow_dispatch: # Supports manual deployment
 
 jobs:
@@ -17,7 +17,7 @@ jobs:
       DEPLOYER_SERVICE_ACCOUNT: ${{ vars.QA_MOBILITY_FEEDS_DEPLOYER_SERVICE_ACCOUNT }}
       FEED_API_IMAGE_VERSION: ${{ github.sha }}
       TF_APPLY: true
-      GLOBAL_RATE_LIMIT_REQ_PER_MINUTE: ${{ vars.GLOBAL_RATE_LIMIT_REQ_PER_MINUTE }}   
+      GLOBAL_RATE_LIMIT_REQ_PER_MINUTE: ${{ vars.GLOBAL_RATE_LIMIT_REQ_PER_MINUTE }}
     secrets:
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
       OAUTH2_CLIENT_ID: ${{ secrets.DEV_MOBILITY_FEEDS_OAUTH2_CLIENT_ID}}

.github/workflows/schedule-load-test.yml

@@ -0,0 +1,58 @@
+name: Schedule Load Test every other Friday
+
+on:
+  schedule:
+    - cron: "0 0 * * 5/2"
+  workflow_dispatch: # Supports manual triggering
+
+env:
+  API_BASE_URL: "api-qa.mobilitydatabase.org"
+  # locust parameters. Refer to https://docs.locust.io/en/stable/configuration.html for explanation
+  LOCUST_USERS: 100
+  LOCUST_RATE: 10
+  LOCUST_DURATION: 180
+
+jobs:
+  load-test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install locust
+
+      - name: Get an access token
+        id: getAccessToken
+        run: |
+          set +e  # Do not exit if error. Handle error messages here.
+          REPLY=`curl --location "https://${API_BASE_URL}/v1/tokens" \
+             --header 'Content-Type: application/json' \
+             --data '{ "refresh_token": "${{ secrets.QA_API_TEST_REFRESH_TOKEN }}" }'`
+          [ $? -ne 0 ] && { echo "Error: Cannot obtain access token Reply = \"$REPLY\""; exit 1; }
+          
+          ACCESS_TOKEN=`echo $REPLY | jq -r .access_token`
+          [ $? -ne 0 ] && { echo "Error: Cannot extract access token from reply \"$REPLY\""; exit 1; }
+          [ -z "$ACCESS_TOKEN" ] && { echo "Error: Access token is empty extracted from $REPLY"; exit 1; }
+          
+          echo "ACCESS_TOKEN=$ACCESS_TOKEN" >> $GITHUB_ENV
+
+      - name: Run the load tests
+        run: |         
+          export FEEDS_AUTH_TOKEN="${{ env.ACCESS_TOKEN }}"    # The locust script uses this variable
+          locust -f ./load-test/gtfs_user_test.py --host=https://${API_BASE_URL} \
+            -u ${LOCUST_USERS} -r ${LOCUST_RATE} --headless -t ${LOCUST_DURATION} --only-summary --csv locust_results
+          
+      - name: Upload load test results as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: load_test_results
+          path: locust_results_*

load-test/README.md

@@ -0,0 +1,63 @@
+# API load tests
+
+This script `gtfs_user_test.py`, defines a set of tasks for load testing the GTFS API. Each task represents a different API endpoint that will be hit during the load test.
+
+## Setup
+
+### Install Locust
+
+Locust is a Python library, so you can install it with pip. Run the following command in your terminal:
+
+```
+pip install locust
+```
+## Authorization
+
+Set the `FEEDS_AUTH_TOKEN` environment variable to the access token you wish to use for connecting to the rest API. 
+All requests sent during the load test will include this token.
+
+## Start a Load Test
+
+To start a load test on QA environment, run the following command in your terminal:
+```
+locust -f gtfs_user_test.py --host=https://api-qa.mobilitydatabase.org -u 100 -r 10
+```
+The -u option specifies the total number of users to simulate, and the -r option specifies the hatch rate (number of users to start per second)
+
+### Tasks
+
+### `feeds`
+
+This task hits the `/v1/feeds` endpoint, which returns a list of all feeds.
+
+### `feed_byId`
+
+This task hits the `/v1/feeds/mdb-10` endpoint, which returns the feed with the ID `mdb-10`.
+
+### `gtfs_feeds`
+
+This task hits the `/v1/gtfs_feeds` endpoint, which returns a list of all GTFS feeds.
+
+### `gtfs_feed_byId`
+
+This task hits the `/v1/gtfs_feeds/mdb-10` endpoint, which returns the GTFS feed with the ID `mdb-10`.
+
+### `gtfs_realtime_feeds`
+
+This task hits the `/v1/gtfs_rt_feeds` endpoint, which returns a list of all GTFS realtime feeds.
+
+### `gtfs_realtime_feed_byId`
+
+This task hits the `/v1/gtfs_rt_feeds/mdb-1852` endpoint, which returns the GTFS realtime feed with the ID `mdb-1852`.
+
+### `gtfs_feeds_datasets`
+
+This task hits the `/v1/gtfs_feeds/mdb-10/datasets` endpoint, which returns a list of all datasets for the GTFS feed with the ID `mdb-10`.
+
+### `gtfs_dataset`
+
+This task hits the `/v1/datasets/gtfs/mdb-10` endpoint, which returns the dataset for the GTFS feed with the ID `mdb-10`.
+
+## Wait Time
+
+The `wait_time` is set to a random duration between 5 and 15 seconds. This means that after each task is executed, the script will wait for a duration between 5 and 15 seconds before executing the next task.

load-test/gtfs_user_test.py

@@ -1,41 +1,72 @@
+import sys
+import json
+
 from locust import HttpUser, TaskSet, task, between
 import os
 
 class gtfs_user(HttpUser):
-    wait_time = between(5, 15)
+
+    wait_time = between(.1, 1)
+
+    def print_response(self, response, indent):
+        print(indent, "Contents of response:")
+        print(indent, "    text: ", response.text)
+        print(indent, "    status code:", response.status_code)
+        print(indent, "    headers:", response.headers)
+        print(indent, "    URL:", response.url)
+        print(indent, "    content:", response.content)
+
+    def get_valid(self, endpoint, allow404=False):
+        try:
+            response = self.client.get(endpoint, allow_redirects=False)
+            if allow404 and response.status_code == 404:
+                return
+            if response.status_code >= 300:
+                print("Error in response.")
+                self.print_response(response, "")
+                sys.exit(1)
+            json_response = response.json()  # Try to parse response content as JSON
+        except json.JSONDecodeError:
+            print("Error: Response not json.")
+            self.print_response(response, "")
+            sys.exit(1)
 
     def on_start(self):
-        self.client.headers = {'Authorization': os.getenv('FEEDS_AUTH_TOKEN')}
+        access_token = os.environ.get('FEEDS_AUTH_TOKEN')
+        if access_token is None or access_token == "":
+            print("Error: FEEDS_AUTH_TOKEN is not defined or empty")
+            sys.exit(1)
+        self.client.headers = {'Authorization': "Bearer " + access_token}
 
     @task
     def feeds(self):
-        self.client.get("/v1/feeds")
-    
+        self.get_valid("/v1/feeds?limit=10")
+
     @task
     def feed_byId(self):
-        self.client.get("/v1/feeds/mdb-10")
-
+        # Allow error 404 since we are not sure the feed ID exists
+        self.get_valid("/v1/feeds/mdb-10", allow404=True)
+
     @task
     def gtfs_feeds(self):
-        self.client.get("/v1/gtfs_feeds")
+        self.get_valid("/v1/gtfs_feeds?limit=1000")
 
     @task
     def gtfs_feed_byId(self):
-        self.client.get("/v1/gtfs_feeds/mdb-10")
+        self.get_valid("/v1/gtfs_feeds/mdb-10", allow404=True)
 
     @task
     def gtfs_realtime_feeds(self):
-        self.client.get("/v1/gtfs_rt_feeds")
+        self.get_valid("/v1/gtfs_rt_feeds?limit=1000")
 
     @task
     def gtfs_realtime_feed_byId(self):
-        self.client.get("/v1/gtfs_rt_feeds/mdb-1852")
-    
+        self.get_valid("/v1/gtfs_rt_feeds/mdb-1333", allow404=True)
+
     @task
     def gtfs_feeds_datasets(self):
-        self.client.get("/v1/gtfs_feeds/mdb-10/datasets")
+        self.get_valid("/v1/gtfs_feeds/mdb-10/datasets", allow404=True)
+
     @task
     def gtfs_dataset(self):
-        self.client.get("/v1/datasets/gtfs/mdb-10")
-
-
+        self.get_valid("/v1/datasets/gtfs/mdb-10-202402071805", allow404=True)