[FAPI Conformance] Add email and Gchat sending option with test result summary

.github/workflows/fapi-oidc-conformance-test.yml

@@ -1,6 +1,6 @@
 # This workflow will test IS for OIDC FAPI conformance
 
-name: FAPI-OIDC-Conformance-Test
+name: FAPI OIDC Conformance Test
 
 on:
   schedule:
@@ -18,6 +18,14 @@ on:
       conformance-suite-version:
         description: 'Conformance suite branch to clone in https://gitlab.com/openid/conformance-suite.git (Ex: release-v5.1.10). If not provided, latest release tag branch is used.'
         required: false
+      send-email:
+        description: 'Send test results to email'
+        required: true
+        default: 'no'
+      send-chat:
+        description: 'Send test results to google chat'
+        required: true
+        default: 'yes'
 
 jobs:
   build:
@@ -216,7 +224,70 @@ jobs:
       with:
         name: test-logs
         path: ./*log.txt
-
-
+
+    - name: Send Email
+      if: always()
+      run: |
+        INPUT=${{github.event.inputs.send-email}}
+        if [[ -z "${INPUT}" ]]; then
+          INPUT="no"
+        fi
+        SEND_EMAIL=${INPUT^^}
+        if [ $SEND_EMAIL == "YES" ]; then
+          echo "============="
+          echo "Sending Email"
+          echo "============="
+          CONFORMANCE_SUITE_URL=https://localhost:8443
+          INPUT_TAG=${{github.event.inputs.tag}}
+          if [[ -z "${INPUT_TAG}" ]]; then
+            RESOURCE="built-from-latest-source"
+          else
+            RESOURCE=${{github.event.inputs.tag}}
+          fi
+          RECEIVER_LIST=${{secrets.FAPI_RECEIVER_LIST}}
+          if [[ -z "${RECEIVER_LIST}" ]]; then
+            RECEIVER_LIST=${{secrets.RECEIVER_LIST}}
+          fi
+          python3 ./product-is/oidc-conformance-tests/send_email.py $CONFORMANCE_SUITE_URL $GITHUB_RUN_NUMBER ${{job.status}} ${{github.repository}} ${{github.run_id}} ${{secrets.SENDER_EMAIL}} ${{secrets.PASSWORD}} ${{secrets.FAPI_RECEIVER_LIST}} $RESOURCE
+        elif [ $SEND_EMAIL == "NO" ]; then
+          echo "========================================"
+          echo "Skipped Sending Email"
+          echo "========================================"
+        else
+          echo "================================================================="
+          echo "Invalid parameter value. Skipped sending email"
+          echo "================================================================="
+        fi
 
+    - name: Send Chat message
+      if: always()
+      run: |
+        INPUT=${{github.event.inputs.send-chat}}
+        if [[ -z "${INPUT}" ]]; then
+          INPUT="yes"
+        fi
+
+        INPUT_TAG=${{github.event.inputs.tag}}
+          if [[ -z "${INPUT_TAG}" ]]; then
+            RESOURCE="built from latest source"
+          else
+            RESOURCE=${{github.event.inputs.tag}}
+          fi
+
+        SEND_CHAT=${INPUT^^}
+        if [ $SEND_CHAT == "YES" ]; then
+          echo "==========================="
+          echo "Sending Google Chat Message"
+          echo "==========================="
+          CONFORMANCE_SUITE_URL=https://localhost:8443
+          python3 ./product-is/oidc-conformance-tests/send_chat.py "$CONFORMANCE_SUITE_URL" "$GITHUB_RUN_NUMBER" "${{job.status}}" "${{github.repository}}" "${{github.run_id}}" "${{secrets.GOOGLE_CHAT_WEBHOOK_OIDC_TEST}}" "$RESOURCE" "FAPI OIDC Conformance Test"
+        elif [ $SEND_CHAT == "NO" ]; then
+          echo "========================================"
+          echo "Skipped Sending Google Chat Message"
+          echo "========================================"
+        else
+          echo "================================================================="
+          echo "Invalid parameter value. Skipped sending google chat message"
+          echo "================================================================="
+        fi
 

.github/workflows/oidc-conformance-test.yml

@@ -184,7 +184,7 @@ jobs:
           echo "Sending Google Chat Message"
           echo "==========================="
           CONFORMANCE_SUITE_URL=https://localhost:8443
-          python3 ./product-is/oidc-conformance-tests/send_chat.py "$CONFORMANCE_SUITE_URL" "$GITHUB_RUN_NUMBER" "${{job.status}}" "${{github.repository}}" "${{github.run_id}}" "${{secrets.GOOGLE_CHAT_WEBHOOK_OIDC_TEST}}" "$RESOURCE"
+          python3 ./product-is/oidc-conformance-tests/send_chat.py "$CONFORMANCE_SUITE_URL" "$GITHUB_RUN_NUMBER" "${{job.status}}" "${{github.repository}}" "${{github.run_id}}" "${{secrets.GOOGLE_CHAT_WEBHOOK_OIDC_TEST}}" "$RESOURCE" "OIDC Conformance Test"
         elif [ $SEND_CHAT == "NO" ]; then
           echo "========================================"
           echo "Skipped Sending Google Chat Message"

oidc-conformance-tests/send_chat.py

@@ -27,6 +27,7 @@
 github_run_id = str(sys.argv[5])
 google_chat_webhook = sys.argv[6]
 wso2_is_version = sys.argv[7]
+workflow_name = sys.argv[8]
 
 failed_count = 0
 warnings_count = 0
@@ -56,7 +57,7 @@
     "cards": [
         {
             "header": {
-                "title": "OIDC Conformance Test",
+                "title": workflow_name,
                 "subtitle": "GitHub Action #" + github_run_number
 
             },
@@ -65,7 +66,7 @@
                     "widgets": [
                         {
                             "textParagraph": {
-                                "text": f"<b>Identity Server {wso2_is_version} </b>"
+                                "text": f"Identity Server version: <b>{wso2_is_version}</b>"
                             }
                         },
                         {

oidc-fapi-conformance-tests/README.md

@@ -0,0 +1,32 @@
+This folder contains the configuration files and scripts that can be used to automate OIDC FAPI conformance testing. These tests can be run using GitHub actions or locally.
+
+## Testing using GitHub actions
+
+OIDC FAPI conformance test workflow can be used to for this purpose.
+1. Go to the Actions tab in the wso2/product-is github repository
+2. Click on FAPI OIDC Conformance Test workflow
+3. Click on Run workflow
+4. You need to provide the product-is release tag version you want to test (by default, it builds the latest IS by source)
+5. Also by default FAPI conformance suite is built from the latest released branch in https://gitlab.com/openid/conformance-suite.git. You can run against a specific conformance suite version also.
+6. Set 'Send test results to email' to 'yes' if you want to send test summary to a list of pre-configured email addresses. (default is 'no')
+7. Click on Run workflow
+8. After tests are completed you can view test results on the test summary page
+9.  Two types of artifacts are saved after the test execution is completed
+   - test-logs - a log file is generated for each test plan. This log contains a summary of test cases with failures and warnings
+   - test-results - a zip file is generated for each test plan. You can use a web browser to view a detailed report of the test plan by extracting this zip file
+
+An email containing the same test summary will also be sent to a pre-configured list of email addresses. The sender email, password and the receiver emil list are the same as for OIDC action build. If you need to have a seperate receiver email list for FAPI Conformance action build,
+* Create a github secret with name `FAPI_RECEIVER_LIST` and add the list of receiver emails seperated by commas.
+
+Default configuration is to use Gmail SMTP server. You can change that by modifying `SMTP_SERVER` and `SMTP_SERVER_PORT` in `constants_fapi.py`
+
+This workflow is scheduled to run daily at 08:30 UTC (2:00 AM SL time) and will also automatically trigger after a release or a pre-release.
+
+To locally setup and run the test suite, follow the [fapi-oidc-conformance-test.yml](.github/workflows/fapi-oidc-conformance-test.yml) script and execute the steps.
+
+## Test Profiles
+
+Running fapi test profiles are in [test_runner_fapi.sh](oidc-fapi-conformance-tests/test_runner_fapi.sh) script. Currently Running profiles,
+* private_key_jwt
+* mtls
+

oidc-fapi-conformance-tests/configure_is_fapi.py

@@ -14,10 +14,6 @@
 from config import browser_configuration
 from config.client_configs import client_configs
 
-# path to product is zip file
-path_to_is_zip = str(sys.argv[1])
-print("Path to zip: ", path_to_is_zip)
-
 def decode_secret(secret):
     decoded_string=base64.b64decode(secret+"=").decode("utf-8")
     decoded_json = json.loads(decoded_string)
@@ -153,7 +149,10 @@ def addCertsToKeystore(rootCertPath, issuerCertPath, ISPath):
 
 
 # unpack product-is zip file and run
-def unpack_and_run(zip_file_name):
+def unpack_and_run():
+    # path to product is zip file
+    zip_file_name = str(sys.argv[1])
+    print("Path to zip: ", zip_file_name)
     try:
         # extract IS zip
         with ZipFile(zip_file_name, 'r') as zip_file:
@@ -261,7 +260,7 @@ def is_process_running(process_name):
 warnings.filterwarnings("ignore")
 
 if not is_process_running("wso2server"):
-    unpack_and_run(path_to_is_zip)
+    unpack_and_run()
 else:
     print("\n>>> IS already running ...")
 print ("==============================================\n")

oidc-fapi-conformance-tests/constants_fapi.py

@@ -156,3 +156,8 @@
     "jwks_uri": JWKS_2,
     "require_pushed_authorization_requests" : "true",
 }
+
+
+SMTP_SERVER = "smtp.gmail.com"
+
+SMTP_SERVER_PORT = 465