diff --git a/.github/workflows/fapi-oidc-conformance-test.yml b/.github/workflows/fapi-oidc-conformance-test.yml
index f18f9165619..115e0529bbf 100644
--- a/.github/workflows/fapi-oidc-conformance-test.yml
+++ b/.github/workflows/fapi-oidc-conformance-test.yml
@@ -1,6 +1,6 @@
# This workflow will test IS for OIDC FAPI conformance
-name: FAPI-OIDC-Conformance-Test
+name: FAPI OIDC Conformance Test
on:
schedule:
@@ -18,6 +18,14 @@ on:
conformance-suite-version:
description: 'Conformance suite branch to clone in https://gitlab.com/openid/conformance-suite.git (Ex: release-v5.1.10). If not provided, latest release tag branch is used.'
required: false
+ send-email:
+ description: 'Send test results to email'
+ required: true
+ default: 'no'
+ send-chat:
+ description: 'Send test results to google chat'
+ required: true
+ default: 'yes'
jobs:
build:
@@ -216,7 +224,70 @@ jobs:
with:
name: test-logs
path: ./*log.txt
-
-
+
+ - name: Send Email
+ if: always()
+ run: |
+ INPUT=${{github.event.inputs.send-email}}
+ if [[ -z "${INPUT}" ]]; then
+ INPUT="no"
+ fi
+ SEND_EMAIL=${INPUT^^}
+ if [ $SEND_EMAIL == "YES" ]; then
+ echo "============="
+ echo "Sending Email"
+ echo "============="
+ CONFORMANCE_SUITE_URL=https://localhost:8443
+ INPUT_TAG=${{github.event.inputs.tag}}
+ if [[ -z "${INPUT_TAG}" ]]; then
+ RESOURCE="built-from-latest-source"
+ else
+ RESOURCE=${{github.event.inputs.tag}}
+ fi
+ RECEIVER_LIST=${{secrets.FAPI_RECEIVER_LIST}}
+ if [[ -z "${RECEIVER_LIST}" ]]; then
+ RECEIVER_LIST=${{secrets.RECEIVER_LIST}}
+ fi
+ python3 ./product-is/oidc-conformance-tests/send_email.py $CONFORMANCE_SUITE_URL $GITHUB_RUN_NUMBER ${{job.status}} ${{github.repository}} ${{github.run_id}} ${{secrets.SENDER_EMAIL}} ${{secrets.PASSWORD}} ${{secrets.FAPI_RECEIVER_LIST}} $RESOURCE
+ elif [ $SEND_EMAIL == "NO" ]; then
+ echo "========================================"
+ echo "Skipped Sending Email"
+ echo "========================================"
+ else
+ echo "================================================================="
+ echo "Invalid parameter value. Skipped sending email"
+ echo "================================================================="
+ fi
+ - name: Send Chat message
+ if: always()
+ run: |
+ INPUT=${{github.event.inputs.send-chat}}
+ if [[ -z "${INPUT}" ]]; then
+ INPUT="yes"
+ fi
+
+ INPUT_TAG=${{github.event.inputs.tag}}
+ if [[ -z "${INPUT_TAG}" ]]; then
+ RESOURCE="built from latest source"
+ else
+ RESOURCE=${{github.event.inputs.tag}}
+ fi
+
+ SEND_CHAT=${INPUT^^}
+ if [ $SEND_CHAT == "YES" ]; then
+ echo "==========================="
+ echo "Sending Google Chat Message"
+ echo "==========================="
+ CONFORMANCE_SUITE_URL=https://localhost:8443
+ python3 ./product-is/oidc-conformance-tests/send_chat.py "$CONFORMANCE_SUITE_URL" "$GITHUB_RUN_NUMBER" "${{job.status}}" "${{github.repository}}" "${{github.run_id}}" "${{secrets.GOOGLE_CHAT_WEBHOOK_OIDC_TEST}}" "$RESOURCE" "FAPI OIDC Conformance Test"
+ elif [ $SEND_CHAT == "NO" ]; then
+ echo "========================================"
+ echo "Skipped Sending Google Chat Message"
+ echo "========================================"
+ else
+ echo "================================================================="
+ echo "Invalid parameter value. Skipped sending google chat message"
+ echo "================================================================="
+ fi
diff --git a/.github/workflows/oidc-conformance-test.yml b/.github/workflows/oidc-conformance-test.yml
index 425e5e8b5e6..9d69277bde9 100644
--- a/.github/workflows/oidc-conformance-test.yml
+++ b/.github/workflows/oidc-conformance-test.yml
@@ -184,7 +184,7 @@ jobs:
echo "Sending Google Chat Message"
echo "==========================="
CONFORMANCE_SUITE_URL=https://localhost:8443
- python3 ./product-is/oidc-conformance-tests/send_chat.py "$CONFORMANCE_SUITE_URL" "$GITHUB_RUN_NUMBER" "${{job.status}}" "${{github.repository}}" "${{github.run_id}}" "${{secrets.GOOGLE_CHAT_WEBHOOK_OIDC_TEST}}" "$RESOURCE"
+ python3 ./product-is/oidc-conformance-tests/send_chat.py "$CONFORMANCE_SUITE_URL" "$GITHUB_RUN_NUMBER" "${{job.status}}" "${{github.repository}}" "${{github.run_id}}" "${{secrets.GOOGLE_CHAT_WEBHOOK_OIDC_TEST}}" "$RESOURCE" "OIDC Conformance Test"
elif [ $SEND_CHAT == "NO" ]; then
echo "========================================"
echo "Skipped Sending Google Chat Message"
diff --git a/oidc-conformance-tests/send_chat.py b/oidc-conformance-tests/send_chat.py
index 0cbaf1c8659..6030cfe5e1e 100644
--- a/oidc-conformance-tests/send_chat.py
+++ b/oidc-conformance-tests/send_chat.py
@@ -27,6 +27,7 @@
github_run_id = str(sys.argv[5])
google_chat_webhook = sys.argv[6]
wso2_is_version = sys.argv[7]
+workflow_name = sys.argv[8]
failed_count = 0
warnings_count = 0
@@ -56,7 +57,7 @@
"cards": [
{
"header": {
- "title": "OIDC Conformance Test",
+ "title": workflow_name,
"subtitle": "GitHub Action #" + github_run_number
},
@@ -65,7 +66,7 @@
"widgets": [
{
"textParagraph": {
- "text": f"Identity Server {wso2_is_version} "
+ "text": f"Identity Server version: {wso2_is_version}"
}
},
{
diff --git a/oidc-fapi-conformance-tests/README.md b/oidc-fapi-conformance-tests/README.md
new file mode 100644
index 00000000000..35800c5982a
--- /dev/null
+++ b/oidc-fapi-conformance-tests/README.md
@@ -0,0 +1,32 @@
+This folder contains the configuration files and scripts that can be used to automate OIDC FAPI conformance testing. These tests can be run using GitHub actions or locally.
+
+## Testing using GitHub actions
+
+OIDC FAPI conformance test workflow can be used to for this purpose.
+1. Go to the Actions tab in the wso2/product-is github repository
+2. Click on FAPI OIDC Conformance Test workflow
+3. Click on Run workflow
+4. You need to provide the product-is release tag version you want to test (by default, it builds the latest IS by source)
+5. Also by default FAPI conformance suite is built from the latest released branch in https://gitlab.com/openid/conformance-suite.git. You can run against a specific conformance suite version also.
+6. Set 'Send test results to email' to 'yes' if you want to send test summary to a list of pre-configured email addresses. (default is 'no')
+7. Click on Run workflow
+8. After tests are completed you can view test results on the test summary page
+9. Two types of artifacts are saved after the test execution is completed
+ - test-logs - a log file is generated for each test plan. This log contains a summary of test cases with failures and warnings
+ - test-results - a zip file is generated for each test plan. You can use a web browser to view a detailed report of the test plan by extracting this zip file
+
+An email containing the same test summary will also be sent to a pre-configured list of email addresses. The sender email, password and the receiver emil list are the same as for OIDC action build. If you need to have a seperate receiver email list for FAPI Conformance action build,
+* Create a github secret with name `FAPI_RECEIVER_LIST` and add the list of receiver emails seperated by commas.
+
+Default configuration is to use Gmail SMTP server. You can change that by modifying `SMTP_SERVER` and `SMTP_SERVER_PORT` in `constants_fapi.py`
+
+This workflow is scheduled to run daily at 08:30 UTC (2:00 AM SL time) and will also automatically trigger after a release or a pre-release.
+
+To locally setup and run the test suite, follow the [fapi-oidc-conformance-test.yml](.github/workflows/fapi-oidc-conformance-test.yml) script and execute the steps.
+
+## Test Profiles
+
+Running fapi test profiles are in [test_runner_fapi.sh](oidc-fapi-conformance-tests/test_runner_fapi.sh) script. Currently Running profiles,
+* private_key_jwt
+* mtls
+
diff --git a/oidc-fapi-conformance-tests/configure_is_fapi.py b/oidc-fapi-conformance-tests/configure_is_fapi.py
index 53e77067ae3..fef6d88614f 100644
--- a/oidc-fapi-conformance-tests/configure_is_fapi.py
+++ b/oidc-fapi-conformance-tests/configure_is_fapi.py
@@ -14,10 +14,6 @@
from config import browser_configuration
from config.client_configs import client_configs
-# path to product is zip file
-path_to_is_zip = str(sys.argv[1])
-print("Path to zip: ", path_to_is_zip)
-
def decode_secret(secret):
decoded_string=base64.b64decode(secret+"=").decode("utf-8")
decoded_json = json.loads(decoded_string)
@@ -153,7 +149,10 @@ def addCertsToKeystore(rootCertPath, issuerCertPath, ISPath):
# unpack product-is zip file and run
-def unpack_and_run(zip_file_name):
+def unpack_and_run():
+ # path to product is zip file
+ zip_file_name = str(sys.argv[1])
+ print("Path to zip: ", zip_file_name)
try:
# extract IS zip
with ZipFile(zip_file_name, 'r') as zip_file:
@@ -261,7 +260,7 @@ def is_process_running(process_name):
warnings.filterwarnings("ignore")
if not is_process_running("wso2server"):
- unpack_and_run(path_to_is_zip)
+ unpack_and_run()
else:
print("\n>>> IS already running ...")
print ("==============================================\n")
diff --git a/oidc-fapi-conformance-tests/constants_fapi.py b/oidc-fapi-conformance-tests/constants_fapi.py
index 69b6e832ff0..988b81952e7 100644
--- a/oidc-fapi-conformance-tests/constants_fapi.py
+++ b/oidc-fapi-conformance-tests/constants_fapi.py
@@ -156,3 +156,8 @@
"jwks_uri": JWKS_2,
"require_pushed_authorization_requests" : "true",
}
+
+
+SMTP_SERVER = "smtp.gmail.com"
+
+SMTP_SERVER_PORT = 465