Add example stub custom actions

.github/workflows/continous-integration.yml

@@ -315,3 +315,83 @@ jobs:
       run: |
           make stop-duckling
 
+  run_e2e_tests_with_stub_custom_actions:
+    name: Run e2e tests with stub custom actions
+    runs-on: ubuntu-22.04
+    needs: [train-model]
+
+    steps:
+    - name: Checkout git repository 🕝
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+    - name: Setup Python
+      uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+      with:
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+
+    - name: Install poetry 🦄
+      uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844
+      with:
+        poetry-version: ${{ env.POETRY_VERSION }}
+
+    - name: Load Poetry Cached Libraries ⬇
+      id: cache-poetry
+      uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
+      with:
+        path: .venv
+        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}
+        restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
+
+    - name: Create virtual environment
+      if: steps.cache-poetry.outputs.cache-hit != 'true'
+      run: python -m venv create .venv
+
+    - name: Set up virtual environment
+      run: poetry config virtualenvs.in-project true
+
+      # Authenticate with gcloud for release registry (where Rasa is published)
+    - id: "auth-release"
+      name: Authenticate with gcloud for release registry 🎫
+      uses: "google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d"
+      with:
+        token_format: 'access_token'
+        credentials_json: "${{ secrets.RASA_RELEASES_READ }}"
+
+    - name: Configure OAuth token for poetry
+      run: |
+        poetry config http-basic.rasa-plus oauth2accesstoken $(gcloud auth print-access-token)
+
+    - name: Install Dependencies 📦
+      run: |
+        make install
+
+    - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+      with:
+        name: trained-model
+        path: models/
+
+    - name: Init LLM Cache
+      id: cache-llm
+      uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
+      with:
+        path: .rasa
+        key: rasa-llm-cache
+
+    - name: Run duckling server
+      run: |
+        make run-duckling
+
+    - name: Run e2e passing tests with stub custom actions
+      env:
+        OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
+        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+        RASA_PRO_BETA_E2E_ASSERTIONS: true
+        RASA_PRO_BETA_STUB_CUSTOM_ACTION: true
+      run: |
+        make test-passing-stub-custom-actions
+
+    - name: Stop Duckling server
+      run: |
+        make stop-duckling
+

Makefile

@@ -100,3 +100,6 @@ test-flaky-assertions: .EXPORT_ALL_VARIABLES
 
 test-failing-assertions: .EXPORT_ALL_VARIABLES
 	poetry run rasa test e2e e2e_tests_with_assertions/failing
+
+make test-passing-stub-custom-actions: .EXPORT_ALL_VARIABLES
+	poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing

e2e_tests_with_assertions/passing/chitchat/user_says_something_surprising.yml

@@ -0,0 +1,13 @@
+test_cases:
+  - test_case: user_says_something_intriguing
+    steps:
+      - user: I just had the most wonderful startup idea.
+        assertions:
+          - bot_uttered:
+              utter_name: utter_intriguing
+          - generative_response_is_relevant:
+              utter_name: utter_intriguing
+              threshold: 0.70
+          - generative_response_is_grounded:
+              utter_name: utter_intriguing
+              threshold: 0.90

e2e_tests_with_assertions/passing/happy_path/user_replace_card_with_linked_flow.yml

@@ -101,6 +101,7 @@ test_cases:
         - bot_uttered:
             utter_name: utter_ask_replacement_reason
     - user: some other reason
+      assertion_order_enabled: true
       assertions:
             - slot_was_set:
                 - name: replacement_reason

e2e_tests_with_stub_custom_actions/passing/happy_path/user_adds_contact_to_their_list.yml

@@ -0,0 +1,37 @@
+stub_custom_actions:
+  add_contact:
+    events:
+      - event: slot
+        name: return_value
+        value: success
+    responses: []
+
+test_cases:
+  - test_case: user_adds_contact_to_their_list
+    steps:
+      - user: I want to add someone to my contact list
+        assertions:
+          - bot_uttered:
+                utter_name: utter_ask_add_contact_handle
+      - user: it's @barts
+        assertions:
+          - slot_was_set:
+                - name: add_contact_handle
+                  value: "@barts"
+          - bot_uttered:
+                utter_name: utter_ask_add_contact_name
+      - user: just Bart
+        assertions:
+          - slot_was_set:
+                - name: add_contact_name
+                  value: Bart
+          - bot_uttered:
+                utter_name: utter_ask_add_contact_confirmation
+      - user: yes
+        assertions:
+          - slot_was_set:
+                - name: add_contact_confirmation
+                  value: true
+          - action_executed: add_contact
+          - bot_uttered:
+                utter_name: utter_contact_added

e2e_tests_with_stub_custom_actions/passing/happy_path/user_checks_balance.yml

@@ -0,0 +1,17 @@
+stub_custom_actions:
+  check_balance:
+    events:
+      - event: slot
+        name: current_balance
+        value: 1200
+    responses: []
+
+test_cases:
+  - test_case: user_checks_balance
+    steps:
+      - user: Show my balance
+        assertions:
+          - bot_uttered:
+              text_matches: You still have 1200 in your account.
+          - bot_uttered:
+              utter_name: utter_can_do_something_else

e2e_tests_with_stub_custom_actions/passing/happy_path/user_checks_balance_then_transactions.yml

@@ -0,0 +1,27 @@
+stub_custom_actions:
+    check_balance:
+        events:
+        - event: slot
+          name: current_balance
+          value: 1300
+        responses: []
+    transaction_search:
+        events:
+        - event: slot
+          name: transactions_list
+          value: '{"amount": 500, "date": "2024-01-01", "description": "rent"}'
+        responses: []
+
+test_cases:
+  - test_case: user_checks_balance_then_transactions
+    steps:
+      - user: Show my balance
+        assertions:
+            - bot_uttered:
+                text_matches: You still have 1300 in your account.
+            - bot_uttered:
+                utter_name: utter_can_do_something_else
+      - user: Show my transactions please
+        assertions:
+            - bot_uttered:
+                text_matches: 'Your current transactions are:\n {"amount": 500, "date": "2024-01-01", "description": "rent"}'

e2e_tests_with_stub_custom_actions/passing/happy_path/user_checks_portfolio.yml

@@ -0,0 +1,47 @@
+stub_custom_actions:
+  action_authenticate_user:
+    events:
+      - event: slot
+        name: is_user_logged_in
+        value: true
+    responses: []
+  action_check_portfolio_exists:
+    events:
+      - event: slot
+        name: portfolio_exists
+        value: true
+    responses: []
+  action_show_portfolio:
+    events:
+      - event: slot
+        name: portfolio_options
+        value: 'mutual_funds'
+    responses: []
+
+test_cases:
+  - test_case: user_checks_portfolio
+    steps:
+      - user: I want to check my portfolio
+        assertions:
+          - bot_uttered:
+              utter_name: utter_ask_user_name
+      - user: Max
+        assertions:
+          - bot_uttered:
+              utter_name: utter_ask_user_password
+      - user: "1234"
+        assertions:
+          - bot_uttered:
+              utter_name: utter_authentication_successful
+          - bot_uttered:
+              utter_name: utter_ask_portfolio_type
+      - user: /SetSlots(portfolio_type=mutual_funds)
+        assertions:
+          - slot_was_set:
+              - name: portfolio_type
+                value: mutual_funds
+              - name: portfolio_exists
+                value: True
+              - name: portfolio_options
+          - bot_uttered:
+              utter_name: utter_portfolio_options_found

e2e_tests_with_stub_custom_actions/passing/happy_path/user_checks_their_transactions.yml

@@ -0,0 +1,16 @@
+stub_custom_actions:
+    transaction_search:
+        events:
+        - event: slot
+          name: transactions_list
+          value: '{"amount": 100, "date": "2024-01-01", "description": "restaurant"}'
+        responses: []
+
+
+test_cases:
+  - test_case: user_checks_their_transactions
+    steps:
+      - user: Show my transactions please
+        assertions:
+            - bot_uttered:
+                text_matches: 'Your current transactions are:\n {"amount": 100, "date": "2024-01-01", "description": "restaurant"}'

e2e_tests_with_stub_custom_actions/passing/happy_path/user_transfer_money.yml

@@ -0,0 +1,71 @@
+stub_custom_actions:
+  user_transfer_money_success::execute_transfer:
+    events:
+      - event: slot
+        name: "transfer_money_transfer_successful"
+        value: true
+    responses: []
+  user_transfer_money_failed::execute_transfer:
+    events:
+      - event: slot
+        name: "transfer_money_transfer_successful"
+        value: false
+    responses: []
+  check_transfer_funds:
+    events:
+      - event: slot
+        name: "transfer_money_has_sufficient_funds"
+        value: true
+
+test_cases:
+  - test_case: user_transfer_money_success
+    steps:
+      - user: I want to transfer money
+        assertions:
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_recipient
+      - user: to John
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_recipient
+                value: John
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: 150$
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_amount_of_money
+                value: "150"
+          - bot_uttered:
+                utter_name: utter_ask_transfer_money_final_confirmation
+      - user: "yes"
+        assertions:
+          - action_executed: execute_transfer
+          - bot_uttered:
+              utter_name: utter_transfer_complete
+
+  - test_case: user_transfer_money_failed
+    steps:
+      - user: I want to transfer money
+        assertions:
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_recipient
+      - user: to John
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_recipient
+                value: John
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: 150$
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_amount_of_money
+                value: "150"
+          - bot_uttered:
+                utter_name: utter_ask_transfer_money_final_confirmation
+      - user: "yes"
+        assertions:
+          - action_executed: execute_transfer
+          - bot_uttered:
+              utter_name: utter_transfer_failed