RasaHQ · twerkmeister · Mar 6, 2024 · Feb 1, 2024 · Feb 1, 2024 · Feb 4, 2024
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -0,0 +1,4 @@
+## Description
+
+## TODOs
+[ ] compared flaky tests with the [known list of flaky tests steps](https://www.notion.so/rasa/Flaky-E2E-Test-Steps-63864d3d8c7b4427a0f3df8052e39f21)
diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml
@@ -69,7 +69,7 @@ jobs:
           OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
           RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
           RASA_PRO_BETA_INTENTLESS: true
-          DUCKLING_URL: ${{secrets.DUCKLING_URL}}
+          RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         run: |
           make train
 
@@ -160,16 +160,28 @@ jobs:
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
         RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_INTENTLESS: true
       run: |
         make actions &
         make test-passing
 
+    - name: Run e2e flaky tests
+      env:
+        OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
+        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+        RASA_PRO_BETA_INTENTLESS: true
+      run: |
+        make actions &
+        make test-flaky || true
+
     - name: Run e2e failing tests
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
         RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_INTENTLESS: true
       run: |
         make actions &
-        make test-failing | grep '0 passed'
+        make test-failing | grep '0 passed'
diff --git a/Makefile b/Makefile
@@ -28,6 +28,9 @@ actions:
 test-passing: .EXPORT_ALL_VARIABLES
 	poetry run rasa test e2e e2e_tests/passing
 
+test-flaky: .EXPORT_ALL_VARIABLES
+	poetry run rasa test e2e e2e_tests/flaky
+
 test-failing: .EXPORT_ALL_VARIABLES
 	poetry run rasa test e2e e2e_tests/failing
 

diff --git a/config.yml b/config.yml
@@ -7,6 +7,8 @@ pipeline:
   llm:
     model_name: gpt-4
     request_timeout: 7
+    temperature: 0.0
+    top_p: 0.0
 
 policies:
 - name: rasa.core.policies.flow_policy.FlowPolicy

diff --git a/data/flows/add_card.yml b/data/flows/add_card.yml
@@ -0,0 +1,6 @@
+flows:
+  add_card:
+    description: add a card to your account
+    name: add a card
+    steps:
+      - action: utter_card_added
diff --git a/data/flows/patterns.yml b/data/flows/patterns.yml
@@ -25,11 +25,13 @@ flows:
     steps:
       - action: action_trigger_chitchat
 
+  # using chitchat here so that intentless is used for better testability
   pattern_search:
     description: handle knowledge-based requests using enterprise search
     steps:
-      - action: action_trigger_search
-
+      - action: action_trigger_chitchat
+      #- action: action_trigger_search
+
   pattern_cancel_flow:
     description: A meta flow that's started when a flow is cancelled.
     steps:

diff --git a/domain/add_card.yml b/domain/add_card.yml
@@ -0,0 +1,5 @@
+version: "3.1"
+
+responses:
+  utter_card_added:
+    - text: "Okay, added another card."
diff --git a/domain/patterns.yml b/domain/patterns.yml
@@ -14,12 +14,11 @@ responses:
           title: Yes
         - payload: no
           title: No, please keep the previous information
-      metadata: 
+      metadata:
         rephrase: True
         template: jinja
-  
+
   utter_not_corrected_previous_input:
     - text: "Ok, I did not correct the previous input."
-      metadata: 
+      metadata:
         rephrase: True
-
diff --git a/...sions/user_asks_for_a_moment_to_think.yml → ...sions/user_asks_for_a_moment_to_think.yml b/...sions/user_asks_for_a_moment_to_think.yml → ...sions/user_asks_for_a_moment_to_think.yml
@@ -8,5 +8,5 @@ test_cases:
       - utter: utter_ask_transfer_money_amount_of_money
       - user: "100"
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: Yes
+      - user: "Yes"
       - utter: utter_transfer_complete
diff --git a/e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml b/e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml
@@ -0,0 +1,9 @@
+test_cases:
+  - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, example 3
+    steps:
+      - user: I want to set up a new recurrent payment
+      - utter: utter_ask_recurrent_payment_type
+      - user: stand order
+      - slot_was_set:
+          - recurrent_payment_type: "standing order"
+      - utter: utter_ask_recipient
diff --git a/e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml b/e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml
@@ -0,0 +1,28 @@
+test_cases:
+  - test_case: user wants to set up a new one off payment and starts a different flow
+    steps:
+      - user: I want to set up a new recurrent payment
+      - utter: utter_ask_recurrent_payment_type
+      - user: international transfer
+      - utter: utter_categorical_slot_rejection
+      - utter: utter_ask_recurrent_payment_type
+      - user: none of these, actually just need a one off payment
+      - utter: utter_ask_transfer_money_recipient
+
+  - test_case: user wants to set up an invalid payment type and the assistant re-asks in a loop until a valid value is given
+    steps:
+      - user: I want to set up a new recurrent payment
+      - utter: utter_ask_recurrent_payment_type
+      - user: international transfer
+      - utter: utter_categorical_slot_rejection
+      - utter: utter_ask_recurrent_payment_type
+      - user: international transfer
+      - utter: utter_categorical_slot_rejection
+      - utter: utter_ask_recurrent_payment_type
+      - user: international transfer
+      - utter: utter_categorical_slot_rejection
+      - utter: utter_ask_recurrent_payment_type
+      - user: direct debit
+      - slot_was_set:
+          - recurrent_payment_type: "direct debit"
+      - utter: utter_ask_recipient
diff --git a/.../users_says_they_dont_want_one_option.yml → .../users_says_they_dont_want_one_option.yml b/.../users_says_they_dont_want_one_option.yml → .../users_says_they_dont_want_one_option.yml
diff --git a/...o_skip_a_question_multiple_times_easy.yml → ...o_skip_a_question_multiple_times_easy.yml b/...o_skip_a_question_multiple_times_easy.yml → ...o_skip_a_question_multiple_times_easy.yml
@@ -38,5 +38,5 @@ test_cases:
 
       - user: okay 50
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_transfer_complete
diff --git a/e2e_tests/passing/cancellations/user_cancels_default_pattern_override_no_confirmation.yml b/e2e_tests/passing/cancellations/user_cancels_default_pattern_override_no_confirmation.yml
@@ -7,4 +7,3 @@ test_cases:
       - utter: utter_ask_book_restaurant_number_of_people
       - user: Actually I cancel that please.
       - utter: utter_can_do_something_else
-
diff --git a/e2e_tests/passing/chitchat/user_says_goodbye.yml b/e2e_tests/passing/chitchat/user_says_goodbye.yml
@@ -3,7 +3,7 @@ test_cases:
     steps:
       - user: I want to send 10 bucks to Spencer
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: Yes
+      - user: "Yes"
       - utter: utter_transfer_complete
       - utter: utter_can_do_something_else
       - user: Bye then

diff --git a/e2e_tests/passing/corrections/user_cancels_correction.yml b/e2e_tests/passing/corrections/user_cancels_correction.yml
@@ -4,13 +4,12 @@ test_cases:
       - user: send John 50
       - slot_was_set:
         - transfer_money_amount_of_money: "50"
-      - slot_was_set:
         - transfer_money_recipient: "John"
       - utter: utter_ask_transfer_money_final_confirmation
       - user: "Ah wait I think I actually owe him 60"
       - utter: utter_ask_confirm_slot_correction
-      - user: No
+      - user: "No"
       - utter: utter_not_corrected_previous_input
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: Yes
+      - user: "Yes"
       - bot: Successfully transferred 50 to John.
diff --git a/e2e_tests/passing/corrections/user_corrects_a_boolean_slot.yml b/e2e_tests/passing/corrections/user_corrects_a_boolean_slot.yml
@@ -5,11 +5,11 @@ test_cases:
       - utter: utter_ask_verify_account_email
       - user: It's [email protected]
       - utter: utter_ask_based_in_california
-      - user: yes
+      - user: "yes"
       - utter: utter_ask_verify_account_sufficient_california_income
-      - user: yes
+      - user: "yes"
       - utter: utter_ask_verify_account_confirmation
       - user: Actually my income is not that high
       - utter: utter_ask_verify_account_sufficient_california_income
-      - user: no
+      - user: "no"
       - utter: utter_ca_income_insufficient
diff --git a/e2e_tests/passing/corrections/user_corrects_a_branching_slot.yml b/e2e_tests/passing/corrections/user_corrects_a_branching_slot.yml
@@ -5,11 +5,14 @@ test_cases:
       - utter: utter_ask_verify_account_email
       - user: It's [email protected]
       - utter: utter_ask_based_in_california
-      - user: yes
+      - user: "yes"
       - utter: utter_ask_verify_account_sufficient_california_income
       - user: Oh, actually I am not based in California
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
+      - slot_was_set:
+          - based_in_california
+      - utter: utter_corrected_previous_input
       - utter: utter_ask_verify_account_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_verify_account_success
diff --git a/e2e_tests/passing/corrections/user_corrects_a_number_slot.yml b/e2e_tests/passing/corrections/user_corrects_a_number_slot.yml
@@ -7,13 +7,13 @@ test_cases:
       - utter: utter_ask_book_restaurant_date
       - user: Ah wait, make that 5 people, Barbara is also coming
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
       - utter: utter_corrected_previous_input
       - utter: utter_ask_book_restaurant_date
       - user: Is 8 pm tomorrow night still possible?
       - utter: utter_restaurant_available
       - utter: utter_ask_book_restaurant_reservation_name
       - user: for Jamie
       - utter: utter_ask_book_restaurant_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_confirm_restaurant_booking
diff --git a/e2e_tests/passing/corrections/user_corrects_amount_of_money_in_the_next_message.yml b/e2e_tests/passing/corrections/user_corrects_amount_of_money_in_the_next_message.yml
@@ -13,10 +13,10 @@ test_cases:
       - utter: utter_ask_transfer_money_final_confirmation
       - user: "Ah, scratch that, I meant 110$"
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
       - slot_was_set:
           - transfer_money_amount_of_money: "110"
       - utter: utter_corrected_previous_input
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_transfer_complete
diff --git a/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_first.yml b/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_first.yml
@@ -13,10 +13,10 @@ test_cases:
       - utter: utter_ask_transfer_money_final_confirmation
       - user: "Ah, not 50, I meant 55"
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
       - slot_was_set:
           - transfer_money_amount_of_money: "55"
       - utter: utter_corrected_previous_input
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_transfer_complete
diff --git a/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_last.yml b/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_last.yml
@@ -9,12 +9,12 @@ test_cases:
       - utter: utter_ask_transfer_money_amount_of_money
       - user: Sorry, I meant to say Jimmy, not John
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
       - slot_was_set:
           - transfer_money_recipient: "Jimmy"
       - utter: utter_corrected_previous_input
       - utter: utter_ask_transfer_money_amount_of_money
       - user: "50$"
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_transfer_complete
diff --git a/e2e_tests/passing/corrections/user_corrects_recipient_in_the_next_message.yml b/e2e_tests/passing/corrections/user_corrects_recipient_in_the_next_message.yml
@@ -9,7 +9,7 @@ test_cases:
       - utter: utter_ask_transfer_money_amount_of_money
       - user: Sorry, I meant to say Joe
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
       - slot_was_set:
           - transfer_money_recipient: "Joe"
       - utter: utter_corrected_previous_input
@@ -18,5 +18,5 @@ test_cases:
       - slot_was_set:
           - transfer_money_amount_of_money: "50"
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_transfer_complete
diff --git a/e2e_tests/passing/corrections/user_corrects_recipient_late.yml b/e2e_tests/passing/corrections/user_corrects_recipient_late.yml
@@ -13,10 +13,10 @@ test_cases:
       - utter: utter_ask_transfer_money_final_confirmation
       - user:  "Oh wait, I want to send it to James!"
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
       - slot_was_set:
           - transfer_money_recipient: "James"
       - utter: utter_corrected_previous_input
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_transfer_complete
diff --git a/e2e_tests/passing/corrections/user_corrects_string_slot.yml b/e2e_tests/passing/corrections/user_corrects_string_slot.yml
@@ -11,7 +11,7 @@ test_cases:
       - utter: utter_ask_add_contact_confirmation
       - user: "Ah, please use Spidey as the name"
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
       - utter: utter_corrected_previous_input
       - utter: utter_ask_add_contact_confirmation
       - user: yes

diff --git a/e2e_tests/passing/corrections/user_corrects_twice_in_row.yml b/e2e_tests/passing/corrections/user_corrects_twice_in_row.yml
@@ -7,7 +7,7 @@ test_cases:
       - utter: utter_ask_confirm_slot_correction
       - user: damn, should be Alex
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
       - slot_was_set:
           - transfer_money_recipient: "Alex"
       - utter: utter_corrected_previous_input

diff --git a/e2e_tests/passing/corrections/user_resets_a_slot_by_slot_name.yml b/e2e_tests/passing/corrections/user_resets_a_slot_by_slot_name.yml
@@ -7,10 +7,10 @@ test_cases:
       - utter: utter_ask_transfer_money_final_confirmation
       - user: I want to change the recipient
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
       - utter: utter_corrected_previous_input
       - utter: utter_ask_transfer_money_recipient
       - user: Eliza
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_transfer_complete
diff --git a/e2e_tests/passing/corrections/user_resets_a_slot_by_value.yml b/e2e_tests/passing/corrections/user_resets_a_slot_by_value.yml
@@ -5,12 +5,12 @@ test_cases:
       - utter: utter_ask_transfer_money_amount_of_money
       - user: Sorry, I didn't mean John
       - utter: utter_ask_confirm_slot_correction
-      - user: yes
+      - user: "yes"
       - utter: utter_corrected_previous_input
       - utter: utter_ask_transfer_money_recipient
       - user: to Joe
       - utter: utter_ask_transfer_money_amount_of_money
       - user: "50"
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_transfer_complete
diff --git a/e2e_tests/passing/digressions/intermittent_flow_switch.yml b/e2e_tests/passing/digressions/intermittent_flow_switch.yml
@@ -18,5 +18,5 @@ test_cases:
       - utter: utter_ask_book_restaurant_reservation_name
       - user: for Thomas
       - utter: utter_ask_book_restaurant_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_confirm_restaurant_booking
diff --git a/...user_answers_and_also_asks_a_question.yml → ...user_answers_and_also_asks_a_question.yml b/...user_answers_and_also_asks_a_question.yml → ...user_answers_and_also_asks_a_question.yml
diff --git a/...ng/digressions/user_asks_what_info_is.yml → ...ng/digressions/user_asks_what_info_is.yml b/...ng/digressions/user_asks_what_info_is.yml → ...ng/digressions/user_asks_what_info_is.yml
diff --git a/e2e_tests/passing/digressions/user_refuses_to_provide_information.yml b/e2e_tests/passing/digressions/user_refuses_to_provide_information.yml
@@ -5,7 +5,7 @@ test_cases:
       - utter: utter_ask_verify_account_email
       - user: It's [email protected]
       - utter: utter_ask_based_in_california
-      - user: Yes
+      - user: "Yes"
       - utter: utter_ask_verify_account_sufficient_california_income
       - user: Why is that your business?
       - utter: utter_california_compliance

diff --git a/e2e_tests/passing/digressions/validation_too_little_money.yml b/e2e_tests/passing/digressions/validation_too_little_money.yml
@@ -1,10 +1,10 @@
 test_cases:
-  - test_case: user makes an intermitten flow switch
+  - test_case: validation too little money
     steps:
       - user: I want to send 5000$ to Martin
       - utter: utter_transfer_money_insufficient_funds
       - utter: utter_ask_transfer_money_amount_of_money
       - user: Oh, I thought I had more, then do 3000$ for now
       - utter: utter_ask_transfer_money_final_confirmation
-      - user: yes
+      - user: "yes"
       - utter: utter_transfer_complete
Original file line number	Diff line number	Diff line change
Expand Up		@@ -7,4 +7,3 @@ test_cases:
		- utter: utter_ask_book_restaurant_number_of_people
		- user: Actually I cancel that please.
		- utter: utter_can_do_something_else