Add assertions for failing and flaky tests

e2e_tests_with_assertions/failing/potential_bugs/user_tries_to_use_abstract_values.yml

@@ -0,0 +1,32 @@
+test_cases:
+  - test_case: user tries to use abstract values
+    steps:
+      - user: send money
+        assertions:
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_recipient
+      # the llm is extracting "good friend" as the recipient
+      - user: to a good friend
+        assertions:
+          - slot_was_not_set:
+              - name: transfer_money_recipient
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_recipient
+      - user: okay, to Mary
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_recipient
+                value: Mary
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: "50"
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_amount_of_money
+                value: "50"
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_final_confirmation
+      - user: "yes"
+        assertions:
+          - bot_uttered:
+              utter_name: utter_transfer_complete

e2e_tests_with_assertions/failing/skip_question/user_tries_to_skip_a_question_and_then_cancels_the_flow_hard.yml

@@ -0,0 +1,23 @@
+test_cases:
+  - test_case: user tries to skip a question and then cancels the flow (hard)
+    steps:
+      - user: send money to John
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_recipient
+                value: John
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: i don't want to answer this
+        assertions:
+          # cancel flow instead of skip question is predicted
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: i don't want to continue
+        assertions:
+          - bot_uttered:
+              utter_name: utter_flow_cancelled_rasa
+          - bot_uttered:
+              utter_name: utter_can_do_something_else

e2e_tests_with_assertions/failing/skip_question/user_tries_to_skip_a_question_multiple_times_hard.yml

@@ -0,0 +1,34 @@
+test_cases:
+  - test_case: user tries to skip a question multiple times (hard)
+    steps:
+      - user: send money to John
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_recipient
+                value: John
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: i don't want to answer this for now
+        assertions:
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: i don't want to answer this
+      # cancel flow instead of skip question is predicted
+        assertions:
+        - bot_uttered:
+            utter_name: utter_skip_question_answer
+        - bot_uttered:
+            utter_name: utter_ask_transfer_money_amount_of_money
+      - user: okay 50
+        assertions:
+          - slot_was_set:
+              - name: utter_ask_transfer_money_amount_of_money
+                value: "50"
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_final_confirmation
+      - user: "yes"
+        assertions:
+          - bot_uttered:
+              utter_name: utter_transfer_complete

e2e_tests_with_assertions/failing/skip_question/user_tries_to_skip_a_question_once_hard.yml

@@ -0,0 +1,28 @@
+test_cases:
+  - test_case: user tries to skip a question once (hard)
+    steps:
+      - user: send money to John
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_recipient
+                value: John
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: i don't want to answer this
+        assertions:
+          # cancel flow instead of skip question is predicted
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: okay 50
+        assertions:
+          - slot_was_set:
+              - name: utter_ask_transfer_money_amount_of_money
+                value: "50"
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_final_confirmation
+      - user: "yes"
+        assertions:
+          - bot_uttered:
+              utter_name: utter_transfer_complete

e2e_tests_with_assertions/flaky/disambiguation/user_sends_short_noun_only_message.yml

@@ -0,0 +1,19 @@
+# instead of clarify a start flow command for "list_contact" is predicted
+
+fixtures:
+  - route_to_calm:
+      - route_session_to_calm: True
+
+test_cases:
+  - test_case: user sends short noun only message
+    fixtures:
+      - route_to_calm
+    steps:
+      - user: contact
+        assertions:
+          - bot_uttered:
+              utter_name: utter_clarification_options_rasa
+      - user: add
+        assertions:
+          - bot_uttered:
+              utter_name: utter_ask_add_contact_handle

e2e_tests_with_assertions/flaky/happy_path/user_sets_up_recurrent_payment.yml

@@ -0,0 +1,16 @@
+# recurrent_payment_type is not mapped to standing order
+
+test_cases:
+  - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, example 3
+    steps:
+      - user: I want to set up a new recurrent payment
+        assertions:
+          - bot_uttered:
+              utter_name: utter_ask_recurrent_payment_type
+      - user: stand order
+        assertions:
+          - slot_was_set:
+              - name: recurrent_payment_type
+                value: standing order
+          - bot_uttered:
+              utter_name: utter_ask_recipient

e2e_tests_with_assertions/flaky/happy_path/user_wants_to_register_to_vote.yml

@@ -0,0 +1,15 @@
+fixtures:
+  - non_california_resident:
+      - based_in_california: False
+      - route_session_to_calm: True
+
+test_cases:
+  - test_case: Register to vote for non-California resident (should not trigger)
+    fixtures:
+      - non_california_resident
+    steps:
+      - user: I want to register to vote
+        # ChitChat is predicted instead of no command being predicted
+        assertions:
+          - bot_uttered:
+              utter_name: utter_cannot_answer

e2e_tests_with_assertions/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml

@@ -0,0 +1,53 @@
+test_cases:
+  - test_case: user wants to set up a new one off payment and starts a different flow
+    steps:
+      - user: I want to set up a new recurrent payment
+        assertions:
+          - bot_uttered:
+              utter_name: utter_ask_recurrent_payment_type
+      - user: international transfer
+      # leads to Clarify('direct debit', 'standing order') which is dropped, so CannotHandle command is predicted and
+      # results in utter_ask_rephrase
+        assertions:
+          - bot_uttered:
+              utter_name: utter_categorical_slot_rejection
+          - bot_uttered:
+              utter_name: utter_ask_recurrent_payment_type
+      - user: none of these, actually just need a one off payment
+        assertions:
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_recipient
+
+  - test_case: user wants to set up an invalid payment type and the assistant re-asks in a loop until a valid value is given
+    steps:
+      - user: I want to set up a new recurrent payment
+        assertions:
+          - bot_uttered:
+              utter_name: utter_ask_recurrent_payment_type
+      - user: international transfer
+      # leads to Clarify('direct debit', 'standing order') which is dropped, so CannotHandle command is predicted and
+      # results in utter_ask_rephrase
+        assertions:
+          - bot_uttered:
+              utter_name: utter_categorical_slot_rejection
+          - bot_uttered:
+              utter_name: utter_ask_recurrent_payment_type
+      - user: international transfer
+        assertions:
+          - bot_uttered:
+              utter_name: utter_categorical_slot_rejection
+          - bot_uttered:
+              utter_name: utter_ask_recurrent_payment_type
+      - user: international transfer
+        assertions:
+          - bot_uttered:
+              utter_name: utter_categorical_slot_rejection
+          - bot_uttered:
+              utter_name: utter_ask_recurrent_payment_type
+      - user: direct debit
+        assertions:
+          - slot_was_set:
+              - name: recurrent_payment_type
+                value: "direct debit"
+          - bot_uttered:
+              utter_name: utter_ask_recipient

e2e_tests_with_assertions/flaky/negations/users_says_they_dont_want_one_option.yml

@@ -0,0 +1,17 @@
+fixtures:
+  - route_to_calm:
+      - route_session_to_calm: True
+
+test_cases:
+  - test_case: user says they don't want one option
+    fixtures:
+      - route_to_calm
+    steps:
+      - user: cash
+        assertions:
+          - bot_uttered:
+              utter_name: utter_clarification_options_rasa
+      - user: not transfer
+        assertions:
+          - bot_uttered:
+              utter_name: utter_current_balance

e2e_tests_with_assertions/flaky/negations/users_says_they_dont_want_one_option_among_three.yml

@@ -0,0 +1,21 @@
+fixtures:
+  - route_to_calm:
+      - route_session_to_calm: True
+
+test_cases:
+  - test_case: user says they don't want one option among three
+    fixtures:
+      - route_to_calm
+    steps:
+      - user: contact
+        assertions:
+          - bot_uttered:
+              utter_name: utter_clarification_options_rasa
+      - user: not remove
+        assertions:
+          - bot_uttered:
+              utter_name: utter_clarification_options_rasa
+      - user: the latter
+        assertions:
+          - bot_uttered:
+              utter_name: utter_list_contacts

e2e_tests_with_assertions/flaky/negations/users_says_they_dont_want_two_options.yml

@@ -0,0 +1,17 @@
+fixtures:
+  - route_to_calm:
+      - route_session_to_calm: True
+
+test_cases:
+  - test_case: user says they don't want two options
+    fixtures:
+      - route_to_calm
+    steps:
+      - user: contact
+        assertions:
+          - bot_uttered:
+              utter_name: utter_clarification_options_rasa
+      - user: not add or remove
+        assertions:
+          - bot_uttered:
+              utter_name: utter_list_contacts

e2e_tests_with_assertions/flaky/potential_bugs/user_asks_to_start_10_transfers.yml

@@ -0,0 +1,31 @@
+# The first utterance ("10 transfers") already sets the amount of money to 10
+
+test_cases:
+  - test_case: user asks to start 10 transfers
+    steps:
+      - user: Please start ten transfers to Jim
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_recipient
+                value: Jim
+          - slot_was_not_set:
+              - name: utter_ask_transfer_money_amount_of_money
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: "10"
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_amount_of_money
+                value: "10"
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_final_confirmation
+      - user: "yes"
+        assertions:
+          - bot_uttered:
+              utter_name: utter_transfer_complete
+          - bot_uttered:
+              utter_name: utter_can_do_something_else
+      - user: show my transactions please
+        assertions:
+          - bot_uttered:
+              utter_name: utter_transactions

e2e_tests_with_assertions/flaky/skip_questions/user_tries_to_skip_a_question_multiple_times_easy.yml

@@ -0,0 +1,72 @@
+# cancel flow is predicted instead of skip question
+
+test_cases:
+  - test_case: user tries to skip a question multiple times (easy)
+    steps:
+      - user: send money to John
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_recipient
+                value: John
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: skip this question
+        assertions:
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: skip this step
+        assertions:
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: go to next question
+        assertions:
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: go to next step
+        assertions:
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: skip this
+        assertions:
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: next one
+        assertions:
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: skip
+        assertions:
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: next
+        assertions:
+          - bot_uttered:
+              utter_name: utter_skip_question_answer
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_amount_of_money
+      - user: okay 50
+        assertions:
+          - slot_was_set:
+              - name: transfer_money_amount_of_money
+                value: "50"
+          - bot_uttered:
+              utter_name: utter_ask_transfer_money_final_confirmation
+      - user: "yes"
+        assertions:
+          - bot_uttered:
+              utter_name: utter_transfer_complete
+