From 7345cc9d8234eee83e87d178a741e38e7e97073c Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Thu, 1 Feb 2024 15:51:45 +0100 Subject: [PATCH 01/21] fixed disambiguation and negation tests --- data/flows/add_card.yml | 6 ++++++ data/flows/patterns.yml | 2 +- domain/add_card.yml | 5 +++++ domain/patterns.yml | 7 +++---- .../users_says_they_dont_want_one_option_among_three.yml | 2 +- ...yml => users_says_they_dont_want_the_former_option.yml} | 4 ++-- 6 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 data/flows/add_card.yml create mode 100644 domain/add_card.yml rename e2e_tests/passing/negations/{users_says_they_dont_want_the_latter_option.yml => users_says_they_dont_want_the_former_option.yml} (60%) diff --git a/data/flows/add_card.yml b/data/flows/add_card.yml new file mode 100644 index 0000000..7aa36e4 --- /dev/null +++ b/data/flows/add_card.yml @@ -0,0 +1,6 @@ +flows: + add_card: + description: add a card to your account + name: add a card + steps: + - action: utter_card_added diff --git a/data/flows/patterns.yml b/data/flows/patterns.yml index 3f2631a..9304971 100644 --- a/data/flows/patterns.yml +++ b/data/flows/patterns.yml @@ -29,7 +29,7 @@ flows: description: handle knowledge-based requests using enterprise search steps: - action: action_trigger_search - + pattern_cancel_flow: description: A meta flow that's started when a flow is cancelled. steps: diff --git a/domain/add_card.yml b/domain/add_card.yml new file mode 100644 index 0000000..71579f1 --- /dev/null +++ b/domain/add_card.yml @@ -0,0 +1,5 @@ +version: "3.1" + +responses: + utter_add_card: + - text: "Okay, added another card." diff --git a/domain/patterns.yml b/domain/patterns.yml index 95b1aef..85804a7 100644 --- a/domain/patterns.yml +++ b/domain/patterns.yml @@ -14,12 +14,11 @@ responses: title: Yes - payload: no title: No, please keep the previous information - metadata: + metadata: rephrase: True template: jinja - + utter_not_corrected_previous_input: - text: "Ok, I did not correct the previous input." - metadata: + metadata: rephrase: True - diff --git a/e2e_tests/passing/negations/users_says_they_dont_want_one_option_among_three.yml b/e2e_tests/passing/negations/users_says_they_dont_want_one_option_among_three.yml index 8d9ae65..a2b201e 100644 --- a/e2e_tests/passing/negations/users_says_they_dont_want_one_option_among_three.yml +++ b/e2e_tests/passing/negations/users_says_they_dont_want_one_option_among_three.yml @@ -5,5 +5,5 @@ test_cases: - utter: utter_clarification_options_rasa - user: not remove - utter: utter_clarification_options_rasa - - user: the former + - user: the latter - utter: utter_list_contacts diff --git a/e2e_tests/passing/negations/users_says_they_dont_want_the_latter_option.yml b/e2e_tests/passing/negations/users_says_they_dont_want_the_former_option.yml similarity index 60% rename from e2e_tests/passing/negations/users_says_they_dont_want_the_latter_option.yml rename to e2e_tests/passing/negations/users_says_they_dont_want_the_former_option.yml index cb77174..db31e80 100644 --- a/e2e_tests/passing/negations/users_says_they_dont_want_the_latter_option.yml +++ b/e2e_tests/passing/negations/users_says_they_dont_want_the_former_option.yml @@ -1,7 +1,7 @@ test_cases: - - test_case: user says they don't want the latter option + - test_case: user says they don't want the former option steps: - user: add - utter: utter_clarification_options_rasa - - user: not the latter + - user: not the former - utter: utter_ask_add_contact_handle From 2b8d8f30dc441c7126e6473104864576fe9a8fbb Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Thu, 1 Feb 2024 15:55:02 +0100 Subject: [PATCH 02/21] fixed utterance name --- domain/add_card.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain/add_card.yml b/domain/add_card.yml index 71579f1..18e14d6 100644 --- a/domain/add_card.yml +++ b/domain/add_card.yml @@ -1,5 +1,5 @@ version: "3.1" responses: - utter_add_card: + utter_card_added: - text: "Okay, added another card." From e45f65b4de6333fc33446822ad813232f8218d9c Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Sun, 4 Feb 2024 16:17:41 +0100 Subject: [PATCH 03/21] test fixes --- .../user_asks_why_information_is_needed.yml | 0 .../user_refuses_to_provide_information.yml | 2 +- ...fault_pattern_override_no_confirmation.yml | 2 +- .../passing/chitchat/user_says_goodbye.yml | 2 +- .../corrections/user_cancels_correction.yml | 5 ++-- .../user_corrects_a_boolean_slot.yml | 6 ++--- .../user_corrects_a_branching_slot.yml | 9 +++++--- .../user_corrects_a_number_slot.yml | 4 ++-- ...ts_amount_of_money_in_the_next_message.yml | 4 ++-- ...er_corrects_mentioning_old_value_first.yml | 4 ++-- ...ser_corrects_mentioning_old_value_last.yml | 4 ++-- ...corrects_recipient_in_the_next_message.yml | 4 ++-- .../user_corrects_recipient_late.yml | 4 ++-- .../corrections/user_corrects_string_slot.yml | 2 +- .../user_corrects_twice_in_row.yml | 2 +- .../user_resets_a_slot_by_slot_name.yml | 4 ++-- .../user_resets_a_slot_by_value.yml | 4 ++-- .../digressions/intermittent_flow_switch.yml | 2 +- .../user_asks_for_a_moment_to_think.yml | 2 +- .../validation_too_little_money.yml | 4 ++-- ...ries_to_register_to_vote_in_california.yml | 6 ++--- .../happy_path/user_books_a_restaurant.yml | 4 +--- ..._a_restaurant_with_an_alternative_time.yml | 6 ++--- ..._restaurant_with_their_own_alternative.yml | 6 ++--- .../user_books_restaurant_with_fixed_date.yml | 3 --- .../happy_path/user_checks_balance.yml | 1 - .../happy_path/user_fills_resetting_slot.yml | 12 ++-------- .../happy_path/user_remove_contact.yml | 2 +- .../user_remove_contact_from_list.yml | 2 +- .../user_replace_card_with_linked_flow.yml | 4 ++-- .../user_sets_up_recurrent_payment.yml | 23 ++++++++++--------- .../happy_path/user_transfer_money.yml | 2 +- ...sfer_money_mentioning_one_slot_upfront.yml | 4 ++-- ..._recurrent_payment_with_invalid_values.yml | 2 +- .../users_says_they_dont_want_one_option.yml | 7 ------ .../users_says_they_dont_want_two_options.yml | 7 ++++++ ...r_asks_for_repition_of_previous_action.yml | 4 ++-- .../user_asks_to_start_10_transfers.yml | 2 +- ...to_skip_a_question_multiple_times_easy.yml | 2 +- ...ser_tries_to_skip_a_question_once_easy.yml | 2 +- ...r_tries_to_skip_a_question_once_medium.yml | 2 +- 41 files changed, 79 insertions(+), 94 deletions(-) rename e2e_tests/{passing => failing}/digressions/user_asks_why_information_is_needed.yml (100%) rename e2e_tests/{passing => failing}/digressions/user_refuses_to_provide_information.yml (96%) delete mode 100644 e2e_tests/passing/happy_path/user_books_restaurant_with_fixed_date.yml delete mode 100644 e2e_tests/passing/negations/users_says_they_dont_want_one_option.yml create mode 100644 e2e_tests/passing/negations/users_says_they_dont_want_two_options.yml diff --git a/e2e_tests/passing/digressions/user_asks_why_information_is_needed.yml b/e2e_tests/failing/digressions/user_asks_why_information_is_needed.yml similarity index 100% rename from e2e_tests/passing/digressions/user_asks_why_information_is_needed.yml rename to e2e_tests/failing/digressions/user_asks_why_information_is_needed.yml diff --git a/e2e_tests/passing/digressions/user_refuses_to_provide_information.yml b/e2e_tests/failing/digressions/user_refuses_to_provide_information.yml similarity index 96% rename from e2e_tests/passing/digressions/user_refuses_to_provide_information.yml rename to e2e_tests/failing/digressions/user_refuses_to_provide_information.yml index a95d330..987a0b7 100644 --- a/e2e_tests/passing/digressions/user_refuses_to_provide_information.yml +++ b/e2e_tests/failing/digressions/user_refuses_to_provide_information.yml @@ -5,7 +5,7 @@ test_cases: - utter: utter_ask_verify_account_email - user: It's thomas@x.com - utter: utter_ask_based_in_california - - user: Yes + - user: "Yes" - utter: utter_ask_verify_account_sufficient_california_income - user: Why is that your business? - utter: utter_california_compliance diff --git a/e2e_tests/passing/cancellations/user_cancels_default_pattern_override_no_confirmation.yml b/e2e_tests/passing/cancellations/user_cancels_default_pattern_override_no_confirmation.yml index cee7fed..c8571a1 100644 --- a/e2e_tests/passing/cancellations/user_cancels_default_pattern_override_no_confirmation.yml +++ b/e2e_tests/passing/cancellations/user_cancels_default_pattern_override_no_confirmation.yml @@ -6,5 +6,5 @@ test_cases: - user: Noma - utter: utter_ask_book_restaurant_number_of_people - user: Actually I cancel that please. + - utter: utter_flow_cancelled_rasa - utter: utter_can_do_something_else - \ No newline at end of file diff --git a/e2e_tests/passing/chitchat/user_says_goodbye.yml b/e2e_tests/passing/chitchat/user_says_goodbye.yml index dc6df1a..0df8ed6 100644 --- a/e2e_tests/passing/chitchat/user_says_goodbye.yml +++ b/e2e_tests/passing/chitchat/user_says_goodbye.yml @@ -3,7 +3,7 @@ test_cases: steps: - user: I want to send 10 bucks to Spencer - utter: utter_ask_transfer_money_final_confirmation - - user: Yes + - user: "Yes" - utter: utter_transfer_complete - utter: utter_can_do_something_else - user: Bye then diff --git a/e2e_tests/passing/corrections/user_cancels_correction.yml b/e2e_tests/passing/corrections/user_cancels_correction.yml index c640757..e5aeea4 100644 --- a/e2e_tests/passing/corrections/user_cancels_correction.yml +++ b/e2e_tests/passing/corrections/user_cancels_correction.yml @@ -4,13 +4,12 @@ test_cases: - user: send John 50 - slot_was_set: - transfer_money_amount_of_money: "50" - - slot_was_set: - transfer_money_recipient: "John" - utter: utter_ask_transfer_money_final_confirmation - user: "Ah wait I think I actually owe him 60" - utter: utter_ask_confirm_slot_correction - - user: No + - user: "No" - utter: utter_not_corrected_previous_input - utter: utter_ask_transfer_money_final_confirmation - - user: Yes + - user: "Yes" - bot: Successfully transferred 50 to John. diff --git a/e2e_tests/passing/corrections/user_corrects_a_boolean_slot.yml b/e2e_tests/passing/corrections/user_corrects_a_boolean_slot.yml index 6967bfb..0c551a6 100644 --- a/e2e_tests/passing/corrections/user_corrects_a_boolean_slot.yml +++ b/e2e_tests/passing/corrections/user_corrects_a_boolean_slot.yml @@ -5,11 +5,11 @@ test_cases: - utter: utter_ask_verify_account_email - user: It's mag@pie.com - utter: utter_ask_based_in_california - - user: yes + - user: "yes" - utter: utter_ask_verify_account_sufficient_california_income - - user: yes + - user: "yes" - utter: utter_ask_verify_account_confirmation - user: Actually my income is not that high - utter: utter_ask_verify_account_sufficient_california_income - - user: no + - user: "no" - utter: utter_ca_income_insufficient diff --git a/e2e_tests/passing/corrections/user_corrects_a_branching_slot.yml b/e2e_tests/passing/corrections/user_corrects_a_branching_slot.yml index 1db511c..dcc0ff7 100644 --- a/e2e_tests/passing/corrections/user_corrects_a_branching_slot.yml +++ b/e2e_tests/passing/corrections/user_corrects_a_branching_slot.yml @@ -5,11 +5,14 @@ test_cases: - utter: utter_ask_verify_account_email - user: It's birdie@example.com - utter: utter_ask_based_in_california - - user: yes + - user: "yes" - utter: utter_ask_verify_account_sufficient_california_income - user: Oh, actually I am not based in California - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" + - slot_was_set: + - based_in_california + - utter: utter_corrected_previous_input - utter: utter_ask_verify_account_confirmation - - user: yes + - user: "yes" - utter: utter_verify_account_success diff --git a/e2e_tests/passing/corrections/user_corrects_a_number_slot.yml b/e2e_tests/passing/corrections/user_corrects_a_number_slot.yml index 682bb90..2268ec7 100644 --- a/e2e_tests/passing/corrections/user_corrects_a_number_slot.yml +++ b/e2e_tests/passing/corrections/user_corrects_a_number_slot.yml @@ -7,7 +7,7 @@ test_cases: - utter: utter_ask_book_restaurant_date - user: Ah wait, make that 5 people, Barbara is also coming - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - utter: utter_corrected_previous_input - utter: utter_ask_book_restaurant_date - user: Is 8 pm tomorrow night still possible? @@ -15,5 +15,5 @@ test_cases: - utter: utter_ask_book_restaurant_reservation_name - user: for Jamie - utter: utter_ask_book_restaurant_confirmation - - user: yes + - user: "yes" - utter: utter_confirm_restaurant_booking diff --git a/e2e_tests/passing/corrections/user_corrects_amount_of_money_in_the_next_message.yml b/e2e_tests/passing/corrections/user_corrects_amount_of_money_in_the_next_message.yml index da5a7b8..de6a09b 100644 --- a/e2e_tests/passing/corrections/user_corrects_amount_of_money_in_the_next_message.yml +++ b/e2e_tests/passing/corrections/user_corrects_amount_of_money_in_the_next_message.yml @@ -13,10 +13,10 @@ test_cases: - utter: utter_ask_transfer_money_final_confirmation - user: "Ah, scratch that, I meant 110$" - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - slot_was_set: - transfer_money_amount_of_money: "110" - utter: utter_corrected_previous_input - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_first.yml b/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_first.yml index 11347f4..534d11f 100644 --- a/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_first.yml +++ b/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_first.yml @@ -13,10 +13,10 @@ test_cases: - utter: utter_ask_transfer_money_final_confirmation - user: "Ah, not 50, I meant 55" - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - slot_was_set: - transfer_money_amount_of_money: "55" - utter: utter_corrected_previous_input - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_last.yml b/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_last.yml index 6626e8a..0c76bbc 100644 --- a/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_last.yml +++ b/e2e_tests/passing/corrections/user_corrects_mentioning_old_value_last.yml @@ -9,12 +9,12 @@ test_cases: - utter: utter_ask_transfer_money_amount_of_money - user: Sorry, I meant to say Jimmy, not John - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - slot_was_set: - transfer_money_recipient: "Jimmy" - utter: utter_corrected_previous_input - utter: utter_ask_transfer_money_amount_of_money - user: "50$" - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/corrections/user_corrects_recipient_in_the_next_message.yml b/e2e_tests/passing/corrections/user_corrects_recipient_in_the_next_message.yml index 81967e2..4a16a46 100644 --- a/e2e_tests/passing/corrections/user_corrects_recipient_in_the_next_message.yml +++ b/e2e_tests/passing/corrections/user_corrects_recipient_in_the_next_message.yml @@ -9,7 +9,7 @@ test_cases: - utter: utter_ask_transfer_money_amount_of_money - user: Sorry, I meant to say Joe - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - slot_was_set: - transfer_money_recipient: "Joe" - utter: utter_corrected_previous_input @@ -18,5 +18,5 @@ test_cases: - slot_was_set: - transfer_money_amount_of_money: "50" - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/corrections/user_corrects_recipient_late.yml b/e2e_tests/passing/corrections/user_corrects_recipient_late.yml index d530e61..6c0469f 100644 --- a/e2e_tests/passing/corrections/user_corrects_recipient_late.yml +++ b/e2e_tests/passing/corrections/user_corrects_recipient_late.yml @@ -13,10 +13,10 @@ test_cases: - utter: utter_ask_transfer_money_final_confirmation - user: "Oh wait, I want to send it to James!" - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - slot_was_set: - transfer_money_recipient: "James" - utter: utter_corrected_previous_input - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/corrections/user_corrects_string_slot.yml b/e2e_tests/passing/corrections/user_corrects_string_slot.yml index 5eaa680..1f07d66 100644 --- a/e2e_tests/passing/corrections/user_corrects_string_slot.yml +++ b/e2e_tests/passing/corrections/user_corrects_string_slot.yml @@ -11,7 +11,7 @@ test_cases: - utter: utter_ask_add_contact_confirmation - user: "Ah, please use Spidey as the name" - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - utter: utter_corrected_previous_input - utter: utter_ask_add_contact_confirmation - user: yes diff --git a/e2e_tests/passing/corrections/user_corrects_twice_in_row.yml b/e2e_tests/passing/corrections/user_corrects_twice_in_row.yml index 856a363..e3df8ad 100644 --- a/e2e_tests/passing/corrections/user_corrects_twice_in_row.yml +++ b/e2e_tests/passing/corrections/user_corrects_twice_in_row.yml @@ -7,7 +7,7 @@ test_cases: - utter: utter_ask_confirm_slot_correction - user: damn, should be Alex - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - slot_was_set: - transfer_money_recipient: "Alex" - utter: utter_corrected_previous_input diff --git a/e2e_tests/passing/corrections/user_resets_a_slot_by_slot_name.yml b/e2e_tests/passing/corrections/user_resets_a_slot_by_slot_name.yml index 0e80b21..c02d845 100644 --- a/e2e_tests/passing/corrections/user_resets_a_slot_by_slot_name.yml +++ b/e2e_tests/passing/corrections/user_resets_a_slot_by_slot_name.yml @@ -7,10 +7,10 @@ test_cases: - utter: utter_ask_transfer_money_final_confirmation - user: I want to change the recipient - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - utter: utter_corrected_previous_input - utter: utter_ask_transfer_money_recipient - user: Eliza - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/corrections/user_resets_a_slot_by_value.yml b/e2e_tests/passing/corrections/user_resets_a_slot_by_value.yml index 148078b..76d70a3 100644 --- a/e2e_tests/passing/corrections/user_resets_a_slot_by_value.yml +++ b/e2e_tests/passing/corrections/user_resets_a_slot_by_value.yml @@ -5,12 +5,12 @@ test_cases: - utter: utter_ask_transfer_money_amount_of_money - user: Sorry, I didn't mean John - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - utter: utter_corrected_previous_input - utter: utter_ask_transfer_money_recipient - user: to Joe - utter: utter_ask_transfer_money_amount_of_money - user: "50" - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/digressions/intermittent_flow_switch.yml b/e2e_tests/passing/digressions/intermittent_flow_switch.yml index 2f2c8e8..4a97847 100644 --- a/e2e_tests/passing/digressions/intermittent_flow_switch.yml +++ b/e2e_tests/passing/digressions/intermittent_flow_switch.yml @@ -18,5 +18,5 @@ test_cases: - utter: utter_ask_book_restaurant_reservation_name - user: for Thomas - utter: utter_ask_book_restaurant_confirmation - - user: yes + - user: "yes" - utter: utter_confirm_restaurant_booking diff --git a/e2e_tests/passing/digressions/user_asks_for_a_moment_to_think.yml b/e2e_tests/passing/digressions/user_asks_for_a_moment_to_think.yml index c1a5b6b..5866288 100644 --- a/e2e_tests/passing/digressions/user_asks_for_a_moment_to_think.yml +++ b/e2e_tests/passing/digressions/user_asks_for_a_moment_to_think.yml @@ -8,5 +8,5 @@ test_cases: - utter: utter_ask_transfer_money_amount_of_money - user: "100" - utter: utter_ask_transfer_money_final_confirmation - - user: Yes + - user: "Yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/digressions/validation_too_little_money.yml b/e2e_tests/passing/digressions/validation_too_little_money.yml index 0294e8a..3f4e006 100644 --- a/e2e_tests/passing/digressions/validation_too_little_money.yml +++ b/e2e_tests/passing/digressions/validation_too_little_money.yml @@ -1,10 +1,10 @@ test_cases: - - test_case: user makes an intermitten flow switch + - test_case: validation too little money steps: - user: I want to send 5000$ to Martin - utter: utter_transfer_money_insufficient_funds - utter: utter_ask_transfer_money_amount_of_money - user: Oh, I thought I had more, then do 3000$ for now - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/flow_guards/user_tries_to_register_to_vote_in_california.yml b/e2e_tests/passing/flow_guards/user_tries_to_register_to_vote_in_california.yml index 5bb898c..3993b3a 100644 --- a/e2e_tests/passing/flow_guards/user_tries_to_register_to_vote_in_california.yml +++ b/e2e_tests/passing/flow_guards/user_tries_to_register_to_vote_in_california.yml @@ -6,11 +6,11 @@ test_cases: - utter: utter_ask_verify_account_email - user: sam@microsoft.com - utter: utter_ask_based_in_california - - user: yes + - user: "yes" - utter: utter_ask_verify_account_sufficient_california_income - - user: yes + - user: "yes" - utter: utter_ask_verify_account_confirmation - - user: yes + - user: "yes" - utter: utter_verify_account_success - utter: utter_can_do_something_else - user: I would like to register to vote in california. diff --git a/e2e_tests/passing/happy_path/user_books_a_restaurant.yml b/e2e_tests/passing/happy_path/user_books_a_restaurant.yml index d4069a0..a10b458 100644 --- a/e2e_tests/passing/happy_path/user_books_a_restaurant.yml +++ b/e2e_tests/passing/happy_path/user_books_a_restaurant.yml @@ -5,13 +5,11 @@ test_cases: - slot_was_set: - book_restaurant_name_of_restaurant: Xaigon - book_restaurant_number_of_people: "4" - - book_restaurant_date: tonight - - book_restaurant_time: 8pm - utter: utter_restaurant_available - utter: utter_ask_book_restaurant_reservation_name - user: Emil - slot_was_set: - book_restaurant_reservation_name: Emil - utter: utter_ask_book_restaurant_confirmation - - user: yes + - user: "yes" - utter: utter_confirm_restaurant_booking diff --git a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml index 88a5f41..fcae694 100644 --- a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml +++ b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml @@ -5,13 +5,11 @@ test_cases: - slot_was_set: - book_restaurant_name_of_restaurant: Xaigon - book_restaurant_number_of_people: "4" - - book_restaurant_date: tonight - - book_restaurant_time: 7pm - utter: utter_restaurant_not_available - utter: utter_ask_book_restaurant_alternative_dummy - user: sounds good! - utter: utter_ask_confirm_slot_correction - - user: yes + - user: "yes" - slot_was_set: - book_restaurant_time: 8pm - utter: utter_corrected_previous_input @@ -21,5 +19,5 @@ test_cases: - slot_was_set: - book_restaurant_reservation_name: Emil - utter: utter_ask_book_restaurant_confirmation - - user: yes + - user: "yes" - utter: utter_confirm_restaurant_booking diff --git a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml index 93f31a1..c224128 100644 --- a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml +++ b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml @@ -11,9 +11,7 @@ test_cases: - utter: utter_ask_book_restaurant_alternative_dummy - user: 8pm is not good, how about a bit later then, like 9pm? - utter: utter_ask_confirm_slot_correction - - user: yes - - slot_was_set: - - book_restaurant_time: 9pm + - user: "yes" - utter: utter_corrected_previous_input - utter: utter_restaurant_available - utter: utter_ask_book_restaurant_reservation_name @@ -21,5 +19,5 @@ test_cases: - slot_was_set: - book_restaurant_reservation_name: Jakub - utter: utter_ask_book_restaurant_confirmation - - user: yes + - user: "yes" - utter: utter_confirm_restaurant_booking diff --git a/e2e_tests/passing/happy_path/user_books_restaurant_with_fixed_date.yml b/e2e_tests/passing/happy_path/user_books_restaurant_with_fixed_date.yml deleted file mode 100644 index 25f6b26..0000000 --- a/e2e_tests/passing/happy_path/user_books_restaurant_with_fixed_date.yml +++ /dev/null @@ -1,3 +0,0 @@ -test_cases: - - test_case: 8ad3f00b4f4d47e2b4db286ad9aa2b72 - steps: [] diff --git a/e2e_tests/passing/happy_path/user_checks_balance.yml b/e2e_tests/passing/happy_path/user_checks_balance.yml index 8bf45da..a1e30a6 100644 --- a/e2e_tests/passing/happy_path/user_checks_balance.yml +++ b/e2e_tests/passing/happy_path/user_checks_balance.yml @@ -4,4 +4,3 @@ test_cases: - user: Show my balance - utter: utter_current_balance - utter: utter_can_do_something_else - - user: no diff --git a/e2e_tests/passing/happy_path/user_fills_resetting_slot.yml b/e2e_tests/passing/happy_path/user_fills_resetting_slot.yml index ad1204e..39e6dba 100644 --- a/e2e_tests/passing/happy_path/user_fills_resetting_slot.yml +++ b/e2e_tests/passing/happy_path/user_fills_resetting_slot.yml @@ -7,9 +7,9 @@ test_cases: - slot_was_set: - verify_account_email: gos@hawk.com - utter: utter_ask_based_in_california - - user: no + - user: "no" - utter: utter_ask_verify_account_confirmation - - user: yes + - user: "yes" - utter: utter_verify_account_success # Check that the slot was reset at the end of the flow - slot_was_set: @@ -20,11 +20,3 @@ test_cases: - utter: utter_can_do_something_else - user: Bye then - utter: utter_goodbye - - user: I would like to confirm my account - - utter: utter_ask_verify_account_email - - user: gos@hawk.com - - slot_was_set: - - verify_account_email: gos@hawk.com - - utter: utter_ask_verify_account_confirmation - - user: yes - - utter: utter_verify_account_success diff --git a/e2e_tests/passing/happy_path/user_remove_contact.yml b/e2e_tests/passing/happy_path/user_remove_contact.yml index 67c669a..3df89e0 100644 --- a/e2e_tests/passing/happy_path/user_remove_contact.yml +++ b/e2e_tests/passing/happy_path/user_remove_contact.yml @@ -7,5 +7,5 @@ test_cases: - slot_was_set: - remove_contact_handle: "@MaryLu" - utter: utter_ask_remove_contact_confirmation - - user: Yes + - user: "Yes" - utter: utter_remove_contact_success diff --git a/e2e_tests/passing/happy_path/user_remove_contact_from_list.yml b/e2e_tests/passing/happy_path/user_remove_contact_from_list.yml index 6056ef3..ec14df7 100644 --- a/e2e_tests/passing/happy_path/user_remove_contact_from_list.yml +++ b/e2e_tests/passing/happy_path/user_remove_contact_from_list.yml @@ -8,5 +8,5 @@ test_cases: - slot_was_set: - remove_contact_handle: "@MaryLu" - utter: utter_ask_remove_contact_confirmation - - user: yes + - user: "yes" - utter: utter_remove_contact_success diff --git a/e2e_tests/passing/happy_path/user_replace_card_with_linked_flow.yml b/e2e_tests/passing/happy_path/user_replace_card_with_linked_flow.yml index 3ba55c7..d6ea26b 100644 --- a/e2e_tests/passing/happy_path/user_replace_card_with_linked_flow.yml +++ b/e2e_tests/passing/happy_path/user_replace_card_with_linked_flow.yml @@ -3,10 +3,10 @@ test_cases: steps: - user: I need to replace my card - utter: utter_ask_confirm_correct_card - - user: yes + - user: "yes" - utter: utter_ask_replacement_reason - user: lost - utter: utter_ask_was_card_used_fraudulently - - user: no + - user: "no" - utter: utter_will_cancel_and_send_new - utter: utter_new_card_has_been_ordered diff --git a/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml b/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml index 65475b4..a9876cd 100644 --- a/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml +++ b/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml @@ -27,7 +27,7 @@ test_cases: - slot_was_set: - recurrent_payment_end_date - utter: utter_ask_recurrent_payment_confirmation - - user: Yes + - user: "Yes" - slot_was_set: - recurrent_payment_confirmation: true - utter: utter_payment_complete @@ -48,7 +48,7 @@ test_cases: - recurrent_payment_type: "direct debit" - utter: utter_ask_recipient - - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, eg2 + - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, example 2 steps: - user: I want to set up a new recurrent payment - utter: utter_ask_recurrent_payment_type @@ -57,7 +57,7 @@ test_cases: - recurrent_payment_type: "direct debit" - utter: utter_ask_recipient - - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, eg3 + - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, example 3 steps: - user: I want to set up a new recurrent payment - utter: utter_ask_recurrent_payment_type @@ -66,14 +66,15 @@ test_cases: - recurrent_payment_type: "standing order" - utter: utter_ask_recipient - - test_case: user wants to set up a new recurrent payment, but specifies over specifies the type - steps: - - user: I want to set up a new recurrent payment - - utter: utter_ask_recurrent_payment_type - - user: direct debit transfer payment - - slot_was_set: - - recurrent_payment_type: "direct debit" - - utter: utter_ask_recipient +# Skipping for now: not clear what the intended behavior would be, it might be better to ask instead of just fill with one +# - test_case: user wants to set up a new recurrent payment, but specifies over specifies the type +# steps: +# - user: I want to set up a new recurrent payment +# - utter: utter_ask_recurrent_payment_type +# - user: direct debit transfer payment +# - slot_was_set: +# - recurrent_payment_type: "direct debit" +# - utter: utter_ask_recipient - test_case: user wants to set up a new recurrent payment, but specifies the type with spelling mistakes steps: diff --git a/e2e_tests/passing/happy_path/user_transfer_money.yml b/e2e_tests/passing/happy_path/user_transfer_money.yml index deb13ff..14758a2 100644 --- a/e2e_tests/passing/happy_path/user_transfer_money.yml +++ b/e2e_tests/passing/happy_path/user_transfer_money.yml @@ -11,5 +11,5 @@ test_cases: - slot_was_set: - transfer_money_amount_of_money: "150" - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/happy_path/user_transfer_money_mentioning_one_slot_upfront.yml b/e2e_tests/passing/happy_path/user_transfer_money_mentioning_one_slot_upfront.yml index 4eba265..5f2168c 100644 --- a/e2e_tests/passing/happy_path/user_transfer_money_mentioning_one_slot_upfront.yml +++ b/e2e_tests/passing/happy_path/user_transfer_money_mentioning_one_slot_upfront.yml @@ -9,7 +9,7 @@ test_cases: - slot_was_set: - transfer_money_amount_of_money: "130" - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete - test_case: user transfers money mentioning money slot upfront steps: @@ -21,5 +21,5 @@ test_cases: - slot_was_set: - transfer_money_recipient: John - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml b/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml index 4e703f1..977c3a1 100644 --- a/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml +++ b/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml @@ -6,7 +6,7 @@ test_cases: - user: international transfer - utter: utter_categorical_slot_rejection - utter: utter_ask_recurrent_payment_type - - user: none of these, just need a one off payment + - user: none of these, actually just need a one off payment - utter: utter_ask_transfer_money_recipient - test_case: user wants to set up an invalid payment type and the assistant re-asks in a loop until a valid value is given diff --git a/e2e_tests/passing/negations/users_says_they_dont_want_one_option.yml b/e2e_tests/passing/negations/users_says_they_dont_want_one_option.yml deleted file mode 100644 index 939fb31..0000000 --- a/e2e_tests/passing/negations/users_says_they_dont_want_one_option.yml +++ /dev/null @@ -1,7 +0,0 @@ -test_cases: - - test_case: user says they don't want one option - steps: - - user: cash - - utter: utter_clarification_options_rasa - - user: not transfer - - utter: utter_current_balance diff --git a/e2e_tests/passing/negations/users_says_they_dont_want_two_options.yml b/e2e_tests/passing/negations/users_says_they_dont_want_two_options.yml new file mode 100644 index 0000000..0509f1d --- /dev/null +++ b/e2e_tests/passing/negations/users_says_they_dont_want_two_options.yml @@ -0,0 +1,7 @@ +test_cases: + - test_case: user says they don't want two options + steps: + - user: contact + - utter: utter_clarification_options_rasa + - user: not add or remove + - utter: utter_list_contacts diff --git a/e2e_tests/passing/potential_bugs/user_asks_for_repition_of_previous_action.yml b/e2e_tests/passing/potential_bugs/user_asks_for_repition_of_previous_action.yml index 15964aa..1f242e0 100644 --- a/e2e_tests/passing/potential_bugs/user_asks_for_repition_of_previous_action.yml +++ b/e2e_tests/passing/potential_bugs/user_asks_for_repition_of_previous_action.yml @@ -4,10 +4,10 @@ test_cases: steps: - user: send 50 USD to John - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete - utter: utter_can_do_something_else - user: now do it again - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/potential_bugs/user_asks_to_start_10_transfers.yml b/e2e_tests/passing/potential_bugs/user_asks_to_start_10_transfers.yml index dc4a6be..dd82961 100644 --- a/e2e_tests/passing/potential_bugs/user_asks_to_start_10_transfers.yml +++ b/e2e_tests/passing/potential_bugs/user_asks_to_start_10_transfers.yml @@ -5,7 +5,7 @@ test_cases: - utter: utter_ask_transfer_money_amount_of_money - user: "10" - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete - utter: utter_can_do_something_else - user: show my transactions please diff --git a/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_multiple_times_easy.yml b/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_multiple_times_easy.yml index 4dadd77..16417c0 100644 --- a/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_multiple_times_easy.yml +++ b/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_multiple_times_easy.yml @@ -38,5 +38,5 @@ test_cases: - user: okay 50 - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_once_easy.yml b/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_once_easy.yml index e929889..15322b5 100644 --- a/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_once_easy.yml +++ b/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_once_easy.yml @@ -8,5 +8,5 @@ test_cases: - utter: utter_ask_transfer_money_amount_of_money - user: okay 50 - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete diff --git a/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_once_medium.yml b/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_once_medium.yml index fd7fa31..1d992a3 100644 --- a/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_once_medium.yml +++ b/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_once_medium.yml @@ -8,5 +8,5 @@ test_cases: - utter: utter_ask_transfer_money_amount_of_money - user: okay 50 - utter: utter_ask_transfer_money_final_confirmation - - user: yes + - user: "yes" - utter: utter_transfer_complete From 33d8da2e8e7cc09380ae656b87f4c1f5644f5f98 Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Sun, 4 Feb 2024 17:48:42 +0100 Subject: [PATCH 04/21] removed cancel confirmation for non transfer money flow --- .../user_cancels_default_pattern_override_no_confirmation.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e_tests/passing/cancellations/user_cancels_default_pattern_override_no_confirmation.yml b/e2e_tests/passing/cancellations/user_cancels_default_pattern_override_no_confirmation.yml index c8571a1..7e21798 100644 --- a/e2e_tests/passing/cancellations/user_cancels_default_pattern_override_no_confirmation.yml +++ b/e2e_tests/passing/cancellations/user_cancels_default_pattern_override_no_confirmation.yml @@ -6,5 +6,4 @@ test_cases: - user: Noma - utter: utter_ask_book_restaurant_number_of_people - user: Actually I cancel that please. - - utter: utter_flow_cancelled_rasa - utter: utter_can_do_something_else From 17d09a0b1b6332a22e2f89e3296932eee379ab16 Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Sun, 4 Feb 2024 18:34:26 +0100 Subject: [PATCH 05/21] removed second time string from test --- .../user_books_a_restaurant_with_an_alternative_time.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml index fcae694..2a64c42 100644 --- a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml +++ b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml @@ -10,8 +10,6 @@ test_cases: - user: sounds good! - utter: utter_ask_confirm_slot_correction - user: "yes" - - slot_was_set: - - book_restaurant_time: 8pm - utter: utter_corrected_previous_input - utter: utter_restaurant_available - utter: utter_ask_book_restaurant_reservation_name From 1b912785aea99d6194324831568f88bedc096f32 Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Mon, 5 Feb 2024 16:58:04 +0100 Subject: [PATCH 06/21] reinstated misunderstood test --- .../user_sets_up_recurrent_payment.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml b/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml index a9876cd..3d28897 100644 --- a/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml +++ b/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml @@ -66,15 +66,14 @@ test_cases: - recurrent_payment_type: "standing order" - utter: utter_ask_recipient -# Skipping for now: not clear what the intended behavior would be, it might be better to ask instead of just fill with one -# - test_case: user wants to set up a new recurrent payment, but specifies over specifies the type -# steps: -# - user: I want to set up a new recurrent payment -# - utter: utter_ask_recurrent_payment_type -# - user: direct debit transfer payment -# - slot_was_set: -# - recurrent_payment_type: "direct debit" -# - utter: utter_ask_recipient + - test_case: user wants to set up a new recurrent payment, but specifies over specifies the type + steps: + - user: I want to set up a new recurrent payment + - utter: utter_ask_recurrent_payment_type + - user: direct debit transfer payment + - slot_was_set: + - recurrent_payment_type: "direct debit" + - utter: utter_ask_recipient - test_case: user wants to set up a new recurrent payment, but specifies the type with spelling mistakes steps: From 78c8891e98420c17f63872950809e63fce91efdf Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Mon, 12 Feb 2024 12:34:06 +0100 Subject: [PATCH 07/21] moved back two tests to passing where they originally where --- .../digressions/user_asks_why_information_is_needed.yml | 0 .../digressions/user_refuses_to_provide_information.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename e2e_tests/{failing => passing}/digressions/user_asks_why_information_is_needed.yml (100%) rename e2e_tests/{failing => passing}/digressions/user_refuses_to_provide_information.yml (100%) diff --git a/e2e_tests/failing/digressions/user_asks_why_information_is_needed.yml b/e2e_tests/passing/digressions/user_asks_why_information_is_needed.yml similarity index 100% rename from e2e_tests/failing/digressions/user_asks_why_information_is_needed.yml rename to e2e_tests/passing/digressions/user_asks_why_information_is_needed.yml diff --git a/e2e_tests/failing/digressions/user_refuses_to_provide_information.yml b/e2e_tests/passing/digressions/user_refuses_to_provide_information.yml similarity index 100% rename from e2e_tests/failing/digressions/user_refuses_to_provide_information.yml rename to e2e_tests/passing/digressions/user_refuses_to_provide_information.yml From e3ff5bac19b74838bad9b9958d1f15edf08bc89b Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Wed, 14 Feb 2024 13:27:28 +0100 Subject: [PATCH 08/21] using intentless instead of enterprise search for testability --- data/flows/patterns.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data/flows/patterns.yml b/data/flows/patterns.yml index 9304971..11cab80 100644 --- a/data/flows/patterns.yml +++ b/data/flows/patterns.yml @@ -25,10 +25,12 @@ flows: steps: - action: action_trigger_chitchat + # using chitchat here so that intentless is used for better testability pattern_search: description: handle knowledge-based requests using enterprise search steps: - - action: action_trigger_search + - action: action_trigger_chitchat + #- action: action_trigger_search pattern_cancel_flow: description: A meta flow that's started when a flow is cancelled. From 5eba7a4310ed88809a59b1d25eff2abd10c0ed97 Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Wed, 14 Feb 2024 13:28:37 +0100 Subject: [PATCH 09/21] reinstated removed test --- .../negations/users_says_they_dont_want_one_option.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 e2e_tests/passing/negations/users_says_they_dont_want_one_option.yml diff --git a/e2e_tests/passing/negations/users_says_they_dont_want_one_option.yml b/e2e_tests/passing/negations/users_says_they_dont_want_one_option.yml new file mode 100644 index 0000000..939fb31 --- /dev/null +++ b/e2e_tests/passing/negations/users_says_they_dont_want_one_option.yml @@ -0,0 +1,7 @@ +test_cases: + - test_case: user says they don't want one option + steps: + - user: cash + - utter: utter_clarification_options_rasa + - user: not transfer + - utter: utter_current_balance From ca3319adb27aa8d3d5c405f9765c0fc116337b00 Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Fri, 16 Feb 2024 12:33:29 +0100 Subject: [PATCH 10/21] readded slot sets for restaurant time and date but without specified values --- e2e_tests/passing/happy_path/user_books_a_restaurant.yml | 2 ++ .../user_books_a_restaurant_with_an_alternative_time.yml | 4 ++++ .../user_books_a_restaurant_with_their_own_alternative.yml | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/e2e_tests/passing/happy_path/user_books_a_restaurant.yml b/e2e_tests/passing/happy_path/user_books_a_restaurant.yml index a10b458..2f67936 100644 --- a/e2e_tests/passing/happy_path/user_books_a_restaurant.yml +++ b/e2e_tests/passing/happy_path/user_books_a_restaurant.yml @@ -5,6 +5,8 @@ test_cases: - slot_was_set: - book_restaurant_name_of_restaurant: Xaigon - book_restaurant_number_of_people: "4" + - book_restaurant_date + - book_restaurant_time - utter: utter_restaurant_available - utter: utter_ask_book_restaurant_reservation_name - user: Emil diff --git a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml index 2a64c42..f3b029c 100644 --- a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml +++ b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_an_alternative_time.yml @@ -5,11 +5,15 @@ test_cases: - slot_was_set: - book_restaurant_name_of_restaurant: Xaigon - book_restaurant_number_of_people: "4" + - book_restaurant_date + - book_restaurant_time - utter: utter_restaurant_not_available - utter: utter_ask_book_restaurant_alternative_dummy - user: sounds good! - utter: utter_ask_confirm_slot_correction - user: "yes" + - slot_was_set: + - book_restaurant_time - utter: utter_corrected_previous_input - utter: utter_restaurant_available - utter: utter_ask_book_restaurant_reservation_name diff --git a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml index c224128..0b09b27 100644 --- a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml +++ b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml @@ -5,6 +5,8 @@ test_cases: - slot_was_set: - book_restaurant_name_of_restaurant: Shiso Burger - book_restaurant_number_of_people: "3" + - book_restaurant_date + - book_restaurant_time - utter: utter_ask_book_restaurant_date - user: I was thinking tomorrow 7pm - utter: utter_restaurant_not_available @@ -12,6 +14,8 @@ test_cases: - user: 8pm is not good, how about a bit later then, like 9pm? - utter: utter_ask_confirm_slot_correction - user: "yes" + - slot_was_set: + - book_restaurant_time - utter: utter_corrected_previous_input - utter: utter_restaurant_available - utter: utter_ask_book_restaurant_reservation_name From 5a880f576eed00d2127fcf478302d8ca1848d9c9 Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Fri, 16 Feb 2024 12:35:49 +0100 Subject: [PATCH 11/21] added checks for setting to none --- e2e_tests/passing/happy_path/user_fills_resetting_slot.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e_tests/passing/happy_path/user_fills_resetting_slot.yml b/e2e_tests/passing/happy_path/user_fills_resetting_slot.yml index 39e6dba..f656f86 100644 --- a/e2e_tests/passing/happy_path/user_fills_resetting_slot.yml +++ b/e2e_tests/passing/happy_path/user_fills_resetting_slot.yml @@ -13,7 +13,8 @@ test_cases: - utter: utter_verify_account_success # Check that the slot was reset at the end of the flow - slot_was_set: - - verify_account_email + - verify_account_email: null + - verify_account_confirmation: null # Check that the slot was not reset at the end of the flow - slot_was_not_set: - based_in_california From 537440180b98b4811bfddb11ef3c0715549cc328 Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Fri, 16 Feb 2024 12:38:34 +0100 Subject: [PATCH 12/21] moved knowledge tests to passing bc using intentless makes them possible --- .../knowledge/knowledge_question_during_flow.yml | 0 .../knowledge/user_follows_up_on_knowledge_question.yml | 0 .../knowledge/user_starts_with_a_knowledge_question.yml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename e2e_tests/{failing => passing}/knowledge/knowledge_question_during_flow.yml (100%) rename e2e_tests/{failing => passing}/knowledge/user_follows_up_on_knowledge_question.yml (100%) rename e2e_tests/{failing => passing}/knowledge/user_starts_with_a_knowledge_question.yml (100%) diff --git a/e2e_tests/failing/knowledge/knowledge_question_during_flow.yml b/e2e_tests/passing/knowledge/knowledge_question_during_flow.yml similarity index 100% rename from e2e_tests/failing/knowledge/knowledge_question_during_flow.yml rename to e2e_tests/passing/knowledge/knowledge_question_during_flow.yml diff --git a/e2e_tests/failing/knowledge/user_follows_up_on_knowledge_question.yml b/e2e_tests/passing/knowledge/user_follows_up_on_knowledge_question.yml similarity index 100% rename from e2e_tests/failing/knowledge/user_follows_up_on_knowledge_question.yml rename to e2e_tests/passing/knowledge/user_follows_up_on_knowledge_question.yml diff --git a/e2e_tests/failing/knowledge/user_starts_with_a_knowledge_question.yml b/e2e_tests/passing/knowledge/user_starts_with_a_knowledge_question.yml similarity index 100% rename from e2e_tests/failing/knowledge/user_starts_with_a_knowledge_question.yml rename to e2e_tests/passing/knowledge/user_starts_with_a_knowledge_question.yml From 7b004db37d92387cd0b001df86106cea46ec3f17 Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Fri, 16 Feb 2024 13:56:15 +0100 Subject: [PATCH 13/21] added duckling url secret to env --- .github/workflows/continous-integration.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index 2502fab..e1cc5fd 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -69,7 +69,7 @@ jobs: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}} RASA_PRO_BETA_INTENTLESS: true - DUCKLING_URL: ${{secrets.DUCKLING_URL}} + RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} run: | make train @@ -160,6 +160,7 @@ jobs: env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}} + RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} RASA_PRO_BETA_INTENTLESS: true run: | make actions & @@ -169,7 +170,8 @@ jobs: env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}} + RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} RASA_PRO_BETA_INTENTLESS: true run: | make actions & - make test-failing | grep '0 passed' \ No newline at end of file + make test-failing | grep '0 passed' From a94feba56f0243d6f8c582b5e9ce33e32630f6dd Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Mon, 19 Feb 2024 14:07:09 +0100 Subject: [PATCH 14/21] moved slot sets to the right user message --- .../user_books_a_restaurant_with_their_own_alternative.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml index 0b09b27..a62757f 100644 --- a/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml +++ b/e2e_tests/passing/happy_path/user_books_a_restaurant_with_their_own_alternative.yml @@ -5,10 +5,11 @@ test_cases: - slot_was_set: - book_restaurant_name_of_restaurant: Shiso Burger - book_restaurant_number_of_people: "3" - - book_restaurant_date - - book_restaurant_time - utter: utter_ask_book_restaurant_date - user: I was thinking tomorrow 7pm + - slot_was_set: + - book_restaurant_date + - book_restaurant_time - utter: utter_restaurant_not_available - utter: utter_ask_book_restaurant_alternative_dummy - user: 8pm is not good, how about a bit later then, like 9pm? From 76eb79a36cb68721d3ae1e993a24900cfc794f6e Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Mon, 19 Feb 2024 14:12:43 +0100 Subject: [PATCH 15/21] setting temp and top p explicitly --- config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.yml b/config.yml index 3155244..c85dc4a 100644 --- a/config.yml +++ b/config.yml @@ -7,6 +7,8 @@ pipeline: llm: model_name: gpt-4 request_timeout: 7 + temperature: 0.0 + top_p: 0.0 policies: - name: rasa.core.policies.flow_policy.FlowPolicy From 1244effcf05779cff242f3424a72e0cb4135c30d Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Tue, 5 Mar 2024 12:23:04 +0100 Subject: [PATCH 16/21] added flaky test category, ci step, and PR template --- .github/pull_request_template.md | 4 +++ .github/workflows/continous-integration.yml | 10 +++++++ Makefile | 3 ++ .../user_asks_for_a_moment_to_think.yml | 0 .../user_sets_up_recurrent_payment.yml | 9 ++++++ ..._recurrent_payment_with_invalid_values.yml | 28 +++++++++++++++++++ .../user_sets_up_recurrent_payment.yml | 9 ------ ..._recurrent_payment_with_invalid_values.yml | 28 ------------------- 8 files changed, 54 insertions(+), 37 deletions(-) create mode 100644 .github/pull_request_template.md rename e2e_tests/{passing => flaky}/digressions/user_asks_for_a_moment_to_think.yml (100%) create mode 100644 e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml create mode 100644 e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..a4c3368 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,4 @@ +## Description + +## TODOs +[ ] checked compared flaky tests with the [known list of flaky tests steps](https://www.notion.so/rasa/Flaky-E2E-Test-Steps-63864d3d8c7b4427a0f3df8052e39f21) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index e1cc5fd..661d000 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -166,6 +166,16 @@ jobs: make actions & make test-passing + - name: Run e2e flaky tests + env: + OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} + RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}} + RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} + RASA_PRO_BETA_INTENTLESS: true + run: | + make actions & + make test-flaky || 1 + - name: Run e2e failing tests env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} diff --git a/Makefile b/Makefile index dd081bd..aaec505 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,9 @@ actions: test-passing: .EXPORT_ALL_VARIABLES poetry run rasa test e2e e2e_tests/passing +test-flaky: .EXPORT_ALL_VARIABLES + poetry run rasa test e2e e2e_tests/flaky + test-failing: .EXPORT_ALL_VARIABLES poetry run rasa test e2e e2e_tests/failing diff --git a/e2e_tests/passing/digressions/user_asks_for_a_moment_to_think.yml b/e2e_tests/flaky/digressions/user_asks_for_a_moment_to_think.yml similarity index 100% rename from e2e_tests/passing/digressions/user_asks_for_a_moment_to_think.yml rename to e2e_tests/flaky/digressions/user_asks_for_a_moment_to_think.yml diff --git a/e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml b/e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml new file mode 100644 index 0000000..fa6c525 --- /dev/null +++ b/e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml @@ -0,0 +1,9 @@ +test_cases: + - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, example 3 + steps: + - user: I want to set up a new recurrent payment + - utter: utter_ask_recurrent_payment_type + - user: stand order + - slot_was_set: + - recurrent_payment_type: "standing order" + - utter: utter_ask_recipient diff --git a/e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml b/e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml new file mode 100644 index 0000000..253620a --- /dev/null +++ b/e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml @@ -0,0 +1,28 @@ +test_cases: + - test_case: user wants to set up a new one off payment and starts a different flow + steps: + - user: I want to set up a new recurrent payment + - utter: utter_ask_recurrent_payment_type + - user: international transfer + - utter: utter_categorical_slot_rejection + - utter: utter_ask_recurrent_payment_type + - user: none of these, actually just need a one off payment + - utter: utter_ask_transfer_money_recipient + + - test_case: user wants to set up an invalid payment type and the assistant re-asks in a loop until a valid value is given + steps: + - user: I want to set up a new recurrent payment + - utter: utter_ask_recurrent_payment_type + - user: international transfer + - utter: utter_categorical_slot_rejection + - utter: utter_ask_recurrent_payment_type + - user: international transfer + - utter: utter_categorical_slot_rejection + - utter: utter_ask_recurrent_payment_type + - user: international transfer + - utter: utter_categorical_slot_rejection + - utter: utter_ask_recurrent_payment_type + - user: direct debit + - slot_was_set: + - recurrent_payment_type: "direct debit" + - utter: utter_ask_recipient diff --git a/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml b/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml index 3d28897..56725a1 100644 --- a/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml +++ b/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml @@ -57,15 +57,6 @@ test_cases: - recurrent_payment_type: "direct debit" - utter: utter_ask_recipient - - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, example 3 - steps: - - user: I want to set up a new recurrent payment - - utter: utter_ask_recurrent_payment_type - - user: stand order - - slot_was_set: - - recurrent_payment_type: "standing order" - - utter: utter_ask_recipient - - test_case: user wants to set up a new recurrent payment, but specifies over specifies the type steps: - user: I want to set up a new recurrent payment diff --git a/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml b/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml index 977c3a1..e7b26b8 100644 --- a/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml +++ b/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml @@ -1,32 +1,4 @@ test_cases: - - test_case: user wants to set up a new one off payment and starts a different flow - steps: - - user: I want to set up a new recurrent payment - - utter: utter_ask_recurrent_payment_type - - user: international transfer - - utter: utter_categorical_slot_rejection - - utter: utter_ask_recurrent_payment_type - - user: none of these, actually just need a one off payment - - utter: utter_ask_transfer_money_recipient - - - test_case: user wants to set up an invalid payment type and the assistant re-asks in a loop until a valid value is given - steps: - - user: I want to set up a new recurrent payment - - utter: utter_ask_recurrent_payment_type - - user: international transfer - - utter: utter_categorical_slot_rejection - - utter: utter_ask_recurrent_payment_type - - user: international transfer - - utter: utter_categorical_slot_rejection - - utter: utter_ask_recurrent_payment_type - - user: international transfer - - utter: utter_categorical_slot_rejection - - utter: utter_ask_recurrent_payment_type - - user: direct debit - - slot_was_set: - - recurrent_payment_type: "direct debit" - - utter: utter_ask_recipient - - test_case: user wants to set up a new recurrent payment and inserts an invalid frequency value steps: - user: I want to set up a new standing order payment. From 4524b4c3b0b3ed7d46d42d1116806675a3b131cd Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Tue, 5 Mar 2024 13:20:35 +0100 Subject: [PATCH 17/21] moved another test to flaky --- .../negations/users_says_they_dont_want_one_option.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename e2e_tests/{passing => flaky}/negations/users_says_they_dont_want_one_option.yml (100%) diff --git a/e2e_tests/passing/negations/users_says_they_dont_want_one_option.yml b/e2e_tests/flaky/negations/users_says_they_dont_want_one_option.yml similarity index 100% rename from e2e_tests/passing/negations/users_says_they_dont_want_one_option.yml rename to e2e_tests/flaky/negations/users_says_they_dont_want_one_option.yml From bceb13378e03f989ec3d3c21ca9aaada60232ad1 Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Tue, 5 Mar 2024 13:58:57 +0100 Subject: [PATCH 18/21] passing step for flaky tests --- .github/workflows/continous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index 661d000..fc69eb6 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -174,7 +174,7 @@ jobs: RASA_PRO_BETA_INTENTLESS: true run: | make actions & - make test-flaky || 1 + make test-flaky || true - name: Run e2e failing tests env: From 3c8c38eefaaee4c854fb7ec46b7caeb49def5d0e Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Wed, 6 Mar 2024 10:39:05 +0100 Subject: [PATCH 19/21] moved tests that are actually passing out of failing category --- .../digressions/user_answers_and_also_asks_a_question.yml | 0 .../{failing => passing}/digressions/user_asks_what_info_is.yml | 0 .../user_tries_to_skip_a_question_multiple_times_medium.yml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename e2e_tests/{failing => passing}/digressions/user_answers_and_also_asks_a_question.yml (100%) rename e2e_tests/{failing => passing}/digressions/user_asks_what_info_is.yml (100%) rename e2e_tests/{failing => passing}/skip_question/user_tries_to_skip_a_question_multiple_times_medium.yml (100%) diff --git a/e2e_tests/failing/digressions/user_answers_and_also_asks_a_question.yml b/e2e_tests/passing/digressions/user_answers_and_also_asks_a_question.yml similarity index 100% rename from e2e_tests/failing/digressions/user_answers_and_also_asks_a_question.yml rename to e2e_tests/passing/digressions/user_answers_and_also_asks_a_question.yml diff --git a/e2e_tests/failing/digressions/user_asks_what_info_is.yml b/e2e_tests/passing/digressions/user_asks_what_info_is.yml similarity index 100% rename from e2e_tests/failing/digressions/user_asks_what_info_is.yml rename to e2e_tests/passing/digressions/user_asks_what_info_is.yml diff --git a/e2e_tests/failing/skip_question/user_tries_to_skip_a_question_multiple_times_medium.yml b/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_multiple_times_medium.yml similarity index 100% rename from e2e_tests/failing/skip_question/user_tries_to_skip_a_question_multiple_times_medium.yml rename to e2e_tests/passing/skip_question/user_tries_to_skip_a_question_multiple_times_medium.yml From a830b4531d56754be7438c5a069e6c8b360e846d Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Wed, 6 Mar 2024 11:05:04 +0100 Subject: [PATCH 20/21] moved test to flaky --- .../user_tries_to_skip_a_question_multiple_times_easy.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename e2e_tests/{passing/skip_question => flaky/skip_questions}/user_tries_to_skip_a_question_multiple_times_easy.yml (100%) diff --git a/e2e_tests/passing/skip_question/user_tries_to_skip_a_question_multiple_times_easy.yml b/e2e_tests/flaky/skip_questions/user_tries_to_skip_a_question_multiple_times_easy.yml similarity index 100% rename from e2e_tests/passing/skip_question/user_tries_to_skip_a_question_multiple_times_easy.yml rename to e2e_tests/flaky/skip_questions/user_tries_to_skip_a_question_multiple_times_easy.yml From d64266b48e7ad2a8784fc3abf5fe4fddbd4e9a3b Mon Sep 17 00:00:00 2001 From: Thomas Werkmeister Date: Wed, 6 Mar 2024 13:19:42 +0100 Subject: [PATCH 21/21] Update .github/pull_request_template.md Co-authored-by: Tanja --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a4c3368..3c240ef 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ ## Description ## TODOs -[ ] checked compared flaky tests with the [known list of flaky tests steps](https://www.notion.so/rasa/Flaky-E2E-Test-Steps-63864d3d8c7b4427a0f3df8052e39f21) +[ ] compared flaky tests with the [known list of flaky tests steps](https://www.notion.so/rasa/Flaky-E2E-Test-Steps-63864d3d8c7b4427a0f3df8052e39f21)