diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..a4c3368 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,4 @@ +## Description + +## TODOs +[ ] checked compared flaky tests with the [known list of flaky tests steps](https://www.notion.so/rasa/Flaky-E2E-Test-Steps-63864d3d8c7b4427a0f3df8052e39f21) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index e1cc5fd..661d000 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -166,6 +166,16 @@ jobs: make actions & make test-passing + - name: Run e2e flaky tests + env: + OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} + RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}} + RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} + RASA_PRO_BETA_INTENTLESS: true + run: | + make actions & + make test-flaky || 1 + - name: Run e2e failing tests env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} diff --git a/Makefile b/Makefile index dd081bd..aaec505 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,9 @@ actions: test-passing: .EXPORT_ALL_VARIABLES poetry run rasa test e2e e2e_tests/passing +test-flaky: .EXPORT_ALL_VARIABLES + poetry run rasa test e2e e2e_tests/flaky + test-failing: .EXPORT_ALL_VARIABLES poetry run rasa test e2e e2e_tests/failing diff --git a/e2e_tests/passing/digressions/user_asks_for_a_moment_to_think.yml b/e2e_tests/flaky/digressions/user_asks_for_a_moment_to_think.yml similarity index 100% rename from e2e_tests/passing/digressions/user_asks_for_a_moment_to_think.yml rename to e2e_tests/flaky/digressions/user_asks_for_a_moment_to_think.yml diff --git a/e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml b/e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml new file mode 100644 index 0000000..fa6c525 --- /dev/null +++ b/e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml @@ -0,0 +1,9 @@ +test_cases: + - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, example 3 + steps: + - user: I want to set up a new recurrent payment + - utter: utter_ask_recurrent_payment_type + - user: stand order + - slot_was_set: + - recurrent_payment_type: "standing order" + - utter: utter_ask_recipient diff --git a/e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml b/e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml new file mode 100644 index 0000000..253620a --- /dev/null +++ b/e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml @@ -0,0 +1,28 @@ +test_cases: + - test_case: user wants to set up a new one off payment and starts a different flow + steps: + - user: I want to set up a new recurrent payment + - utter: utter_ask_recurrent_payment_type + - user: international transfer + - utter: utter_categorical_slot_rejection + - utter: utter_ask_recurrent_payment_type + - user: none of these, actually just need a one off payment + - utter: utter_ask_transfer_money_recipient + + - test_case: user wants to set up an invalid payment type and the assistant re-asks in a loop until a valid value is given + steps: + - user: I want to set up a new recurrent payment + - utter: utter_ask_recurrent_payment_type + - user: international transfer + - utter: utter_categorical_slot_rejection + - utter: utter_ask_recurrent_payment_type + - user: international transfer + - utter: utter_categorical_slot_rejection + - utter: utter_ask_recurrent_payment_type + - user: international transfer + - utter: utter_categorical_slot_rejection + - utter: utter_ask_recurrent_payment_type + - user: direct debit + - slot_was_set: + - recurrent_payment_type: "direct debit" + - utter: utter_ask_recipient diff --git a/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml b/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml index 3d28897..56725a1 100644 --- a/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml +++ b/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml @@ -57,15 +57,6 @@ test_cases: - recurrent_payment_type: "direct debit" - utter: utter_ask_recipient - - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, example 3 - steps: - - user: I want to set up a new recurrent payment - - utter: utter_ask_recurrent_payment_type - - user: stand order - - slot_was_set: - - recurrent_payment_type: "standing order" - - utter: utter_ask_recipient - - test_case: user wants to set up a new recurrent payment, but specifies over specifies the type steps: - user: I want to set up a new recurrent payment diff --git a/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml b/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml index 977c3a1..e7b26b8 100644 --- a/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml +++ b/e2e_tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml @@ -1,32 +1,4 @@ test_cases: - - test_case: user wants to set up a new one off payment and starts a different flow - steps: - - user: I want to set up a new recurrent payment - - utter: utter_ask_recurrent_payment_type - - user: international transfer - - utter: utter_categorical_slot_rejection - - utter: utter_ask_recurrent_payment_type - - user: none of these, actually just need a one off payment - - utter: utter_ask_transfer_money_recipient - - - test_case: user wants to set up an invalid payment type and the assistant re-asks in a loop until a valid value is given - steps: - - user: I want to set up a new recurrent payment - - utter: utter_ask_recurrent_payment_type - - user: international transfer - - utter: utter_categorical_slot_rejection - - utter: utter_ask_recurrent_payment_type - - user: international transfer - - utter: utter_categorical_slot_rejection - - utter: utter_ask_recurrent_payment_type - - user: international transfer - - utter: utter_categorical_slot_rejection - - utter: utter_ask_recurrent_payment_type - - user: direct debit - - slot_was_set: - - recurrent_payment_type: "direct debit" - - utter: utter_ask_recipient - - test_case: user wants to set up a new recurrent payment and inserts an invalid frequency value steps: - user: I want to set up a new standing order payment.