added flaky test category, ci step, and PR template

RasaHQ · Mar 5, 2024 · 1244eff · 1244eff
1 parent dd4b16c
commit 1244eff
Show file tree

Hide file tree

Showing 8 changed files with 54 additions and 37 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -0,0 +1,4 @@
+## Description
+
+## TODOs
+[ ] checked compared flaky tests with the [known list of flaky tests steps](https://www.notion.so/rasa/Flaky-E2E-Test-Steps-63864d3d8c7b4427a0f3df8052e39f21)
diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml
@@ -166,6 +166,16 @@ jobs:
         make actions &
         make test-passing
 
+    - name: Run e2e flaky tests
+      env:
+        OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
+        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+        RASA_PRO_BETA_INTENTLESS: true
+      run: |
+        make actions &
+        make test-flaky || 1
+
     - name: Run e2e failing tests
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}

diff --git a/Makefile b/Makefile
@@ -28,6 +28,9 @@ actions:
 test-passing: .EXPORT_ALL_VARIABLES
 	poetry run rasa test e2e e2e_tests/passing
 
+test-flaky: .EXPORT_ALL_VARIABLES
+	poetry run rasa test e2e e2e_tests/flaky
+
 test-failing: .EXPORT_ALL_VARIABLES
 	poetry run rasa test e2e e2e_tests/failing
 

diff --git a/...sions/user_asks_for_a_moment_to_think.yml → ...sions/user_asks_for_a_moment_to_think.yml b/...sions/user_asks_for_a_moment_to_think.yml → ...sions/user_asks_for_a_moment_to_think.yml
diff --git a/e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml b/e2e_tests/flaky/happy_path/user_sets_up_recurrent_payment.yml
@@ -0,0 +1,9 @@
+test_cases:
+  - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, example 3
+    steps:
+      - user: I want to set up a new recurrent payment
+      - utter: utter_ask_recurrent_payment_type
+      - user: stand order
+      - slot_was_set:
+          - recurrent_payment_type: "standing order"
+      - utter: utter_ask_recipient
diff --git a/e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml b/e2e_tests/flaky/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml
@@ -0,0 +1,28 @@
+test_cases:
+  - test_case: user wants to set up a new one off payment and starts a different flow
+    steps:
+      - user: I want to set up a new recurrent payment
+      - utter: utter_ask_recurrent_payment_type
+      - user: international transfer
+      - utter: utter_categorical_slot_rejection
+      - utter: utter_ask_recurrent_payment_type
+      - user: none of these, actually just need a one off payment
+      - utter: utter_ask_transfer_money_recipient
+
+  - test_case: user wants to set up an invalid payment type and the assistant re-asks in a loop until a valid value is given
+    steps:
+      - user: I want to set up a new recurrent payment
+      - utter: utter_ask_recurrent_payment_type
+      - user: international transfer
+      - utter: utter_categorical_slot_rejection
+      - utter: utter_ask_recurrent_payment_type
+      - user: international transfer
+      - utter: utter_categorical_slot_rejection
+      - utter: utter_ask_recurrent_payment_type
+      - user: international transfer
+      - utter: utter_categorical_slot_rejection
+      - utter: utter_ask_recurrent_payment_type
+      - user: direct debit
+      - slot_was_set:
+          - recurrent_payment_type: "direct debit"
+      - utter: utter_ask_recipient
diff --git a/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml b/e2e_tests/passing/happy_path/user_sets_up_recurrent_payment.yml
@@ -57,15 +57,6 @@ test_cases:
           - recurrent_payment_type: "direct debit"
       - utter: utter_ask_recipient
 
-  - test_case: user wants to set up a new recurrent payment, but specifies the type incompletely, example 3
-    steps:
-      - user: I want to set up a new recurrent payment
-      - utter: utter_ask_recurrent_payment_type
-      - user: stand order
-      - slot_was_set:
-          - recurrent_payment_type: "standing order"
-      - utter: utter_ask_recipient
-
   - test_case: user wants to set up a new recurrent payment, but specifies over specifies the type
     steps:
       - user: I want to set up a new recurrent payment

diff --git a/...tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml b/...tests/passing/invalid_path/user_tries_to_set_up_recurrent_payment_with_invalid_values.yml
@@ -1,32 +1,4 @@
 test_cases:
-  - test_case: user wants to set up a new one off payment and starts a different flow
-    steps:
-      - user: I want to set up a new recurrent payment
-      - utter: utter_ask_recurrent_payment_type
-      - user: international transfer
-      - utter: utter_categorical_slot_rejection
-      - utter: utter_ask_recurrent_payment_type
-      - user: none of these, actually just need a one off payment
-      - utter: utter_ask_transfer_money_recipient
-
-  - test_case: user wants to set up an invalid payment type and the assistant re-asks in a loop until a valid value is given
-    steps:
-      - user: I want to set up a new recurrent payment
-      - utter: utter_ask_recurrent_payment_type
-      - user: international transfer
-      - utter: utter_categorical_slot_rejection
-      - utter: utter_ask_recurrent_payment_type
-      - user: international transfer
-      - utter: utter_categorical_slot_rejection
-      - utter: utter_ask_recurrent_payment_type
-      - user: international transfer
-      - utter: utter_categorical_slot_rejection
-      - utter: utter_ask_recurrent_payment_type
-      - user: direct debit
-      - slot_was_set:
-          - recurrent_payment_type: "direct debit"
-      - utter: utter_ask_recipient
-
   - test_case: user wants to set up a new recurrent payment and inserts an invalid frequency value
     steps:
       - user: I want to set up a new standing order payment.