From 6132f04cbaafbdb2e860a88c4fba0bf16a2e6f51 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 5 Oct 2023 01:32:42 +0100 Subject: [PATCH] unittests/get_response_cascade (#366) Co-authored-by: Swen Gross --- .../session/skill-converse_test/__init__.py | 20 +- .../en-us/test_get_response_cascade.intent | 1 + test/end2end/session/test_get_response.py | 330 ++++++++++++++++++ 3 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 test/end2end/session/skill-converse_test/locale/en-us/test_get_response_cascade.intent diff --git a/test/end2end/session/skill-converse_test/__init__.py b/test/end2end/session/skill-converse_test/__init__.py index 959ca9da16b5..e0dde686429e 100644 --- a/test/end2end/session/skill-converse_test/__init__.py +++ b/test/end2end/session/skill-converse_test/__init__.py @@ -1,7 +1,8 @@ +from time import sleep + +from mycroft.skills import intent_file_handler from ovos_workshop.decorators import killable_intent from ovos_workshop.skills.ovos import OVOSSkill -from mycroft.skills import intent_file_handler -from time import sleep class TestAbortSkill(OVOSSkill): @@ -11,9 +12,11 @@ class TestAbortSkill(OVOSSkill): send "my.own.abort.msg" and confirm intent3 is aborted say "stop" and confirm all intents are aborted """ + def initialize(self): self.stop_called = False self._converse = False + self.items = [] self.bus.on("test_activate", self.do_activate) self.bus.on("test_deactivate", self.do_deactivate) @@ -46,6 +49,19 @@ def handle_test_get_response3(self, message): ans = self.get_response(num_retries=3) self.speak(ans or "ERROR") + @intent_file_handler("test_get_response_cascade.intent") + def handle_test_get_response_cascade(self, message): + quit = False + self.items = [] + self.speak("give me items", wait=True) + while not quit: + response = self.get_response(num_retries=0) + if response is None: + quit = True + else: + self.items.append(response) + self.bus.emit(message.forward("skill_items", {"items": self.items})) + @killable_intent(callback=handle_intent_aborted) @intent_file_handler("test.intent") def handle_test_abort_intent(self, message): diff --git a/test/end2end/session/skill-converse_test/locale/en-us/test_get_response_cascade.intent b/test/end2end/session/skill-converse_test/locale/en-us/test_get_response_cascade.intent new file mode 100644 index 000000000000..b8a7494fc509 --- /dev/null +++ b/test/end2end/session/skill-converse_test/locale/en-us/test_get_response_cascade.intent @@ -0,0 +1 @@ +test get items \ No newline at end of file diff --git a/test/end2end/session/test_get_response.py b/test/end2end/session/test_get_response.py index 9b641e79d651..b7b950c47cb1 100644 --- a/test/end2end/session/test_get_response.py +++ b/test/end2end/session/test_get_response.py @@ -720,3 +720,333 @@ def answer_get_response(msg): # test deserialization of payload sess = Session.deserialize(messages[26].data["session_data"]) self.assertEqual(sess.session_id, "default") + + def test_nested(self): + SessionManager.sessions = {} + SessionManager.default_session = SessionManager.sessions["default"] = Session("default") + SessionManager.default_session.lang = "en-us" + + messages = [] + + def new_msg(msg): + nonlocal messages + m = Message.deserialize(msg) + if m.msg_type in ["ovos.skills.settings_changed"]: + return # skip these, only happen in 1st run + messages.append(m) + print(len(messages), msg) + + def wait_for_n_messages(n): + nonlocal messages + t = time.time() + while len(messages) < n: + sleep(0.1) + if time.time() - t > 10: + raise RuntimeError("did not get the number of expected messages under 10 seconds") + + self.core.bus.on("message", new_msg) + + items = ["A", "B", "C"] + + def answer_get_response(msg): + nonlocal items + sleep(0.5) + if not len(items): + utt = Message("recognizer_loop:utterance", + {"utterances": ["cancel"]}, + {"session": SessionManager.default_session.serialize()}) + else: + utt = Message("recognizer_loop:utterance", + {"utterances": [items[0]]}, + {"session": SessionManager.default_session.serialize()}) + self.core.bus.emit(utt) + items = items[1:] + + self.core.bus.on("mycroft.mic.listen", answer_get_response) + + # trigger get_response + utt = Message("recognizer_loop:utterance", + {"utterances": ["test get items"]}) + self.core.bus.emit(utt) + + # confirm all expected messages are sent + expected_messages = [ + "recognizer_loop:utterance", # no session + "skill.converse.ping", # default session injected + "skill.converse.pong", # test skill + "skill.converse.pong", # hello world skill + + # skill selected + "intent.service.skills.activated", + f"{self.skill_id}.activate", + f"{self.skill_id}:test_get_response_cascade.intent", + + # intent code before self.get_response + "mycroft.skill.handler.start", + "enclosure.active_skill", + "speak", # "give me items" + + # first get_response + "skill.converse.get_response.enable", # start of get_response + "ovos.session.update_default", # sync get_response status + "mycroft.mic.listen", # no dialog in self.get_response + "recognizer_loop:utterance", # A + "skill.converse.ping", + "skill.converse.pong", + "skill.converse.pong", + "skill.converse.get_response", # A + "intent.service.skills.activated", + f"{self.skill_id}.activate", + "ovos.session.update_default", # sync skill trigger + "skill.converse.get_response.disable", + "ovos.session.update_default", # sync get_response status + + # second get_response + "skill.converse.get_response.enable", # start of get_response + "ovos.session.update_default", # sync get_response status + "mycroft.mic.listen", # no dialog in self.get_response + "recognizer_loop:utterance", # B + "skill.converse.ping", + "skill.converse.pong", + "skill.converse.pong", + "skill.converse.get_response", # B + "intent.service.skills.activated", + f"{self.skill_id}.activate", + "ovos.session.update_default", # sync skill trigger + "skill.converse.get_response.disable", + "ovos.session.update_default", # sync get_response status + + # 3rd get_response + "skill.converse.get_response.enable", # start of get_response + "ovos.session.update_default", # sync get_response status + "mycroft.mic.listen", # no dialog in self.get_response + "recognizer_loop:utterance", # C + "skill.converse.ping", + "skill.converse.pong", + "skill.converse.pong", + "skill.converse.get_response", # C + "intent.service.skills.activated", + f"{self.skill_id}.activate", + "ovos.session.update_default", # sync skill trigger + "skill.converse.get_response.disable", + "ovos.session.update_default", # sync get_response status + + # cancel get_response + "skill.converse.get_response.enable", # start of get_response + "ovos.session.update_default", # sync get_response status + "mycroft.mic.listen", # no dialog in self.get_response + "recognizer_loop:utterance", # cancel + "skill.converse.ping", + "skill.converse.pong", + "skill.converse.pong", + "skill.converse.get_response", # cancel + "intent.service.skills.activated", + f"{self.skill_id}.activate", + "ovos.session.update_default", # sync skill trigger + "skill.converse.get_response.disable", + "ovos.session.update_default", # sync get_response status + + "skill_items", # skill emitted message [A, B, C] + + "mycroft.skill.handler.complete", # original intent finished executing + + # session updated at end of intent pipeline + "ovos.session.update_default" + + ] + wait_for_n_messages(len(expected_messages)) + + self.assertEqual(len(expected_messages), len(messages)) + + mtypes = [m.msg_type for m in messages] + for m in expected_messages: + self.assertTrue(m in mtypes) + + # verify that "session" is injected + # (missing in utterance message) and kept in all messages + for m in messages[1:]: + print(m.msg_type, m.context["session"]["session_id"]) + self.assertEqual(m.context["session"]["session_id"], "default") + + # converse intent pipeline + self.assertEqual(messages[1].msg_type, "skill.converse.ping") + self.assertEqual(messages[2].msg_type, "skill.converse.pong") + self.assertEqual(messages[3].msg_type, "skill.converse.pong") + self.assertEqual(messages[2].data["skill_id"], messages[2].context["skill_id"]) + self.assertEqual(messages[3].data["skill_id"], messages[3].context["skill_id"]) + # assert it reports converse method has been implemented by skill + if messages[2].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[2].data["can_handle"]) + self.assertFalse(messages[3].data["can_handle"]) + if messages[3].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[3].data["can_handle"]) + self.assertFalse(messages[2].data["can_handle"]) + + # verify skill is activated by intent service (intent pipeline matched) + self.assertEqual(messages[4].msg_type, "intent.service.skills.activated") + self.assertEqual(messages[4].data["skill_id"], self.skill_id) + self.assertEqual(messages[5].msg_type, f"{self.skill_id}.activate") + + # verify intent triggers + self.assertEqual(messages[6].msg_type, f"{self.skill_id}:test_get_response_cascade.intent") + + # verify intent execution + self.assertEqual(messages[7].msg_type, "mycroft.skill.handler.start") + self.assertEqual(messages[7].data["name"], "TestAbortSkill.handle_test_get_response_cascade") + + # post self.get_response intent code + self.assertEqual(messages[8].msg_type, "enclosure.active_skill") + self.assertEqual(messages[8].data["skill_id"], self.skill_id) + self.assertEqual(messages[9].msg_type, "speak") + self.assertEqual(messages[9].data["lang"], "en-us") + self.assertFalse(messages[9].data["expect_response"]) + self.assertEqual(messages[9].data["utterance"], "give me items") + self.assertEqual(messages[9].data["meta"]["skill"], self.skill_id) + + # enable get_response for this session + self.assertEqual(messages[10].msg_type, "skill.converse.get_response.enable") + self.assertEqual(messages[11].msg_type, "ovos.session.update_default") + + # 3 sound prompts (no dialog in this test) + self.assertEqual(messages[12].msg_type, "mycroft.mic.listen") + + # check utterance goes through converse cycle + self.assertEqual(messages[13].msg_type, "recognizer_loop:utterance") + self.assertEqual(messages[14].msg_type, "skill.converse.ping") + + # assert it reports converse method has been implemented by skill + if messages[2].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[15].data["can_handle"]) + self.assertFalse(messages[16].data["can_handle"]) + if messages[3].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[16].data["can_handle"]) + self.assertFalse(messages[15].data["can_handle"]) + + # captured utterance sent to get_response handler that is waiting + self.assertEqual(messages[17].msg_type, "skill.converse.get_response") + self.assertEqual(messages[17].data["skill_id"], self.skill_id) + self.assertEqual(messages[17].data["utterances"], ["A"]) + + # converse pipeline activates the skill last_used timestamp + self.assertEqual(messages[18].msg_type, "intent.service.skills.activated") + self.assertEqual(messages[19].msg_type, f"{self.skill_id}.activate") + self.assertEqual(messages[20].msg_type, "ovos.session.update_default") + + # disable get_response for this session + self.assertEqual(messages[21].msg_type, "skill.converse.get_response.disable") + self.assertEqual(messages[22].msg_type, "ovos.session.update_default") + + ## response 2 + + # enable get_response for this session + self.assertEqual(messages[23].msg_type, "skill.converse.get_response.enable") + self.assertEqual(messages[24].msg_type, "ovos.session.update_default") + + # 3 sound prompts (no dialog in this test) + self.assertEqual(messages[25].msg_type, "mycroft.mic.listen") + + # check utterance goes through converse cycle + self.assertEqual(messages[26].msg_type, "recognizer_loop:utterance") + self.assertEqual(messages[27].msg_type, "skill.converse.ping") + + # assert it reports converse method has been implemented by skill + if messages[2].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[28].data["can_handle"]) + self.assertFalse(messages[29].data["can_handle"]) + if messages[3].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[29].data["can_handle"]) + self.assertFalse(messages[28].data["can_handle"]) + + # captured utterance sent to get_response handler that is waiting + self.assertEqual(messages[30].msg_type, "skill.converse.get_response") + self.assertEqual(messages[30].data["skill_id"], self.skill_id) + self.assertEqual(messages[30].data["utterances"], ["B"]) + + # converse pipeline activates the skill last_used timestamp + self.assertEqual(messages[31].msg_type, "intent.service.skills.activated") + self.assertEqual(messages[32].msg_type, f"{self.skill_id}.activate") + self.assertEqual(messages[33].msg_type, "ovos.session.update_default") + + # disable get_response for this session + self.assertEqual(messages[34].msg_type, "skill.converse.get_response.disable") + self.assertEqual(messages[35].msg_type, "ovos.session.update_default") + + ## response 3 + + # enable get_response for this session + self.assertEqual(messages[36].msg_type, "skill.converse.get_response.enable") + self.assertEqual(messages[37].msg_type, "ovos.session.update_default") + + # 3 sound prompts (no dialog in this test) + self.assertEqual(messages[38].msg_type, "mycroft.mic.listen") + + # check utterance goes through converse cycle + self.assertEqual(messages[39].msg_type, "recognizer_loop:utterance") + self.assertEqual(messages[40].msg_type, "skill.converse.ping") + + # assert it reports converse method has been implemented by skill + if messages[2].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[41].data["can_handle"]) + self.assertFalse(messages[42].data["can_handle"]) + if messages[3].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[42].data["can_handle"]) + self.assertFalse(messages[41].data["can_handle"]) + + # captured utterance sent to get_response handler that is waiting + self.assertEqual(messages[43].msg_type, "skill.converse.get_response") + self.assertEqual(messages[43].data["skill_id"], self.skill_id) + self.assertEqual(messages[43].data["utterances"], ["C"]) + + # converse pipeline activates the skill last_used timestamp + self.assertEqual(messages[44].msg_type, "intent.service.skills.activated") + self.assertEqual(messages[45].msg_type, f"{self.skill_id}.activate") + self.assertEqual(messages[46].msg_type, "ovos.session.update_default") + + # disable get_response for this session + self.assertEqual(messages[47].msg_type, "skill.converse.get_response.disable") + self.assertEqual(messages[48].msg_type, "ovos.session.update_default") + + ## response 3 + + # enable get_response for this session + self.assertEqual(messages[49].msg_type, "skill.converse.get_response.enable") + self.assertEqual(messages[50].msg_type, "ovos.session.update_default") + + # 3 sound prompts (no dialog in this test) + self.assertEqual(messages[51].msg_type, "mycroft.mic.listen") + + # check utterance goes through converse cycle + self.assertEqual(messages[52].msg_type, "recognizer_loop:utterance") + self.assertEqual(messages[53].msg_type, "skill.converse.ping") + + # assert it reports converse method has been implemented by skill + if messages[2].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[54].data["can_handle"]) + self.assertFalse(messages[55].data["can_handle"]) + if messages[3].data["skill_id"] == self.skill_id: # we dont know order of pong responses + self.assertTrue(messages[55].data["can_handle"]) + self.assertFalse(messages[54].data["can_handle"]) + + # captured utterance sent to get_response handler that is waiting + self.assertEqual(messages[56].msg_type, "skill.converse.get_response") + self.assertEqual(messages[56].data["skill_id"], self.skill_id) + self.assertEqual(messages[56].data["utterances"], ["cancel"]) + + # converse pipeline activates the skill last_used timestamp + self.assertEqual(messages[57].msg_type, "intent.service.skills.activated") + self.assertEqual(messages[58].msg_type, f"{self.skill_id}.activate") + self.assertEqual(messages[59].msg_type, "ovos.session.update_default") + + # disable get_response for this session + self.assertEqual(messages[60].msg_type, "skill.converse.get_response.disable") + self.assertEqual(messages[61].msg_type, "ovos.session.update_default") + + # intent return + self.assertEqual(messages[62].msg_type, "skill_items") + self.assertEqual(messages[62].data, {"items": ["A", "B", "C"]}) + + # report handler complete + self.assertEqual(messages[63].msg_type, "mycroft.skill.handler.complete") + self.assertEqual(messages[63].data["name"], "TestAbortSkill.handle_test_get_response_cascade") + + self.assertEqual(messages[64].msg_type, "ovos.session.update_default")