Add intent tests with Adapt and Padatious support

Update resource tests to only test resources and not skill init

neon_minerva/cli.py

@@ -27,25 +27,59 @@
 import os
 import click
 
-from os.path import expanduser, relpath, isfile
+from os.path import expanduser, relpath, isfile, isdir
 from click_default_group import DefaultGroup
 from unittest.runner import TextTestRunner
 from unittest import makeSuite
 
 from neon_minerva.version import __version__
 
 
-def _init_test_dir():
+def _init_tests(debug: bool = False):
     from os.path import join
     from os import makedirs
     from tempfile import mkdtemp
     base_dir = mkdtemp()
     config = join(base_dir, "config")
     data = join(base_dir, "data")
+    cache = join(base_dir, "cache")
     makedirs(config, exist_ok=True)
     makedirs(data, exist_ok=True)
+    makedirs(cache, exist_ok=True)
     os.environ["XDG_CONFIG_HOME"] = config
     os.environ["XDG_DATA_HOME"] = data
+    os.environ["XDG_CACHE_HOME"] = cache
+
+    if debug:
+        os.environ["OVOS_DEFAULT_LOG_LEVEL"] = "DEBUG"
+
+
+def _get_test_file(test_file: str) -> str:
+    """
+    Parse an input path to locate a test file that may be relative to `~` or the
+    current working directory.
+    @param test_file: test file argument
+    @returns: best guess at the desired file path (may not exist)
+    """
+    test_file = expanduser(test_file)
+    if not isfile(test_file):
+        test_file = relpath(test_file)
+    return test_file
+
+
+def _get_skill_entrypoint(skill_entrypoint: str) -> str:
+    """
+    Parse an input skill entrypoint and resolve either a locally installed skill
+    path, or an entrypoint for a plugin skill.
+    @param skill_entrypoint: Plugin entrypoint or path to skill
+    @returns: absolute file path if exists, else input entrypoint
+    """
+    skill_path = expanduser(skill_entrypoint)
+    if not isdir(skill_path):
+        skill_path = relpath(skill_path)
+    if isdir(skill_path):
+        return skill_path
+    return skill_entrypoint
 
 
 @click.group("minerva", cls=DefaultGroup,
@@ -61,18 +95,34 @@ def neon_minerva_cli(version: bool = False):
 
 
 @neon_minerva_cli.command
+@click.option('--debug', is_flag=True, default=False,
+              help="Flag to enable debug logging")
 @click.argument("skill_entrypoint")
 @click.argument("test_file")
-def test_resources(skill_entrypoint, test_file):
-    _init_test_dir()
-    os.environ["TEST_SKILL_ENTRYPOINT"] = skill_entrypoint
-    test_file = expanduser(test_file)
-    if not isfile(test_file):
-        test_file = relpath(test_file)
+def test_resources(skill_entrypoint, test_file, debug):
+    _init_tests(debug)
+    os.environ["TEST_SKILL_ENTRYPOINT"] = _get_skill_entrypoint(skill_entrypoint)
+    test_file = _get_test_file(test_file)
     if not isfile(test_file):
         click.echo(f"Could not find test file: {test_file}")
         exit(2)
     os.environ["RESOURCE_TEST_FILE"] = test_file
-    from neon_minerva.tests.test_skill_resources import TestSkillLoading
-    TextTestRunner().run(makeSuite(TestSkillLoading))
+    from neon_minerva.tests.test_skill_resources import TestSkillResources
+    TextTestRunner().run(makeSuite(TestSkillResources))
+
 
+@neon_minerva_cli.command
+@click.option('--debug', is_flag=True, default=False,
+              help="Flag to enable debug logging")
+@click.argument("skill_entrypoint")
+@click.argument("test_file")
+def test_intents(skill_entrypoint, test_file, debug):
+    _init_tests(debug)
+    os.environ["TEST_SKILL_ENTRYPOINT"] = _get_skill_entrypoint(skill_entrypoint)
+    test_file = _get_test_file(test_file)
+    if not isfile(test_file):
+        click.echo(f"Could not find test file: {test_file}")
+        exit(2)
+    os.environ["INTENT_TEST_FILE"] = test_file
+    from neon_minerva.tests.test_skill_intents import TestSkillIntentMatching
+    TextTestRunner().run(makeSuite(TestSkillIntentMatching))

neon_minerva/intent_services/__init__.py

@@ -0,0 +1,30 @@
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System
+# All trademark and other rights reserved by their respective owners
+# Copyright 2008-2021 Neongecko.com Inc.
+# BSD-3
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from this
+#    software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS  BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS;  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from collections import namedtuple
+
+IntentMatch = namedtuple('IntentMatch',
+                         ['intent_service', 'intent_type',
+                          'intent_data', 'skill_id', 'utterance'])

neon_minerva/intent_services/adapt.py

@@ -0,0 +1,88 @@
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework
+# All trademark and other rights reserved by their respective owners
+# Copyright 2008-2022 Neongecko.com Inc.
+# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds,
+# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo
+# BSD-3 License
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from this
+#    software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS  BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS;  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from typing import Optional
+from adapt.engine import IntentDeterminationEngine
+from ovos_utils.intents.intent_service_interface import open_intent_envelope
+from ovos_utils.log import LOG
+from ovos_utils.messagebus import FakeBus, get_message_lang
+
+from neon_minerva.exceptions import IntentNotMatched, ConfidenceTooLow
+from neon_minerva.intent_services import IntentMatch
+
+
+class AdaptContainer:
+    def __init__(self, lang: str, bus: FakeBus):
+        self.lang = lang.lower()
+        self.bus = bus
+        self.adapt = IntentDeterminationEngine()
+        self.bus.on('register_vocab', self.handle_register_vocab)
+        self.bus.on('register_intent', self.handle_register_intent)
+
+    def handle_register_vocab(self, message):
+        entity_value = message.data.get('entity_value')
+        entity_type = message.data.get('entity_type')
+        regex_str = message.data.get('regex')
+        alias_of = message.data.get('alias_of')
+        lang = get_message_lang(message)
+        if lang != self.lang:
+            return
+        if regex_str:
+            self.adapt.register_regex_entity(regex_str)
+        else:
+            self.adapt.register_entity(entity_value, entity_type,
+                                       alias_of=alias_of)
+
+    def handle_register_intent(self, message):
+        intent = open_intent_envelope(message)
+        self.adapt.register_intent_parser(intent)
+
+    def test_intent(self, utterance: str) -> Optional[IntentMatch]:
+        best_intent = None
+        try:
+            intents = [i for i in self.adapt.determine_intent(
+                utterance, 100,
+                include_tags=True)]
+            if intents:
+                best_intent = max(intents,
+                                  key=lambda x: x.get('confidence', 0.0))
+        except Exception as err:
+            LOG.exception(err)
+
+        if not best_intent:
+            raise IntentNotMatched(utterance)
+        LOG.debug(best_intent)
+        skill_id = best_intent['intent_type'].split(":")[0]
+        _norm_id = skill_id.replace('.', '_')
+        intent_data = {k.replace(_norm_id, '', 1): v for k, v in
+                       best_intent.items() if k.startswith(_norm_id) and
+                       isinstance(v, str)}
+        LOG.debug(intent_data)
+        ret = IntentMatch('Adapt', best_intent['intent_type'], intent_data,
+                          skill_id, utterance)
+        return ret

neon_minerva/intent_services/padatious.py

@@ -0,0 +1,100 @@
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework
+# All trademark and other rights reserved by their respective owners
+# Copyright 2008-2022 Neongecko.com Inc.
+# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds,
+# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo
+# BSD-3 License
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from this
+#    software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS  BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS;  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from padatious import IntentContainer
+from ovos_utils.log import LOG
+from ovos_utils.messagebus import FakeBus
+
+from neon_minerva.exceptions import IntentNotMatched, ConfidenceTooLow
+from neon_minerva.intent_services import IntentMatch
+
+
+class PadatiousContainer:
+    def __init__(self, lang: str, cache_path: str, bus: FakeBus):
+        self.lang = lang.lower()
+        self.bus = bus
+        self.padatious = IntentContainer(cache_path)
+        self.bus.on('padatious:register_intent', self.register_intent)
+        self.bus.on('padatious:register_entity', self.register_entity)
+
+    def register_intent(self, message):
+        """Messagebus handler for registering intents.
+
+        Args:
+            message (Message): message triggering action
+        """
+        lang = message.data.get('lang', self.lang)
+        lang = lang.lower()
+        if lang == self.lang:
+            LOG.debug(f"Loading intent: {message.data['name']}")
+            self.padatious.load_intent(message.data['name'],
+                                       message.data['file_name'])
+        else:
+            LOG.debug(f"Ignoring {message.data['name']}")
+
+    def register_entity(self, message):
+        """Messagebus handler for registering entities.
+
+        Args:
+            message (Message): message triggering action
+        """
+        lang = message.data.get('lang', self.lang)
+        lang = lang.lower()
+        if lang == self.lang:
+            self.padatious.load_entity(message.data['name'],
+                                       message.data['file_name'])
+
+    def calc_intent(self, utt: str) -> dict:
+        intent = self.padatious.calc_intent(utt)
+        LOG.debug(intent)
+        return intent.__dict__ if intent else dict()
+
+
+class TestPadatiousMatcher:
+    def __init__(self, container: PadatiousContainer,
+                 include_med: bool = True, include_low: bool = False):
+        LOG.debug("Creating test Padatious Matcher")
+        if include_low:
+            self.min_conf = 0.5
+        elif include_med:
+            self.min_conf = 0.8
+        else:
+            self.min_conf = 0.95
+        self.padatious = container
+
+    def test_intent(self, utterance: str) -> IntentMatch:
+        intent = self.padatious.calc_intent(utterance)
+        if not intent:
+            raise IntentNotMatched(utterance)
+        conf = intent.get("conf") or 0.0
+        if conf < self.min_conf:
+            raise ConfidenceTooLow(f"{conf} less than minimum {self.min_conf}")
+        skill_id = intent.get('name').split(':')[0]
+        sentence = ' '.join(intent.get('sent'))
+        return IntentMatch('Padatious', intent.get('name'),
+                           intent.get('matches'), skill_id, sentence)

neon_minerva/skill.py

@@ -23,28 +23,34 @@
 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-from typing import Optional
 
 import yaml
 
-from os.path import expanduser, isfile
-
+from os.path import expanduser, isfile, isdir
+from typing import Optional
 from ovos_utils.messagebus import FakeBus
 from ovos_workshop.skills.base import BaseSkill
+from ovos_utils.log import LOG
 
 
 def get_skill_object(skill_entrypoint: str, bus: FakeBus,
                      skill_id: str, config_patch: Optional[dict] = None) -> BaseSkill:
     """
     Get an initialized skill object by entrypoint with the requested skill_id.
-    @param skill_entrypoint: Skill plugin entrypoint
+    @param skill_entrypoint: Skill plugin entrypoint or directory path
     @param bus: FakeBus instance to bind to skill for testing
     @param skill_id: skill_id to initialize skill with
     @returns: Initialized skill object
     """
     if config_patch:
         from ovos_config.config import update_mycroft_config
         update_mycroft_config(config_patch)
+    if isdir(skill_entrypoint):
+        LOG.info(f"Loading local skill: {skill_entrypoint}")
+        from ovos_workshop.skill_launcher import SkillLoader
+        loader = SkillLoader(bus, skill_entrypoint, skill_id)
+        if loader.load():
+            return loader.instance
     from ovos_plugin_manager.skills import find_skill_plugins
     plugins = find_skill_plugins()
     if skill_entrypoint not in plugins:
@@ -68,5 +74,15 @@ def load_resource_tests(test_file: str) -> dict:
     return resources
 
 
-if __name__ == "__main__":
-    get_skill_object("skill-about.neongeckocom", FakeBus(), "test")
+def load_intent_tests(test_file: str) -> dict:
+    """
+    Load intent tests from a file
+    @param test_file: Test file to load
+    @returns: Loaded test spec
+    """
+    test_file = expanduser(test_file)
+    if not isfile(test_file):
+        raise FileNotFoundError(test_file)
+    with open(test_file) as f:
+        intents = yaml.safe_load(f)
+    return intents