feat/runtime skill installer

ovos_core/__main__.py

@@ -29,6 +29,7 @@
 from ovos_utils import wait_for_exit_signal
 from ovos_utils.log import LOG, init_service_logger
 from ovos_utils.process_utils import reset_sigint_handler
+from ovos_core.skill_installer import SkillsStore
 from ovos_workshop.skills.fallback import FallbackSkill
 
 
@@ -52,6 +53,9 @@ def main(alive_hook=on_alive, started_hook=on_started, ready_hook=on_ready,
     event_scheduler = EventScheduler(bus, autostart=False)
     event_scheduler.daemon = True
     event_scheduler.start()
+
+    osm = SkillsStore(bus)
+
     SkillApi.connect_bus(bus)
     skill_manager = SkillManager(bus, watchdog,
                                  alive_hook=alive_hook,
@@ -64,7 +68,7 @@ def main(alive_hook=on_alive, started_hook=on_started, ready_hook=on_ready,
 
     wait_for_exit_signal()
 
-    shutdown(skill_manager, event_scheduler)
+    shutdown(skill_manager, event_scheduler, osm)
 
 
 def _register_intent_services(bus):
@@ -82,14 +86,16 @@ def _register_intent_services(bus):
     return service
 
 
-def shutdown(skill_manager, event_scheduler):
+def shutdown(skill_manager, event_scheduler, osm):
     LOG.info('Shutting down Skills service')
     if event_scheduler is not None:
         event_scheduler.shutdown()
     # Terminate all running threads that update skills
     if skill_manager is not None:
         skill_manager.stop()
         skill_manager.join()
+    if osm is not None:
+        osm.shutdown()
     LOG.info('Skills service shutdown complete!')
 
 

ovos_core/skill_installer.py

@@ -0,0 +1,229 @@
+import enum
+import sys
+from importlib import reload
+from os.path import join, exists
+from subprocess import Popen, PIPE
+from tempfile import gettempdir
+from typing import Optional
+
+from combo_lock import ComboLock
+from ovos_config.config import Configuration
+
+import ovos_plugin_manager
+from ovos_bus_client import Message
+from ovos_utils.log import LOG
+
+
+class InstallError(str, enum.Enum):
+    DISABLED = "pip disabled in mycroft.conf"
+    PIP_ERROR = "error in pip subprocess"
+    BAD_URL = "skill url validation failed"
+    NO_PKGS = "no packages to install"
+
+
+class SkillsStore:
+    # default constraints to use if none are given
+    DEFAULT_CONSTRAINTS = '/etc/mycroft/constraints.txt'
+    PIP_LOCK = ComboLock(join(gettempdir(), "ovos_pip.lock"))
+
+    def __init__(self, bus, config=None):
+        self.config = config or Configuration().get("skills", {}).get("installer", {})
+        self.bus = bus
+        self.bus.on("ovos.skills.install", self.handle_install_skill)
+        self.bus.on("ovos.skills.uninstall", self.handle_uninstall_skill)
+        self.bus.on("ovos.pip.install", self.handle_install_python)
+        self.bus.on("ovos.pip.uninstall", self.handle_uninstall_python)
+
+    def shutdown(self):
+        pass
+
+    def play_error_sound(self):
+        snd = Configuration().get("sounds", {}).get("pip_error", "snd/error.mp3")
+        self.bus.emit(Message("mycroft.audio.play_sound", {"uri": snd}))
+
+    def play_success_sound(self):
+        snd = Configuration().get("sounds", {}).get("pip_success", "snd/acknowledge.mp3")
+        self.bus.emit(Message("mycroft.audio.play_sound", {"uri": snd}))
+
+    def pip_install(self, packages: list,
+                    constraints: Optional[str] = None,
+                    print_logs: bool = True):
+        if not len(packages):
+            LOG.error("no package list provided to install")
+            self.play_error_sound()
+            return False
+        # Use constraints to limit the installed versions
+        if constraints and not exists(constraints):
+            LOG.error('Couldn\'t find the constraints file')
+            self.play_error_sound()
+            return False
+        elif exists(SkillsStore.DEFAULT_CONSTRAINTS):
+            constraints = SkillsStore.DEFAULT_CONSTRAINTS
+
+        pip_args = [sys.executable, '-m', 'pip', 'install']
+        if constraints:
+            pip_args += ['-c', constraints]
+        if self.config.get("break_system_packages", False):
+            pip_args += ["--break-system-packages"]
+
+        with SkillsStore.PIP_LOCK:
+            """
+            Iterate over the individual Python packages and
+            install them one by one to enforce the order specified
+            in the manifest.
+            """
+            for dependent_python_package in packages:
+                LOG.info("(pip) Installing " + dependent_python_package)
+                pip_command = pip_args + [dependent_python_package]
+                LOG.debug(" ".join(pip_command))
+                if print_logs:
+                    proc = Popen(pip_command)
+                else:
+                    proc = Popen(pip_command, stdout=PIPE, stderr=PIPE)
+                pip_code = proc.wait()
+                if pip_code != 0:
+                    stderr = proc.stderr.read().decode()
+                    self.play_error_sound()
+                    raise RuntimeError(stderr)
+
+        reload(ovos_plugin_manager)  # force core to pick new entry points
+        self.play_success_sound()
+        return True
+
+    def pip_uninstall(self, packages: list,
+                      constraints: Optional[str] = None,
+                      print_logs: bool = True):
+        if not len(packages):
+            LOG.error("no package list provided to uninstall")
+            self.play_error_sound()
+            return False
+
+        # Use constraints to limit package removal
+        if constraints and not exists(constraints):
+            LOG.error('Couldn\'t find the constraints file')
+            self.play_error_sound()
+            return False
+        elif exists(SkillsStore.DEFAULT_CONSTRAINTS):
+            constraints = SkillsStore.DEFAULT_CONSTRAINTS
+
+        if constraints:
+            with open(constraints) as f:
+                # remove version pinning and normalize _ to - (pip accepts both)
+                cpkgs = [p.split("~")[0].split("<")[0].split(">")[0].split("=")[0].replace("_", "-")
+                         for p in f.read().split("\n") if p.strip()]
+        else:
+            cpkgs = ["ovos-core", "ovos-utils", "ovos-plugin-manager",
+                     "ovos-config", "ovos-bus-client", "ovos-workshop"]
+
+        # normalize _ to - (pip accepts both)
+        if any(p.replace("_", "-") in cpkgs for p in packages):
+            LOG.error(f'tried to uninstall a protected package: {cpkgs}')
+            self.play_error_sound()
+            return False
+
+        pip_args = [sys.executable, '-m', 'pip', 'uninstall', '-y']
+        if self.config.get("break_system_packages", False):
+            pip_args += ["--break-system-packages"]
+
+        with SkillsStore.PIP_LOCK:
+            """
+            Iterate over the individual Python packages and
+            install them one by one to enforce the order specified
+            in the manifest.
+            """
+            for dependent_python_package in packages:
+                LOG.info("(pip) Uninstalling " + dependent_python_package)
+                pip_command = pip_args + [dependent_python_package]
+                LOG.debug(" ".join(pip_command))
+                if print_logs:
+                    proc = Popen(pip_command)
+                else:
+                    proc = Popen(pip_command, stdout=PIPE, stderr=PIPE)
+                pip_code = proc.wait()
+                if pip_code != 0:
+                    stderr = proc.stderr.read().decode()
+                    self.play_error_sound()
+                    raise RuntimeError(stderr)
+
+        reload(ovos_plugin_manager)  # force core to pick new entry points
+        self.play_success_sound()
+        return True
+
+    def validate_skill(self, url):
+        if not url.startswith("https://github.com/"):
+            return False
+        # TODO - check if setup.py
+        # TODO - check if not using MycroftSkill class
+        # TODO - check if not mycroft CommonPlay
+        return True
+
+    def handle_install_skill(self, message: Message):
+        if not self.config.get("allow_pip"):
+            LOG.error(InstallError.DISABLED.value)
+            self.play_error_sound()
+            self.bus.emit(message.reply("ovos.skills.install.failed",
+                                        {"error": InstallError.DISABLED.value}))
+            return
+
+        url = message.data["url"]
+        if self.validate_skill(url):
+            success = self.pip_install([f"git+{url}"])
+            if success:
+                self.bus.emit(message.reply("ovos.skills.install.complete"))
+            else:
+                self.bus.emit(message.reply("ovos.skills.install.failed",
+                                            {"error": InstallError.PIP_ERROR.value}))
+        else:
+            LOG.error("invalid skill url, does not appear to be a github skill")
+            self.play_error_sound()
+            self.bus.emit(message.reply("ovos.skills.install.failed",
+                                        {"error": InstallError.BAD_URL.value}))
+
+    def handle_uninstall_skill(self, message: Message):
+        if not self.config.get("allow_pip"):
+            LOG.error(InstallError.DISABLED.value)
+            self.play_error_sound()
+            self.bus.emit(message.reply("ovos.skills.uninstall.failed",
+                                        {"error": InstallError.DISABLED.value}))
+            return
+        # TODO
+        LOG.error("pip uninstall not yet implemented")
+        self.play_error_sound()
+        self.bus.emit(message.reply("ovos.skills.uninstall.failed",
+                                    {"error": "not implemented"}))
+
+    def handle_install_python(self, message: Message):
+        if not self.config.get("allow_pip"):
+            LOG.error(InstallError.DISABLED.value)
+            self.play_error_sound()
+            self.bus.emit(message.reply("ovos.pip.install.failed",
+                                        {"error": InstallError.DISABLED.value}))
+            return
+        pkgs = message.data["packages"]
+        if pkgs:
+            if self.pip_install(pkgs):
+                self.bus.emit(message.reply("ovos.pip.install.complete"))
+            else:
+                self.bus.emit(message.reply("ovos.pip.install.failed",
+                                            {"error": InstallError.PIP_ERROR.value}))
+        else:
+            self.bus.emit(message.reply("ovos.pip.install.failed",
+                                        {"error": InstallError.NO_PKGS.value}))
+
+    def handle_uninstall_python(self, message: Message):
+        if not self.config.get("allow_pip"):
+            LOG.error(InstallError.DISABLED.value)
+            self.play_error_sound()
+            self.bus.emit(message.reply("ovos.pip.uninstall.failed",
+                                        {"error": InstallError.DISABLED.value}))
+            return
+        pkgs = message.data["packages"]
+        if pkgs:
+            if self.pip_uninstall(pkgs):
+                self.bus.emit(message.reply("ovos.pip.uninstall.complete"))
+            else:
+                self.bus.emit(message.reply("ovos.pip.uninstall.failed",
+                                            {"error": InstallError.PIP_ERROR.value}))
+        else:
+            self.bus.emit(message.reply("ovos.pip.uninstall.failed",
+                                        {"error": InstallError.NO_PKGS.value}))