Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat/runtime skill installer #347

Merged
merged 18 commits into from
Sep 19, 2023
10 changes: 8 additions & 2 deletions ovos_core/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from ovos_utils import wait_for_exit_signal
from ovos_utils.log import LOG, init_service_logger
from ovos_utils.process_utils import reset_sigint_handler
from ovos_core.skill_installer import SkillsStore
from ovos_workshop.skills.fallback import FallbackSkill


Expand All @@ -52,6 +53,9 @@ def main(alive_hook=on_alive, started_hook=on_started, ready_hook=on_ready,
event_scheduler = EventScheduler(bus, autostart=False)
event_scheduler.daemon = True
event_scheduler.start()

osm = SkillsStore(bus)

SkillApi.connect_bus(bus)
skill_manager = SkillManager(bus, watchdog,
alive_hook=alive_hook,
Expand All @@ -64,7 +68,7 @@ def main(alive_hook=on_alive, started_hook=on_started, ready_hook=on_ready,

wait_for_exit_signal()

shutdown(skill_manager, event_scheduler)
shutdown(skill_manager, event_scheduler, osm)


def _register_intent_services(bus):
Expand All @@ -82,14 +86,16 @@ def _register_intent_services(bus):
return service


def shutdown(skill_manager, event_scheduler):
def shutdown(skill_manager, event_scheduler, osm):
LOG.info('Shutting down Skills service')
if event_scheduler is not None:
event_scheduler.shutdown()
# Terminate all running threads that update skills
if skill_manager is not None:
skill_manager.stop()
skill_manager.join()
if osm is not None:
osm.shutdown()
LOG.info('Skills service shutdown complete!')


Expand Down
229 changes: 229 additions & 0 deletions ovos_core/skill_installer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
import enum
import sys
from importlib import reload
from os.path import join, exists
from subprocess import Popen, PIPE
from tempfile import gettempdir
from typing import Optional

from combo_lock import ComboLock
from ovos_config.config import Configuration

import ovos_plugin_manager
from ovos_bus_client import Message
from ovos_utils.log import LOG


class InstallError(str, enum.Enum):
DISABLED = "pip disabled in mycroft.conf"
PIP_ERROR = "error in pip subprocess"
BAD_URL = "skill url validation failed"
NO_PKGS = "no packages to install"


class SkillsStore:
# default constraints to use if none are given
DEFAULT_CONSTRAINTS = '/etc/mycroft/constraints.txt'
PIP_LOCK = ComboLock(join(gettempdir(), "ovos_pip.lock"))

def __init__(self, bus, config=None):
self.config = config or Configuration().get("skills", {}).get("installer", {})
self.bus = bus
self.bus.on("ovos.skills.install", self.handle_install_skill)
self.bus.on("ovos.skills.uninstall", self.handle_uninstall_skill)
self.bus.on("ovos.pip.install", self.handle_install_python)
self.bus.on("ovos.pip.uninstall", self.handle_uninstall_python)

def shutdown(self):
pass

def play_error_sound(self):
snd = Configuration().get("sounds", {}).get("pip_error", "snd/error.mp3")
self.bus.emit(Message("mycroft.audio.play_sound", {"uri": snd}))

def play_success_sound(self):
snd = Configuration().get("sounds", {}).get("pip_success", "snd/acknowledge.mp3")
self.bus.emit(Message("mycroft.audio.play_sound", {"uri": snd}))

def pip_install(self, packages: list,
constraints: Optional[str] = None,
print_logs: bool = True):
if not len(packages):
LOG.error("no package list provided to install")
self.play_error_sound()
return False
# Use constraints to limit the installed versions
if constraints and not exists(constraints):
LOG.error('Couldn\'t find the constraints file')
self.play_error_sound()
return False
elif exists(SkillsStore.DEFAULT_CONSTRAINTS):
constraints = SkillsStore.DEFAULT_CONSTRAINTS

pip_args = [sys.executable, '-m', 'pip', 'install']
if constraints:
pip_args += ['-c', constraints]
if self.config.get("break_system_packages", False):
pip_args += ["--break-system-packages"]
JarbasAl marked this conversation as resolved.
Show resolved Hide resolved

with SkillsStore.PIP_LOCK:
"""
Iterate over the individual Python packages and
install them one by one to enforce the order specified
in the manifest.
"""
for dependent_python_package in packages:
LOG.info("(pip) Installing " + dependent_python_package)
pip_command = pip_args + [dependent_python_package]
LOG.debug(" ".join(pip_command))
if print_logs:
proc = Popen(pip_command)
else:
proc = Popen(pip_command, stdout=PIPE, stderr=PIPE)
pip_code = proc.wait()
if pip_code != 0:
stderr = proc.stderr.read().decode()
self.play_error_sound()
raise RuntimeError(stderr)
JarbasAl marked this conversation as resolved.
Show resolved Hide resolved

reload(ovos_plugin_manager) # force core to pick new entry points
self.play_success_sound()
return True

def pip_uninstall(self, packages: list,
constraints: Optional[str] = None,
print_logs: bool = True):
if not len(packages):
LOG.error("no package list provided to uninstall")
self.play_error_sound()
return False

# Use constraints to limit package removal
if constraints and not exists(constraints):
LOG.error('Couldn\'t find the constraints file')
self.play_error_sound()
return False
elif exists(SkillsStore.DEFAULT_CONSTRAINTS):
constraints = SkillsStore.DEFAULT_CONSTRAINTS

if constraints:
with open(constraints) as f:
# remove version pinning and normalize _ to - (pip accepts both)
cpkgs = [p.split("~")[0].split("<")[0].split(">")[0].split("=")[0].replace("_", "-")
for p in f.read().split("\n") if p.strip()]
else:
cpkgs = ["ovos-core", "ovos-utils", "ovos-plugin-manager",
"ovos-config", "ovos-bus-client", "ovos-workshop"]

# normalize _ to - (pip accepts both)
if any(p.replace("_", "-") in cpkgs for p in packages):
LOG.error(f'tried to uninstall a protected package: {cpkgs}')
self.play_error_sound()
return False

pip_args = [sys.executable, '-m', 'pip', 'uninstall', '-y']
if self.config.get("break_system_packages", False):
pip_args += ["--break-system-packages"]

with SkillsStore.PIP_LOCK:
"""
Iterate over the individual Python packages and
install them one by one to enforce the order specified
in the manifest.
"""
for dependent_python_package in packages:
LOG.info("(pip) Uninstalling " + dependent_python_package)
pip_command = pip_args + [dependent_python_package]
LOG.debug(" ".join(pip_command))
if print_logs:
proc = Popen(pip_command)
else:
proc = Popen(pip_command, stdout=PIPE, stderr=PIPE)
pip_code = proc.wait()
if pip_code != 0:
stderr = proc.stderr.read().decode()
self.play_error_sound()
raise RuntimeError(stderr)
JarbasAl marked this conversation as resolved.
Show resolved Hide resolved

reload(ovos_plugin_manager) # force core to pick new entry points
JarbasAl marked this conversation as resolved.
Show resolved Hide resolved
self.play_success_sound()
return True

def validate_skill(self, url):
if not url.startswith("https://github.com/"):
return False
# TODO - check if setup.py
# TODO - check if not using MycroftSkill class
# TODO - check if not mycroft CommonPlay
return True

def handle_install_skill(self, message: Message):
if not self.config.get("allow_pip"):
JarbasAl marked this conversation as resolved.
Show resolved Hide resolved
LOG.error(InstallError.DISABLED.value)
self.play_error_sound()
self.bus.emit(message.reply("ovos.skills.install.failed",
{"error": InstallError.DISABLED.value}))
return

url = message.data["url"]
if self.validate_skill(url):
success = self.pip_install([f"git+{url}"])
if success:
self.bus.emit(message.reply("ovos.skills.install.complete"))
else:
self.bus.emit(message.reply("ovos.skills.install.failed",
{"error": InstallError.PIP_ERROR.value}))
else:
LOG.error("invalid skill url, does not appear to be a github skill")
self.play_error_sound()
self.bus.emit(message.reply("ovos.skills.install.failed",
{"error": InstallError.BAD_URL.value}))

def handle_uninstall_skill(self, message: Message):
if not self.config.get("allow_pip"):
LOG.error(InstallError.DISABLED.value)
self.play_error_sound()
self.bus.emit(message.reply("ovos.skills.uninstall.failed",
{"error": InstallError.DISABLED.value}))
return
# TODO
LOG.error("pip uninstall not yet implemented")
self.play_error_sound()
self.bus.emit(message.reply("ovos.skills.uninstall.failed",
{"error": "not implemented"}))

def handle_install_python(self, message: Message):
if not self.config.get("allow_pip"):
LOG.error(InstallError.DISABLED.value)
self.play_error_sound()
self.bus.emit(message.reply("ovos.pip.install.failed",
{"error": InstallError.DISABLED.value}))
return
pkgs = message.data["packages"]
if pkgs:
if self.pip_install(pkgs):
self.bus.emit(message.reply("ovos.pip.install.complete"))
else:
self.bus.emit(message.reply("ovos.pip.install.failed",
{"error": InstallError.PIP_ERROR.value}))
else:
self.bus.emit(message.reply("ovos.pip.install.failed",
{"error": InstallError.NO_PKGS.value}))

def handle_uninstall_python(self, message: Message):
if not self.config.get("allow_pip"):
LOG.error(InstallError.DISABLED.value)
self.play_error_sound()
self.bus.emit(message.reply("ovos.pip.uninstall.failed",
{"error": InstallError.DISABLED.value}))
return
pkgs = message.data["packages"]
if pkgs:
if self.pip_uninstall(pkgs):
JarbasAl marked this conversation as resolved.
Show resolved Hide resolved
self.bus.emit(message.reply("ovos.pip.uninstall.complete"))
else:
self.bus.emit(message.reply("ovos.pip.uninstall.failed",
{"error": InstallError.PIP_ERROR.value}))
else:
self.bus.emit(message.reply("ovos.pip.uninstall.failed",
{"error": InstallError.NO_PKGS.value}))
Loading