diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml new file mode 100644 index 0000000..203ba25 --- /dev/null +++ b/.github/workflows/license_tests.yml @@ -0,0 +1,10 @@ +name: Run License Tests +on: + push: + workflow_dispatch: + +jobs: + license_tests: + uses: neongeckocom/.github/.github/workflows/license_tests.yml@master + with: + packages-exclude: '^(precise-runner|fann2|tqdm|bs4|ovos-phal-plugin|ovos-skill|neon-core|nvidia|neon-phal-plugin|bitstruct|audioread).*' \ No newline at end of file diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index 65ee102..ff73f4a 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -7,34 +7,10 @@ on: - master jobs: - tag_release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Get Version - run: | - VERSION=$(python setup.py --version) - echo "VERSION=${VERSION}" >> $GITHUB_ENV - - uses: ncipollo/release-action@v1 - with: - token: ${{secrets.GITHUB_TOKEN}} - tag: ${{env.VERSION}} - build_and_publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install Build Tools - run: | - python -m pip install build wheel - - - name: Build Distribution Packages - run: | - python setup.py bdist_wheel - - name: Publish to Test PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{secrets.PYPI_TOKEN}} \ No newline at end of file + build_and_publish_pypi_and_release: + uses: neongeckocom/.github/.github/workflows/publish_stable_release.yml@master + secrets: inherit + build_and_publish_docker: + needs: build_and_publish_pypi_and_release + uses: neongeckocom/.github/.github/workflows/publish_docker.yml@master + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/publish_test_build.yml b/.github/workflows/publish_test_build.yml index c53fcab..742ba53 100644 --- a/.github/workflows/publish_test_build.yml +++ b/.github/workflows/publish_test_build.yml @@ -9,31 +9,13 @@ on: - 'neon_iris/version.py' jobs: - build_and_publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.head_ref }} - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install Build Tools - run: | - python -m pip install build wheel - - name: Increment Version - run: | - VER=$(python setup.py --version) - python version_bump.py - - name: Push Version Change - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: Increment Version - - name: Build Distribution Packages - run: | - python setup.py bdist_wheel - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{secrets.PYPI_TOKEN}} \ No newline at end of file + publish_alpha_release: + uses: neongeckocom/.github/.github/workflows/publish_alpha_release.yml@master + secrets: inherit + with: + version_file: "neon_iris/version.py" + setup_py: "setup.py" + build_and_publish_docker: + needs: publish_alpha_release + uses: neongeckocom/.github/.github/workflows/publish_docker.yml@master + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/pull_master.yml b/.github/workflows/pull_master.yml index 8e9c5a8..8ab60d3 100644 --- a/.github/workflows/pull_master.yml +++ b/.github/workflows/pull_master.yml @@ -5,17 +5,10 @@ on: push: branches: - dev - workflow_dispatch: jobs: pull_changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: pull-request-action - uses: repo-sync/pull-request@v2 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - pr_reviewer: 'neonreviewers' - pr_assignee: 'neondaniel' - pr_draft: true \ No newline at end of file + uses: neongeckocom/.github/.github/workflows/pull_master.yml@master + with: + pr_reviewer: neonreviewers + pr_assignee: neondaniel diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index b7d0f41..69e957a 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -6,20 +6,8 @@ on: workflow_dispatch: jobs: - build_tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install Build Tools - run: | - python -m pip install build wheel - - name: Build Distribution Packages - run: | - python setup.py bdist_wheel + py_build_tests: + uses: neongeckocom/.github/.github/workflows/python_build_tests.yml@master unit_tests: strategy: matrix: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..139a09a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.8-slim + +LABEL vendor=neon.ai \ + ai.neon.name="neon-iris" + +ENV OVOS_CONFIG_BASE_FOLDER neon +ENV OVOS_CONFIG_FILENAME neon.yaml +ENV XDG_CONFIG_HOME /config + +RUN apt update && \ + apt install -y ffmpeg + +ADD . /neon_iris +WORKDIR /neon_iris + +RUN pip install wheel && \ + pip install .[gradio] + +COPY docker_overlay/ / + +CMD ["iris", "start-gradio"] \ No newline at end of file diff --git a/README.md b/README.md index fa8e145..07cf069 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,35 @@ interacting with Neon systems remotely, via [MQ](https://github.com/NeonGeckoCom Install the Iris Python package with: `pip install neon-iris` The `iris` entrypoint is available to interact with a bus via CLI. Help is available via `iris --help`. + +## Configuration +Configuration files can be specified via environment variables. By default, +`Iris` will read configuration from `~/.config/neon/diana.yaml` where +`XDG_CONFIG_HOME` is set to the default `~/.config`. +More information about configuration handling can be found +[in the docs](https://neongeckocom.github.io/neon-docs/quick_reference/configuration/). + +A default configuration might look like: +```yaml +MQ: + server: neonaialpha.com + port: 25672 + users: + mq_handler: + user: neon_api_utils + password: Klatchat2021 +iris: + default_lang: en-us + languages: + - en-us + - uk-ua + webui_chatbot_label: "Neon AI" + webui_mic_label: "Speak with Neon" + webui_input_placeholder: "Chat with Neon" +``` + +## Interfacing with a Diana installation +The `iris` CLI includes utilities for interacting with a `Diana` backend. + + + diff --git a/docker_overlay/etc/neon/neon.yaml b/docker_overlay/etc/neon/neon.yaml new file mode 100644 index 0000000..bf3a9b3 --- /dev/null +++ b/docker_overlay/etc/neon/neon.yaml @@ -0,0 +1,19 @@ +MQ: + server: neon-rabbitmq + port: 5672 + users: + mq_handler: + user: neon_api_utils + password: Klatchat2021 +iris: + webui_title: Neon AI + webui_description: Chat with Neon + webui_input_placeholder: Ask me something + server_address: "0.0.0.0" + server_port: 7860 + default_lang: en-us + languages: + - en-us + - fr-fr + - es-es + - uk-ua \ No newline at end of file diff --git a/neon_iris/cli.py b/neon_iris/cli.py index be4b248..e7e6cef 100644 --- a/neon_iris/cli.py +++ b/neon_iris/cli.py @@ -24,19 +24,31 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import json import logging +from pprint import pformat + import click -import yaml +from os import environ from os.path import expanduser, isfile from time import sleep from click_default_group import DefaultGroup +from ovos_utils.log import LOG -from neon_utils.logger import LOG -from neon_iris.client import CLIClient +from neon_iris.util import load_config_file from neon_iris.version import __version__ +environ.setdefault("OVOS_CONFIG_BASE_FOLDER", "neon") +environ.setdefault("OVOS_CONFIG_FILENAME", "diana.yaml") +# TODO: Define default config file from this package + + +def _print_config(): + from ovos_config.config import Configuration + config = Configuration().get('MQ') + mq_endpoint = f"{config.get('server')}:{config.get('port', 5672)}" + click.echo(f"Connecting to {mq_endpoint}") + @click.group("iris", cls=DefaultGroup, no_args_is_help=True, invoke_without_command=True, @@ -59,21 +71,17 @@ def neon_iris_cli(version: bool = False): @click.option('--audio', '-a', is_flag=True, default=False, help="Flag to enable audio playback") def start_client(mq_config, user_config, lang, audio): + from neon_iris.client import CLIClient + _print_config() if mq_config: - with open(mq_config) as f: - try: - mq_config = json.load(f) - except Exception as e: - f.seek(0) - mq_config = yaml.safe_load(f) + mq_config = load_config_file(expanduser(mq_config)) + else: + from ovos_config.config import Configuration + mq_config = Configuration().get("MQ") if user_config: - with open(user_config) as f: - try: - user_config = json.load(f) - except Exception as e: - user_config = None + user_config = load_config_file(expanduser(user_config)) client = CLIClient(mq_config, user_config) - LOG.init({"level": logging.WARNING}) + LOG.init({"level": logging.WARNING}) # TODO: Debug flag? client.audio_enabled = audio click.echo("Enter '!{lang}' to change language\n" @@ -111,5 +119,138 @@ def start_client(mq_config, user_config, lang, audio): client.shutdown() -if __name__ == "__main__": - start_client(None, None, "en-us") +@neon_iris_cli.command(help="Create an MQ listener session") +def start_listener(): + from neon_iris.voice_client import NeonVoiceClient + from ovos_utils import wait_for_exit_signal + client = NeonVoiceClient() + _print_config() + wait_for_exit_signal() + client.shutdown() + + +@neon_iris_cli.command(help="Create a GradIO Client session") +def start_gradio(): + from neon_iris.web_client import GradIOClient + chat = GradIOClient() + chat.run() + + +@neon_iris_cli.command(help="Transcribe an audio file") +@click.option('--lang', '-l', default='en-us', + help="language of input audio") +@click.argument("audio_file") +def get_stt(audio_file, lang): + from neon_iris.util import get_stt + _print_config() + resp = get_stt(audio_file, lang) + click.echo(pformat(resp)) + + +@neon_iris_cli.command(help="Transcribe an audio file") +@click.option('--lang', '-l', default='en-us', + help="language of input audio") +@click.argument("utterance") +def get_tts(utterance, lang): + from neon_iris.util import get_tts + _print_config() + resp = get_tts(utterance, lang) + click.echo(pformat(resp)) + + +# Backend +@neon_iris_cli.command(help="Query a weather endpoint") +@click.option('--unit', '-u', default='imperial', + help="units to use ('metric' or 'imperial')") +@click.option('--latitude', '--lat', default=47.6815, + help="location latitude") +@click.option('--longitude', '--lon', default=-122.2087, + help="location latitude") +@click.option('--api', '-a', default='onecall', + help="api to query ('onecall' or 'weather')") +def get_weather(unit, latitude, longitude, api): + from neon_iris.util import query_api + _print_config() + query = {"lat": latitude, + "lon": longitude, + "units": unit, + "api": api, + "service": "open_weather_map"} + resp = query_api(query) + click.echo(pformat(resp)) + + +@neon_iris_cli.command(help="Query a stock price endpoint") +@click.argument('symbol') +def get_stock_quote(symbol): + from neon_iris.util import query_api + _print_config() + query = {"symbol": symbol, + "api": "quote", + "service": "alpha_vantage"} + resp = query_api(query) + click.echo(pformat(resp)) + + +@neon_iris_cli.command(help="Query a stock symbol endpoint") +@click.argument('company') +def get_stock_symbol(company): + from neon_iris.util import query_api + _print_config() + query = {"company": company, + "api": "symbol", + "service": "alpha_vantage"} + resp = query_api(query) + click.echo(pformat(resp)) + + +@neon_iris_cli.command(help="Query a WolframAlpha endpoint") +@click.option('--api', '-a', default='short', + help="Wolfram|Alpha API to query") +@click.option('--unit', '-u', default='imperial', + help="units to use ('metric' or 'imperial')") +@click.option('--latitude', '--lat', default=47.6815, + help="location latitude") +@click.option('--longitude', '--lon', default=-122.2087, + help="location latitude") +@click.argument('question') +def get_wolfram_response(api, unit, latitude, longitude, question): + from neon_iris.util import query_api + _print_config() + query = {"api": api, + "units": unit, + "latlong": f"{latitude},{longitude}", + "query": question, + "service": "wolfram_alpha"} + resp = query_api(query) + click.echo(pformat(resp)) + + +@neon_iris_cli.command(help="Converse with an LLM") +@click.option('--llm', default="chat_gpt", + help="LLM Queue to interact with ('chat_gpt' or 'fastchat')") +def start_llm_chat(llm): + from neon_iris.llm import LLMConversation + _print_config() + conversation = LLMConversation(llm) + while True: + query = click.prompt(">") + resp = conversation.get_response(query) + click.echo(resp) + + +@neon_iris_cli.command(help="Converse with an LLM") +def get_coupons(): + from neon_iris.util import get_brands_coupons + data = get_brands_coupons() + click.echo(pformat(data)) + + +@neon_iris_cli.command(help="Parse a Neon CCL script") +@click.argument("script_file") +def parse_script(script_file): + from neon_iris.util import parse_ccl_script + data = parse_ccl_script(script_file) + click.echo(pformat(data)) + +# TODO: email, metrics diff --git a/neon_iris/client.py b/neon_iris/client.py index fd2f148..cb637f7 100644 --- a/neon_iris/client.py +++ b/neon_iris/client.py @@ -37,7 +37,8 @@ from time import time from typing import Optional from uuid import uuid4 -from mycroft_bus_client import Message +from ovos_bus_client.message import Message +from ovos_utils.json_helper import merge_dict from pika.exceptions import StreamLostError from neon_utils.configuration_utils import get_neon_user_config from neon_utils.mq_utils import NeonMQHandler @@ -74,6 +75,10 @@ def uid(self) -> str: """ return self._uid + @property + def default_username(self) -> str: + return self._user_config["user"]["username"] + @property def user_config(self) -> dict: """ @@ -136,6 +141,8 @@ def handle_neon_response(self, channel, method, _, body): self._handle_profile_update(message) elif message.msg_type == "neon.clear_data": self._handle_clear_data(message) + elif message.msg_type == "klat.error": + self.handle_error_response(message) elif message.msg_type.endswith(".response"): self.handle_api_response(message) else: @@ -222,61 +229,78 @@ def _clear_audio_cache(): def send_utterance(self, utterance: str, lang: str = "en-us", username: Optional[str] = None, - user_profiles: Optional[list] = None): + user_profiles: Optional[list] = None, + context: Optional[dict] = None): """ Optionally override this to queue text inputs or do any pre-parsing :param utterance: utterance to submit to skills module :param lang: language code associated with request :param username: username associated with request :param user_profiles: user profiles expecting a response + :param context: Optional dict context to add to emitted message """ - self._send_utterance(utterance, lang, username, user_profiles) + self._send_utterance(utterance, lang, username, user_profiles, context) def send_audio(self, audio_file: str, lang: str = "en-us", username: Optional[str] = None, - user_profiles: Optional[list] = None): + user_profiles: Optional[list] = None, + context: Optional[dict] = None): """ Optionally override this to queue audio inputs or do any pre-parsing :param audio_file: path to audio file to send to speech module :param lang: language code associated with request :param username: username associated with request :param user_profiles: user profiles expecting a response + :param context: Optional dict context to add to emitted message """ - self._send_audio(audio_file, lang, username, user_profiles) + self._send_audio(audio_file, lang, username, user_profiles, context) def _build_message(self, msg_type: str, data: dict, username: Optional[str] = None, user_profiles: Optional[list] = None, ident: str = None) -> Message: + user_profiles = user_profiles or [self.user_config] + username = username or user_profiles[0]['user']['username'] return Message(msg_type, data, {"client_name": self.client_name, "client": self._client, "ident": ident or str(time()), "username": username, - "user_profiles": user_profiles or list(), - "klat_data": {"routing_key": self.uid} + "user_profiles": user_profiles, + "mq": {"routing_key": self.uid, + "message_id": self.connection.create_unique_id()} }) def _send_utterance(self, utterance: str, lang: str, - username: str, user_profiles: list): + username: str, user_profiles: list, + context: Optional[dict] = None): + context = context or dict() + username = username or self.default_username + user_profiles = user_profiles or [self.user_config] message = self._build_message("recognizer_loop:utterance", {"utterances": [utterance], "lang": lang}, username, user_profiles) serialized = {"msg_type": message.msg_type, "data": message.data, - "context": message.context} + "context": merge_dict(message.context, context, + new_only=True)} self._send_serialized_message(serialized) def _send_audio(self, audio_file: str, lang: str, - username: str, user_profiles: list): + username: str, user_profiles: list, + context: Optional[dict] = None): + context = context or dict() audio_data = encode_file_to_base64_string(audio_file) message = self._build_message("neon.audio_input", {"lang": lang, - "audio_data": audio_data}, + "audio_data": audio_data, + "utterances": []}, + # TODO: `utterances` patching mq connector username, user_profiles) serialized = {"msg_type": message.msg_type, "data": message.data, - "context": message.context} + "context": merge_dict(message.context, context, + new_only=True)} self._send_serialized_message(serialized) def _send_serialized_message(self, serialized: dict): @@ -285,12 +309,14 @@ def _send_serialized_message(self, serialized: dict): self._connection.connection, queue="neon_chat_api_request", request_data=serialized) + LOG.debug(f"emitted {serialized.get('msg_type')}") except Exception as e: LOG.exception(e) self.shutdown() def _init_mq_connection(self): - mq_connection = NeonMQHandler(self._config, "mq_handler", self._vhost) + mq_config = self._config.get("MQ") or self._config + mq_connection = NeonMQHandler(mq_config, "mq_handler", self._vhost) mq_connection.register_consumer("neon_response_handler", self._vhost, self.uid, self.handle_neon_response, auto_ack=False) diff --git a/version_bump.py b/neon_iris/llm.py similarity index 67% rename from version_bump.py rename to neon_iris/llm.py index 9ea1a80..6cb0b65 100644 --- a/version_bump.py +++ b/neon_iris/llm.py @@ -26,31 +26,20 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import fileinput -from os.path import join, dirname +from neon_mq_connector.utils.client_utils import send_mq_request -with open(join(dirname(__file__), "neon_iris", "version.py"), - "r", encoding="utf-8") as v: - for line in v.readlines(): - if line.startswith("__version__"): - if '"' in line: - version = line.split('"')[1] - else: - version = line.split("'")[1] -if "a" not in version: - parts = version.split('.') - parts[-1] = str(int(parts[-1]) + 1) - version = '.'.join(parts) - version = f"{version}a0" -else: - post = version.split("a")[1] - new_post = int(post) + 1 - version = version.replace(f"a{post}", f"a{new_post}") +class LLMConversation: + def __init__(self, llm: str = "chat_gpt"): + self.history = list() + self.queue = f"{llm}_input" -for line in fileinput.input(join(dirname(__file__), "neon_iris", - "version.py"), inplace=True): - if line.startswith("__version__"): - print(f"__version__ = \"{version}\"") - else: - print(line.rstrip('\n')) + def get_response(self, query: str): + resp = send_mq_request("/llm", {'query': query, + 'history': self.history}, self.queue, + timeout=90) + reply = resp.get("response") or "" + if reply: + self.history.append(("user", query)) + self.history.append(("llm", reply)) + return reply diff --git a/neon_iris/res/start_listening.wav b/neon_iris/res/start_listening.wav new file mode 100644 index 0000000..c320218 Binary files /dev/null and b/neon_iris/res/start_listening.wav differ diff --git a/neon_iris/util.py b/neon_iris/util.py new file mode 100644 index 0000000..95f8201 --- /dev/null +++ b/neon_iris/util.py @@ -0,0 +1,132 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + +import json +import yaml + +from os.path import isfile +from ovos_utils.log import LOG + +from neon_utils.file_utils import encode_file_to_base64_string + + +def load_config_file(file_path: str) -> dict: + """ + Load a config file (json or yaml) and return the dict contents + :param file_path: path to config file to load + """ + if not isfile(file_path): + raise FileNotFoundError(f"Requested config file not found: {file_path}") + with open(file_path) as f: + try: + config = json.load(f) + except Exception as e: + LOG.debug(e) + f.seek(0) + config = yaml.safe_load(f) + return config + + +def query_api(query_params: dict, timeout: int = 10) -> dict: + """ + Query an API service on the `/neon_api` vhost. + :param query_params: dict query to send + :param timeout: seconds to wait for a response + :returns: dict MQ response + """ + from neon_mq_connector.utils.client_utils import send_mq_request + response = send_mq_request("/neon_api", query_params, "neon_api_input", + "neon_api_output", timeout) + return response + + +def get_brands_coupons(timeout: int = 5) -> dict: + """ + Get brands/coupons data on the `/neon_coupons` vhost. + :param timeout: seconds to wait for a response + :returns: dict MQ response + """ + from neon_mq_connector.utils.client_utils import send_mq_request + response = send_mq_request("/neon_coupons", {}, "neon_coupons_input", + "neon_coupons_output", timeout) + return response + + +def parse_ccl_script(script_path: str, metadata: dict = None, + timeout: int = 30) -> dict: + """ + Parse a nct script file into an ncs formatted file + :param script_path: path to file to parse + :param metadata: Optional dict metadata to include in output + :param timeout: seconds to wait for a response + :returns: dict MQ response + """ + from neon_mq_connector.utils.client_utils import send_mq_request + with open(script_path, 'r') as f: + text = f.read() + metadata = metadata or {} + response = send_mq_request("/neon_script_parser", {"text": text, + "metadata": metadata}, + "neon_script_parser_input", + "neon_script_parser_output", timeout) + return response + + +def query_neon(msg_type: str, data: dict, timeout: int = 10) -> dict: + """ + Query a Neon Core service on the `/neon_chat_api` + :param msg_type: string message type to emit + :param data: message data to send + :param timeout: seconds to wait for a response + """ + from neon_mq_connector.utils.client_utils import send_mq_request + query = {"msg_type": msg_type, "data": data, "context": {"source": "iris"}} + response = send_mq_request("/neon_chat_api", query, "neon_chat_api_request", + timeout=timeout) + if response: + response["context"]["session"] = \ + set(response["context"].pop("session").keys()) + return response + + +def get_stt(audio_file: str, lang: str = "en-us") -> dict: + data = {"audio_file": audio_file, + "audio_data": encode_file_to_base64_string(audio_file), + "utterances": [""], # TODO: For MQ Connector compat. + "lang": lang} + response = query_neon("neon.get_stt", data, 20) + return response + + +def get_tts(string: str, lang: str = "en-us") -> dict: + data = {"text": string, + "utterance": string, # TODO: For MQ Connector compat. + "utterances": [""], # TODO: For MQ Connector compat. + "speaker": {"name": "Neon", + "language": lang, + "gender": "female"}, # TODO: For neon_audio compat. + "lang": lang} + response = query_neon("neon.get_tts", data) + return response diff --git a/neon_iris/version.py b/neon_iris/version.py index cf52b5b..0d68822 100644 --- a/neon_iris/version.py +++ b/neon_iris/version.py @@ -26,4 +26,4 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "0.0.4" +__version__ = "0.0.5a10" diff --git a/neon_iris/voice_client.py b/neon_iris/voice_client.py new file mode 100644 index 0000000..d8efd90 --- /dev/null +++ b/neon_iris/voice_client.py @@ -0,0 +1,148 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import wave + +from threading import Event, Thread +from time import time +from unittest.mock import Mock +from os.path import join, isdir, dirname +from os import makedirs + +from ovos_plugin_manager.microphone import OVOSMicrophoneFactory +from ovos_plugin_manager.vad import OVOSVADFactory +from ovos_dinkum_listener.voice_loop.voice_loop import DinkumVoiceLoop +from ovos_dinkum_listener.voice_loop.hotwords import HotwordContainer +from ovos_config.config import Configuration +from ovos_utils.messagebus import FakeBus +from ovos_utils.log import LOG +from ovos_utils.xdg_utils import xdg_data_home +from ovos_utils.sound import play_wav +from ovos_bus_client.message import Message +from neon_utils.file_utils import decode_base64_string_to_file +from neon_iris.client import NeonAIClient + + +class MockTransformers(Mock): + def transform(self, chunk): + return chunk, dict() + + +class NeonVoiceClient(NeonAIClient): + def __init__(self, bus=None): + self.config = Configuration() + NeonAIClient.__init__(self, self.config.get("MQ")) + self.bus = bus or FakeBus() + self._mic = OVOSMicrophoneFactory.create(self.config) + self._mic.start() + self._hotwords = HotwordContainer(self.bus) + self._hotwords.load_hotword_engines() + self._vad = OVOSVADFactory.create(self.config) + + self._voice_loop = DinkumVoiceLoop(mic=self._mic, + hotwords=self._hotwords, + stt=Mock(), + fallback_stt=Mock(), + vad=self._vad, + transformers=MockTransformers(), + stt_audio_callback=self.on_stt_audio, + listenword_audio_callback=self.on_hotword_audio) + self._voice_loop.start() + self._voice_thread = None + + self._stt_audio_path = join(xdg_data_home(), "iris", "stt") + self._tts_audio_path = join(xdg_data_home(), "iris", "tts") + if not isdir(self._stt_audio_path): + makedirs(self._stt_audio_path) + if not isdir(self._tts_audio_path): + makedirs(self._tts_audio_path) + + self._listening_sound = join(dirname(__file__), "res", + "start_listening.wav") + + self.run() + + def run(self): + self._voice_thread = Thread(target=self._voice_loop.run, daemon=True) + self._voice_thread.start() + + def on_stt_audio(self, audio_bytes: bytes, context: dict): + LOG.info(f"Got {len(audio_bytes)} bytes of audio") + wav_path = join(self._stt_audio_path, f"{time()}.wav") + with open(wav_path, "wb") as wav_io, \ + wave.open(wav_io, "wb") as wav_file: + wav_file.setframerate(self._mic.sample_rate) + wav_file.setsampwidth(self._mic.sample_width) + wav_file.setnchannels(self._mic.sample_channels) + wav_file.writeframes(audio_bytes) + + self.send_audio(wav_path) + LOG.debug("Sent Audio to MQ") + + def on_hotword_audio(self, audio: bytes, context: dict): + payload = context + msg_type = "recognizer_loop:wakeword" + play_wav(self._listening_sound) + LOG.info(f"Emitting hotword event: {msg_type}") + # emit ww event + self.bus.emit(Message(msg_type, payload, context)) + + def handle_klat_response(self, message: Message): + responses = message.data.get('responses') + for lang, data in responses.items(): + text = data.get('sentence') + LOG.info(text) + file_basename = f"{hash(text)}.wav" + genders = data.get('genders', []) + for gender in genders: + audio_data = data["audio"].get(gender) + audio_file = join(self._tts_audio_path, lang, gender, + file_basename) + try: + decode_base64_string_to_file(audio_data, audio_file) + except FileExistsError: + pass + play_wav(audio_file) + + def handle_complete_intent_failure(self, message: Message): + LOG.info(f"{message.data}") + + def handle_api_response(self, message: Message): + LOG.info(f"{message.data}") + + def handle_error_response(self, message: Message): + LOG.error(f"Got error response: {message.data}") + + def clear_caches(self, message: Message): + pass + + def clear_media(self, message: Message): + pass + + def shutdown(self): + self._voice_loop.stop() + self._voice_thread.join(30) + NeonAIClient.shutdown(self) diff --git a/neon_iris/web_client.py b/neon_iris/web_client.py new file mode 100644 index 0000000..b9a0db2 --- /dev/null +++ b/neon_iris/web_client.py @@ -0,0 +1,341 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from os import makedirs +from os.path import isfile, join, isdir +from time import time +from typing import List, Optional, Dict +from uuid import uuid4 + +import gradio + +from threading import Event +from ovos_bus_client import Message +from ovos_config import Configuration +from ovos_utils import LOG +from ovos_utils.json_helper import merge_dict + +from neon_utils.file_utils import decode_base64_string_to_file +from ovos_utils.xdg_utils import xdg_data_home + +from neon_iris.client import NeonAIClient +import librosa +import soundfile as sf + + +class GradIOClient(NeonAIClient): + def __init__(self, lang: str = None): + config = Configuration() + self.config = config.get('iris') or dict() + NeonAIClient.__init__(self, config.get("MQ")) + self._await_response = Event() + self._response = None + self._current_tts = dict() + self._profiles: Dict[str, dict] = dict() + self._audio_path = join(xdg_data_home(), "iris", "stt") + if not isdir(self._audio_path): + makedirs(self._audio_path) + self.default_lang = lang or self.config.get('default_lang') + self.chat_ui = gradio.Blocks() + LOG.name = "iris" + LOG.init(self.config.get("logs")) + + def get_lang(self, session_id: str): + if session_id and session_id in self._profiles: + return self._profiles[session_id]['speech']['stt_language'] + return self.user_config['speech']['stt_language'] or self.default_lang + + @property + def supported_languages(self) -> List[str]: + """ + Get a list of supported languages from configuration + @returns: list of BCP-47 language codes + """ + return self.config.get('languages') or [self.default_lang] + + def _start_session(self): + sid = uuid4().hex + self._current_tts[sid] = None + self._profiles[sid] = self.user_config + self._profiles[sid]['user']['username'] = sid + return sid + + def update_profile(self, stt_lang: str, tts_lang: str, tts_lang_2: str, + time: int, date: str, uom: str, city: str, state: str, + country: str, first: str, middle: str, last: str, + pref_name: str, email: str, session_id: str): + """ + Callback to handle user settings changes from the web UI + """ + location_dict = dict() + if any((city, state, country)): + from neon_utils.location_utils import get_coordinates, get_timezone + try: + location_dict = {"city": city, "state": state, + "country": country} + lat, lon = get_coordinates(location_dict) + location_dict["lat"] = lat + location_dict["lng"] = lon + location_dict["tz"], location_dict["utc"] = get_timezone(lat, + lon) + LOG.debug(f"Got location update: {location_dict}") + except Exception as e: + LOG.exception(e) + + profile_update = {"speech": {"stt_language": stt_lang, + "tts_language": tts_lang, + "secondary_tts_language": tts_lang_2}, + "units": {"time": time, "date": date, "measure": uom}, + "location": location_dict, + "user": {"first_name": first, "middle_name": middle, + "last_name": last, + "preferred_name": pref_name, "email": email}} + old_profile = self._profiles.get(session_id) or self.user_config + self._profiles[session_id] = merge_dict(old_profile, profile_update) + LOG.info(f"Updated profile for: {session_id}") + return session_id + + def send_audio(self, audio_file: str, lang: str = "en-us", + username: Optional[str] = None, + user_profiles: Optional[list] = None, + context: Optional[dict] = None): + """ + @param audio_file: path to wav audio file to send to speech module + @param lang: language code associated with request + @param username: username associated with request + @param user_profiles: user profiles expecting a response + """ + # TODO: Audio conversion is really slow here. check ovos-stt-http-server + audio_file = self.convert_audio(audio_file) + self._send_audio(audio_file, lang, username, user_profiles, context) + + def convert_audio(self, audio_file: str, target_sr=16000, target_channels=1, + dtype='int16') -> str: + """ + @param audio_file: path to audio file to convert for speech model + @returns: path to converted audio file + """ + # Load the audio file + y, sr = librosa.load(audio_file, sr=None, mono=False) # Load without changing sample rate or channels + + # If the file has more than one channel, mix it down to one channel + if y.ndim > 1 and target_channels == 1: + y = librosa.to_mono(y) + + # Resample the audio to the target sample rate + y_resampled = librosa.resample(y, orig_sr=sr, target_sr=target_sr) + + # Ensure the audio array is in the correct format (int16 for 2-byte samples) + y_resampled = (y_resampled * (2 ** (8 * 2 - 1))).astype(dtype) + + output_path = join(self._audio_path, f"{time()}.wav") + # Save the audio file with the new sample rate and sample width + sf.write(output_path, y_resampled, target_sr, format='WAV', subtype='PCM_16') + LOG.info(f"Converted audio file to {output_path}") + return output_path + + def on_user_input(self, utterance: str, *args, **kwargs) -> str: + """ + Callback to handle textual user input + @param utterance: String utterance submitted by the user + @returns: String response from Neon (or "ERROR") + """ + LOG.debug(f"Input received") + if not self._await_response.wait(30): + LOG.error("Previous response not completed after 30 seconds") + LOG.debug(f"args={args}|kwargs={kwargs}") + self._await_response.clear() + self._response = None + gradio_id = args[2] + lang = self.get_lang(gradio_id) + if utterance: + LOG.info(f"Sending utterance: {utterance} with lang: {lang}") + self.send_utterance(utterance, lang, username=gradio_id, + user_profiles=[self._profiles[gradio_id]], + context={"gradio": {"session": gradio_id}}) + else: + LOG.info(f"Sending audio: {args[1]} with lang: {lang}") + self.send_audio(args[1], lang, username=gradio_id, + user_profiles=[self._profiles[gradio_id]], + context={"gradio": {"session": gradio_id}}) + self._await_response.wait(30) + self._response = self._response or "ERROR" + LOG.info(f"Got response={self._response}") + return self._response + + def play_tts(self, session_id: str): + LOG.info(f"Playing most recent TTS file {self._current_tts}") + return self._current_tts.get(session_id), session_id + + def run(self): + """ + Blocking method to start the web server + """ + self._await_response.set() + title = self.config.get("webui_title", "Neon AI") + description = self.config.get("webui_description", "Chat With Neon") + chatbot = self.config.get("webui_chatbot_label") or description + speech = self.config.get("webui_mic_label") or description + placeholder = self.config.get("webui_input_placeholder", + "Ask me something") + address = self.config.get("server_address") or "0.0.0.0" + port = self.config.get("server_port") or 7860 + + chatbot = gradio.Chatbot(label=chatbot) + textbox = gradio.Textbox(placeholder=placeholder) + + with self.chat_ui as blocks: + client_session = gradio.State(self._start_session()) + client_session.attach_load_event(self._start_session, None) + # Define primary UI + audio_input = gradio.Audio(source="microphone", + type="filepath", + label=speech) + gradio.ChatInterface(self.on_user_input, + chatbot=chatbot, + textbox=textbox, + additional_inputs=[audio_input, client_session], + title=title, + retry_btn=None, + undo_btn=None, ) + tts_audio = gradio.Audio(autoplay=True, visible=True, + label="Neon's Response") + tts_button = gradio.Button("Play TTS") + tts_button.click(self.play_tts, + inputs=[client_session], + outputs=[tts_audio, client_session]) + # Define settings UI + with gradio.Row(): + with gradio.Column(): + lang = self.get_lang(client_session.value) + stt_lang = gradio.Radio(label="Input Language", + choices=self.supported_languages, + value=lang) + tts_lang = gradio.Radio(label="Response Language", + choices=self.supported_languages, + value=lang) + tts_lang_2 = gradio.Radio(label="Second Response Language", + choices=[None] + + self.supported_languages, + value=None) + with gradio.Column(): + time_format = gradio.Radio(label="Time Format", + choices=[12, 24], + value=12) + date_format = gradio.Radio(label="Date Format", + choices=["MDY", "YMD", "DMY", + "YDM"], + value="MDY") + unit_of_measure = gradio.Radio(label="Units of Measure", + choices=["imperial", + "metric"], + value="imperial") + with gradio.Column(): + city = gradio.Textbox(label="City") + state = gradio.Textbox(label="State") + country = gradio.Textbox(label="Country") + with gradio.Column(): + first_name = gradio.Textbox(label="First Name") + middle_name = gradio.Textbox(label="Middle Name") + last_name = gradio.Textbox(label="Last Name") + pref_name = gradio.Textbox(label="Preferred Name") + email_addr = gradio.Textbox(label="Email Address") + # TODO: DoB, pic, about, phone? + submit = gradio.Button("Update User Settings") + submit.click(self.update_profile, + inputs=[stt_lang, tts_lang, tts_lang_2, time_format, + date_format, unit_of_measure, city, state, + country, first_name, middle_name, last_name, + pref_name, email_addr, client_session], + outputs=[client_session]) + blocks.launch(server_name=address, server_port=port) + + def handle_klat_response(self, message: Message): + """ + Handle a valid response from Neon. This includes text and base64-encoded + audio in all requested languages. + @param message: Neon response message + """ + LOG.debug(f"gradio context={message.context['gradio']}") + resp_data = message.data["responses"] + files = [] + sentences = [] + session = message.context['gradio']['session'] + for lang, response in resp_data.items(): + sentences.append(response.get("sentence")) + if response.get("audio"): + for gender, data in response["audio"].items(): + filepath = "/".join([self.audio_cache_dir] + + response[gender].split('/')[-4:]) + # TODO: This only plays the most recent, so it doesn't + # support multiple languages or multi-utterance responses + self._current_tts[session] = filepath + files.append(filepath) + if not isfile(filepath): + decode_base64_string_to_file(data, filepath) + self._response = "\n".join(sentences) + self._await_response.set() + + def handle_complete_intent_failure(self, message: Message): + """ + Handle an intent failure response from Neon. This should not happen and + indicates the Neon service is probably not yet ready. + @param message: Neon intent failure response message + """ + self._response = "ERROR" + self._await_response.set() + + def handle_api_response(self, message: Message): + """ + Catch-all handler for `.response` messages routed to this client that + are not explicitly handled (i.e. get_stt, get_tts) + @param message: Response message to something emitted by this client + """ + LOG.debug(f"Got {message.msg_type}: {message.data}") + + def handle_error_response(self, message: Message): + """ + Handle an error response from the MQ service attached to Neon. This + usually indicates a malformed input. + @param message: Response message indicating reason for failure + """ + LOG.error(f"Error response: {message.data}") + + def clear_caches(self, message: Message): + """ + Handle a request from Neon to clear cached data. + @param message: Message requesting cache deletion. The context of this + message will include the requesting user for user-specific caches + """ + # TODO: remove cached TTS audio responses + + def clear_media(self, message: Message): + """ + Handle a request from Neon to clear local multimedia. This method does + not apply to this client as there is no user-generated media to clear. + @param message: Message requesting media deletion + """ + pass diff --git a/requirements/gradio.txt b/requirements/gradio.txt new file mode 100644 index 0000000..fb2c552 --- /dev/null +++ b/requirements/gradio.txt @@ -0,0 +1,3 @@ +gradio~=3.28 +librosa~=0.9 +soundfile~=0.12 \ No newline at end of file diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 2c64f5f..f8a89d2 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,6 @@ click~=8.0 click-default-group~=1.2 -neon_utils~=1.0 -pyyaml~=5.4 +neon-utils~=1.0 +pyyaml>=5.4,<7.0.0 +neon-mq-connector~=0.7,>=0.7.1a4 +ovos-bus-client~=0.0.3 \ No newline at end of file diff --git a/setup.py b/setup.py index 540638d..99fc4f4 100644 --- a/setup.py +++ b/setup.py @@ -31,18 +31,21 @@ from os import path +BASE_PATH = path.abspath(path.dirname(__file__)) + + def get_requirements(requirements_filename: str): - requirements_file = path.join(path.abspath(path.dirname(__file__)), "requirements", requirements_filename) + requirements_file = path.join(BASE_PATH, "requirements", requirements_filename) with open(requirements_file, 'r', encoding='utf-8') as r: requirements = r.readlines() requirements = [r.strip() for r in requirements if r.strip() and not r.strip().startswith("#")] return requirements -with open("README.md", "r") as f: +with open(path.join(BASE_PATH, "README.md"), "r") as f: long_description = f.read() -with open("neon_iris/version.py", "r", encoding="utf-8") as v: +with open(path.join(BASE_PATH, "neon_iris", "version.py"), "r", encoding="utf-8") as v: for line in v.readlines(): if line.startswith("__version__"): if '"' in line: @@ -68,6 +71,7 @@ def get_requirements(requirements_filename: str): ], python_requires='>=3.6', install_requires=get_requirements("requirements.txt"), + extras_require={"gradio": get_requirements("gradio.txt")}, entry_points={ 'console_scripts': ['iris=neon_iris.cli:neon_iris_cli'] } diff --git a/tests/test_client.py b/tests/test_client.py index bdfec89..183c4c9 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -33,7 +33,7 @@ from neon_utils.mq_utils import NeonMQHandler sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) -from neon_iris.client import NeonAIClient, CLIClient +from neon_iris.client import NeonAIClient _test_config = { "MQ": { @@ -53,6 +53,8 @@ class TestClient(unittest.TestCase): def test_client_create(self): client = NeonAIClient(_test_config) self.assertIsInstance(client.uid, str) + self.assertEqual(client._config, _test_config) + self.assertEqual(client._connection.config, _test_config["MQ"]) self.assertTrue(os.path.isdir(client.audio_cache_dir)) self.assertIsInstance(client.client_name, str) self.assertIsInstance(client.connection, NeonMQHandler)