From ff9f8e898dc11fe38a8731f9240e037b5cae1129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Fri, 8 Jan 2021 21:40:22 +0100 Subject: [PATCH 1/9] Update identity location for VK test Moved from .mycroft to XDG folder --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9888f3fb4fe9..30b69a6e7a76 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -57,7 +57,7 @@ pipeline { sh 'mkdir -p $HOME/core/$BRANCH_ALIAS/allure' sh 'mkdir -p $HOME/core/$BRANCH_ALIAS/mycroft-logs' sh 'docker run \ - -v "$HOME/voight-kampff/identity:/root/.mycroft/identity" \ + -v "$HOME/voight-kampff/identity:/root/.config/mycroft/identity" \ -v "$HOME/core/$BRANCH_ALIAS/allure:/root/allure" \ -v "$HOME/core/$BRANCH_ALIAS/mycroft-logs:/var/log/mycroft" \ --label build=${JOB_NAME} \ From e20443b82458a1185743fe82ced5bf1788b8d3f0 Mon Sep 17 00:00:00 2001 From: Bart Ribbers Date: Fri, 8 May 2020 21:32:33 +0200 Subject: [PATCH 2/9] Use XDG Base directories for settings, cache and runtime data Improve deprecation warning message --- README.md | 8 +-- bin/mycroft-config | 6 +- mycroft/api/__init__.py | 17 ++---- mycroft/client/enclosure/__main__.py | 6 +- mycroft/client/enclosure/mark1/__init__.py | 5 ++ mycroft/client/speech/hotword_factory.py | 34 +++++++++-- mycroft/client/text/text_client.py | 36 ++++++++++- mycroft/configuration/__init__.py | 1 - mycroft/configuration/config.py | 66 +++++++++++++++++---- mycroft/configuration/locations.py | 7 ++- mycroft/configuration/mycroft.conf | 8 +-- mycroft/filesystem/__init__.py | 10 +++- mycroft/messagebus/send_func.py | 7 +-- mycroft/skills/event_scheduler.py | 11 +++- mycroft/skills/skill_updater.py | 6 +- mycroft/util/file_utils.py | 19 ++++-- mycroft/util/log.py | 11 ++-- test/unittests/skills/test_skill_updater.py | 6 +- test/unittests/util/commented.json | 2 +- test/unittests/util/plain.json | 2 +- 20 files changed, 197 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 5e620404687f..d34d08e3474b 100644 --- a/README.md +++ b/README.md @@ -73,20 +73,20 @@ Mycroft is nothing without skills. There are a handful of default skills that a ### Pairing Information Pairing information generated by registering with Home is stored in: -`~/.mycroft/identity/identity2.json` <-- DO NOT SHARE THIS WITH OTHERS! +`~/.config/mycroft/identity/identity2.json` <-- DO NOT SHARE THIS WITH OTHERS! ### Configuration Mycroft's configuration consists of 4 possible locations: - `mycroft-core/mycroft/configuration/mycroft.conf`(Defaults) - [Mycroft Home](https://home.mycroft.ai) (Remote) -- `/etc/mycroft/mycroft.conf`(Machine) -- `$HOME/.mycroft/mycroft.conf`(User) +- `/etc/mycroft/mycroft.conf` (Machine) +- `$XDG_CONFIG_DIR/mycroft/mycroft.conf` (which is by default `$HOME/.config/mycroft/mycroft.conf`) (USER) When the configuration loader starts, it looks in these locations in this order, and loads ALL configurations. Keys that exist in multiple configuration files will be overridden by the last file to contain the value. This process results in a minimal amount being written for a specific device and user, without modifying default distribution files. ### Using Mycroft Without Home -If you do not wish to use the Mycroft Home service, before starting Mycroft for the first time, create `$HOME/.mycroft/mycroft.conf` with the following contents: +If you do not wish to use the Mycroft Home service, before starting Mycroft for the first time, create `$HOME/.config/mycroft/mycroft.conf` with the following contents: ``` { diff --git a/bin/mycroft-config b/bin/mycroft-config index a1271b328dfc..981f2c5d0c65 100755 --- a/bin/mycroft-config +++ b/bin/mycroft-config @@ -95,13 +95,13 @@ function validate_config_file() { return $result } -_conf_file="~/.mycroft/mycroft.conf" +_conf_file="~/.config/mycroft/mycroft.conf" function name_to_path() { case ${1} in "system") _conf_file="/etc/mycroft/mycroft.conf" ;; - "user") _conf_file=$(readlink -f ~/.mycroft/mycroft.conf) ;; + "user") _conf_file=$(readlink -f ~/.config/mycroft/mycroft.conf) ;; "default") _conf_file="$DIR/../mycroft/configuration/mycroft.conf" ;; - "remote") _conf_file="/var/tmp/mycroft_web_cache.json" ;; + "remote") _conf_file="$HOME/.cache/mycroft/web_cache.json" ;; *) echo "ERROR: Unknown name '${1}'." diff --git a/mycroft/api/__init__.py b/mycroft/api/__init__.py index 2e74268bebc1..9da1ca0b5803 100644 --- a/mycroft/api/__init__.py +++ b/mycroft/api/__init__.py @@ -20,8 +20,6 @@ from requests import HTTPError, RequestException from mycroft.configuration import Configuration -from mycroft.configuration.config import DEFAULT_CONFIG, SYSTEM_CONFIG, \ - USER_CONFIG from mycroft.identity import IdentityManager, identity_lock from mycroft.version import VersionManager from mycroft.util import get_arch, connected, LOG @@ -49,12 +47,9 @@ class Api: def __init__(self, path): self.path = path - # Load the config, skipping the REMOTE_CONFIG since we are + # Load the config, skipping the remote config since we are # getting the info needed to get to it! - config = Configuration.get([DEFAULT_CONFIG, - SYSTEM_CONFIG, - USER_CONFIG], - cache=False) + config = Configuration.get(cache=False, remote=False) config_server = config.get("server") self.url = config_server.get("url") self.version = config_server.get("version") @@ -239,9 +234,7 @@ def activate(self, state, token): platform_build = "" # load just the local configs to get platform info - config = Configuration.get([SYSTEM_CONFIG, - USER_CONFIG], - cache=False) + config = Configuration.get(cache=False, remote=False) if "enclosure" in config: platform = config.get("enclosure").get("platform", "unknown") platform_build = config.get("enclosure").get("platform_build", "") @@ -263,9 +256,7 @@ def update_version(self): platform_build = "" # load just the local configs to get platform info - config = Configuration.get([SYSTEM_CONFIG, - USER_CONFIG], - cache=False) + config = Configuration.get(cache=False, remote=False) if "enclosure" in config: platform = config.get("enclosure").get("platform", "unknown") platform_build = config.get("enclosure").get("platform_build", "") diff --git a/mycroft/client/enclosure/__main__.py b/mycroft/client/enclosure/__main__.py index f7bdb5dde9b6..0ce5b361c12b 100644 --- a/mycroft/client/enclosure/__main__.py +++ b/mycroft/client/enclosure/__main__.py @@ -17,7 +17,7 @@ This provides any "enclosure" specific functionality, for example GUI or control over the Mark-1 Faceplate. """ -from mycroft.configuration import LocalConf, SYSTEM_CONFIG +from mycroft.configuration import Configuration from mycroft.util.log import LOG from mycroft.util import wait_for_exit_signal, reset_sigint_handler @@ -70,8 +70,8 @@ def main(ready_hook=on_ready, error_hook=on_error, stopping_hook=on_stopping): only the GUI bus will be started. """ # Read the system configuration - system_config = LocalConf(SYSTEM_CONFIG) - platform = system_config.get("enclosure", {}).get("platform") + config = Configuration.get(remote=False) + platform = config.get("enclosure", {}).get("platform") enclosure = create_enclosure(platform) if enclosure: diff --git a/mycroft/client/enclosure/mark1/__init__.py b/mycroft/client/enclosure/mark1/__init__.py index 2741aa0eed1c..2095945705d0 100644 --- a/mycroft/client/enclosure/mark1/__init__.py +++ b/mycroft/client/enclosure/mark1/__init__.py @@ -15,8 +15,10 @@ import subprocess import time import sys +import os from alsaaudio import Mixer from threading import Thread, Timer +from xdg import BaseDirectory import serial @@ -164,6 +166,9 @@ def process(self, data): if "unit.factory-reset" in data: self.bus.emit(Message("speak", { 'utterance': mycroft.dialog.get("reset to factory defaults")})) + subprocess.call( + 'rm ~/.config/mycroft/identity/identity2.json', + shell=True) subprocess.call( 'rm ~/.mycroft/identity/identity2.json', shell=True) diff --git a/mycroft/client/speech/hotword_factory.py b/mycroft/client/speech/hotword_factory.py index 91ca8947ac66..72a5e0ee3775 100644 --- a/mycroft/client/speech/hotword_factory.py +++ b/mycroft/client/speech/hotword_factory.py @@ -26,13 +26,15 @@ from threading import Timer, Thread from time import time, sleep from urllib.error import HTTPError +from xdg import BaseDirectory from petact import install_package import requests -from mycroft.configuration import Configuration, LocalConf, USER_CONFIG -from mycroft.util.monotonic_event import MonotonicEvent +from mycroft.configuration import Configuration, LocalConf +from mycroft.configuration.locations import OLD_USER_CONFIG from mycroft.util.log import LOG +from mycroft.util.monotonic_event import MonotonicEvent from mycroft.util.plugins import load_plugin RECOGNIZER_DIR = join(abspath(dirname(__file__)), "recognizer") @@ -193,9 +195,30 @@ def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"): from precise_runner import ( PreciseRunner, PreciseEngine, ReadWriteStream ) - local_conf = LocalConf(USER_CONFIG) + + # We need to save to a writeable location, but the key we need + # might be stored in a different, unwriteable, location + # Make sure we pick the key we need from wherever it's located, + # but save to a writeable location only + local_conf = LocalConf(join(BaseDirectory.save_config_path('mycroft'), + 'mycroft.conf')) + + for dir in BaseDirectory.load_config_paths('mycroft'): + conf = LocalConf(join(dir, 'mycroft.conf')) + # If the current config contains the precise key use it, + # otherwise continue to the next file + if conf.get('precise', None) is not None: + local_conf['precise'] = conf.get('precise', None) + break + + # If the key is not found yet, it might still exist on the old + # (deprecated) location + if local_conf.get('precise', None) is None: + local_conf = LocalConf(OLD_USER_CONFIG) + if not local_conf.get('precise', {}).get('use_precise', True): raise PreciseUnavailable + if (local_conf.get('precise', {}).get('dist_url') == 'http://bootstrap.mycroft.ai/artifacts/static/daily/'): del local_conf['precise']['dist_url'] @@ -253,7 +276,10 @@ def update_precise(self, precise_config): @property def folder(self): - return join(expanduser('~'), '.mycroft', 'precise') + old_path = join(expanduser('~'), '.mycroft', 'precise') + if os.path.isdir(old_path): + return old_path + return join(BaseDirectory.save_data_path('mycroft', 'precise')) @property def install_destination(self): diff --git a/mycroft/client/text/text_client.py b/mycroft/client/text/text_client.py index 5d382cb52269..8b2ca3d3ea4d 100644 --- a/mycroft/client/text/text_client.py +++ b/mycroft/client/text/text_client.py @@ -15,6 +15,7 @@ import sys import io from math import ceil +from xdg import BaseDirectory from .gui_server import start_qml_gui @@ -142,7 +143,7 @@ def handleNonAscii(text): ############################################################################## # Settings -config_file = os.path.join(os.path.expanduser("~"), ".mycroft_cli.conf") +filename = "mycroft_cli.conf" def load_mycroft_config(bus): @@ -171,6 +172,35 @@ def load_settings(): global max_log_lines global show_meter + config_file = None + + # Old location + path = os.path.join(os.path.expanduser("~"), ".mycroft_cli.conf") + if os.path.isfile(path): + LOG.warning(" ===============================================") + LOG.warning(" == DEPRECATION WARNING ==") + LOG.warning(" ===============================================") + LOG.warning(" You still have a config file at " + + path) + LOG.warning(" Note that this location is deprecated and will" + + " not be used in the future") + LOG.warning(" Please move it to " + + os.path.join(BaseDirectory.save_config_path('mycroft'), + filename)) + config_file = path + + # Check XDG_CONFIG_DIR + if config_file is None: + for dir in BaseDirectory.load_config_paths('mycroft'): + file = os.path.join(dir, filename) + if os.path.isfile(file): + config_file = file + break + + # Check /etc/mycroft + if config_file is None: + config_file = os.path.join("/etc/mycroft", filename) + try: with io.open(config_file, 'r') as f: config = json.load(f) @@ -196,6 +226,10 @@ def save_settings(): config["show_last_key"] = show_last_key config["max_log_lines"] = max_log_lines config["show_meter"] = show_meter + + config_file = os.path.join( + BaseDirectory.save_config_path("mycroft"), filename) + with io.open(config_file, 'w') as f: f.write(str(json.dumps(config, ensure_ascii=False))) diff --git a/mycroft/configuration/__init__.py b/mycroft/configuration/__init__.py index c7981df30958..8f7d1a4ed607 100644 --- a/mycroft/configuration/__init__.py +++ b/mycroft/configuration/__init__.py @@ -14,4 +14,3 @@ # from .config import Configuration, LocalConf, RemoteConf from .locale import set_default_lf_lang -from .locations import SYSTEM_CONFIG, USER_CONFIG diff --git a/mycroft/configuration/config.py b/mycroft/configuration/config.py index 3ff92912f025..029c19bdb11d 100644 --- a/mycroft/configuration/config.py +++ b/mycroft/configuration/config.py @@ -17,14 +17,15 @@ import re import json import inflection -from os.path import exists, isfile +from os.path import exists, isfile, join, expanduser, dirname from requests import RequestException +from xdg import BaseDirectory from mycroft.util.json_helper import load_commented_json, merge_dict from mycroft.util.log import LOG -from .locations import (DEFAULT_CONFIG, SYSTEM_CONFIG, USER_CONFIG, - WEB_CONFIG_CACHE) +from .locations import DEFAULT_CONFIG, OLD_USER_CONFIG, USER_CONFIG +from .locations import SYSTEM_CONFIG def is_remote_list(values): @@ -127,7 +128,8 @@ class RemoteConf(LocalConf): def __init__(self, cache=None): super(RemoteConf, self).__init__(None) - cache = cache or WEB_CONFIG_CACHE + cache = cache or join(BaseDirectory.save_cache_path('mycroft'), + 'web_cache.json') from mycroft.api import is_paired if not is_paired(): self.load_local(cache) @@ -174,7 +176,7 @@ class Configuration: __patch = {} # Patch config that skills can update to override config @staticmethod - def get(configs=None, cache=True): + def get(configs=None, cache=True, remote=True): """Get configuration Returns cached instance if available otherwise builds a new @@ -183,6 +185,9 @@ def get(configs=None, cache=True): Args: configs (list): List of configuration dicts cache (boolean): True if the result should be cached + remote (boolean): False if the Mycroft Home settings shouldn't + be loaded + Returns: (dict) configuration dictionary. @@ -190,23 +195,62 @@ def get(configs=None, cache=True): if Configuration.__config: return Configuration.__config else: - return Configuration.load_config_stack(configs, cache) + return Configuration.load_config_stack(configs, cache, remote) @staticmethod - def load_config_stack(configs=None, cache=False): + def load_config_stack(configs=None, cache=False, remote=True): """Load a stack of config dicts into a single dict Args: configs (list): list of dicts to load cache (boolean): True if result should be cached - + remote (boolean): False if the Mycroft Home settings shouldn't + be loaded Returns: (dict) merged dict of all configuration files """ if not configs: - configs = [LocalConf(DEFAULT_CONFIG), RemoteConf(), - LocalConf(SYSTEM_CONFIG), LocalConf(USER_CONFIG), - Configuration.__patch] + configs = configs or [] + + # First use the patched config + configs.append(Configuration.__patch) + + # Then use XDG config + # This includes both the user config and + # /etc/xdg/mycroft/mycroft.conf + for dir in BaseDirectory.load_config_paths('mycroft'): + configs.append(LocalConf(join(dir, 'mycroft.conf'))) + + # Then check the old user config + if isfile(OLD_USER_CONFIG): + LOG.warning(" ===============================================") + LOG.warning(" == DEPRECATION WARNING ==") + LOG.warning(" ===============================================") + LOG.warning(" You still have a config file at " + + OLD_USER_CONFIG) + LOG.warning(" Note that this location is deprecated and will" + + " not be used in the future") + LOG.warning(" Please move it to " + + BaseDirectory.save_config_path('mycroft')) + configs.append(LocalConf(OLD_USER_CONFIG)) + + # Then check the XDG user config + if isfile(USER_CONFIG): + configs.append(LocalConf(USER_CONFIG)) + + # Then use remote config + if remote: + configs.append(RemoteConf()) + + # Then use the system config (/etc/mycroft/mycroft.conf) + configs.append(LocalConf(SYSTEM_CONFIG)) + + # Then use the config that comes with the package + configs.append(LocalConf(DEFAULT_CONFIG)) + + # Make sure we reverse the array, as merge_dict will put every new + # file on top of the previous one + configs = reversed(configs) else: # Handle strings in stack for index, item in enumerate(configs): diff --git a/mycroft/configuration/locations.py b/mycroft/configuration/locations.py index 28f57a2894d7..77177736b244 100644 --- a/mycroft/configuration/locations.py +++ b/mycroft/configuration/locations.py @@ -13,11 +13,16 @@ # limitations under the License. import os from os.path import join, dirname, expanduser, exists +from xdg import BaseDirectory DEFAULT_CONFIG = join(dirname(__file__), 'mycroft.conf') SYSTEM_CONFIG = os.environ.get('MYCROFT_SYSTEM_CONFIG', '/etc/mycroft/mycroft.conf') -USER_CONFIG = join(expanduser('~'), '.mycroft/mycroft.conf') +# Make sure we support the old location still +# Deprecated and will be removed eventually +OLD_USER_CONFIG = join(expanduser('~'), '.mycroft/mycroft.conf') +USER_CONFIG = join(BaseDirectory.save_config_path('mycroft'), + 'mycroft.conf') REMOTE_CONFIG = "mycroft.ai" WEB_CONFIG_CACHE = os.environ.get('MYCROFT_WEB_CACHE', '/var/tmp/mycroft_web_cache.json') diff --git a/mycroft/configuration/mycroft.conf b/mycroft/configuration/mycroft.conf index 1b63c4654230..b6922259aa43 100644 --- a/mycroft/configuration/mycroft.conf +++ b/mycroft/configuration/mycroft.conf @@ -5,7 +5,7 @@ // overridden at the REMOTE level (set by the user via // https://home.mycroft.ai), at the SYSTEM level (typically in the file // '/etc/mycroft/mycroft.conf'), or at the USER level (typically in the - // file '~/.mycroft/mycroft.conf'). + // file '~/.config/mycroft/mycroft.conf'). // // The load order of settings is: // DEFAULT @@ -107,7 +107,7 @@ }, "upload_skill_manifest": true, // Directory to look for user skills - "directory": "~/.mycroft/skills", + "directory": "~/.local/share/mycroft/skills", // Enable auto update by msm "auto_update": true, // blacklisted skills to not load @@ -221,7 +221,7 @@ "threshold": 1e-90, "lang": "en-us" // Specify custom model via: - // "local_model_file": "~/.mycroft/precise/models/something.pb" + // "local_model_file": "~/.local/share/mycroft/precise/models/something.pb" // Precise options: // "sensitivity": 0.5, // Higher = more sensitive // "trigger_level": 3 // Higher = more delay & less sensitive @@ -324,7 +324,7 @@ }, "padatious": { - "intent_cache": "~/.mycroft/intent_cache", + "intent_cache": "~/.local/share/mycroft/intent_cache", "train_delay": 4, "single_thread": false }, diff --git a/mycroft/filesystem/__init__.py b/mycroft/filesystem/__init__.py index df6776f0e9b2..c0e5c4ae1cdc 100644 --- a/mycroft/filesystem/__init__.py +++ b/mycroft/filesystem/__init__.py @@ -13,7 +13,9 @@ # limitations under the License. # import os +import shutil from os.path import join, expanduser, isdir +from xdg import BaseDirectory class FileSystemAccess: @@ -31,7 +33,13 @@ def __init__(self, path): def __init_path(path): if not isinstance(path, str) or len(path) == 0: raise ValueError("path must be initialized as a non empty string") - path = join(expanduser('~'), '.mycroft', path) + + old_path = join(expanduser('~'), '.mycroft', path) + path = join(BaseDirectory.save_config_path('mycroft'), path) + + # Migrate from the old location if it still exists + if isdir(old_path): + shutil.move(old_path, path) if not isdir(path): os.makedirs(path) diff --git a/mycroft/messagebus/send_func.py b/mycroft/messagebus/send_func.py index 403db9f45c1d..a11a9b0a591d 100644 --- a/mycroft/messagebus/send_func.py +++ b/mycroft/messagebus/send_func.py @@ -15,8 +15,6 @@ from websocket import create_connection from mycroft.configuration import Configuration -from mycroft.configuration.locations import (DEFAULT_CONFIG, SYSTEM_CONFIG, - USER_CONFIG) from mycroft.messagebus.client import MessageBusClient from mycroft.messagebus.message import Message @@ -32,10 +30,7 @@ def send(message_to_send, data_to_send=None): data_to_send = data_to_send or {} # Calculate the standard Mycroft messagebus websocket address - config = Configuration.get([DEFAULT_CONFIG, - SYSTEM_CONFIG, - USER_CONFIG], - cache=False) + config = Configuration.get(cache=False, remote=False) config = config.get("websocket") url = MessageBusClient.build_url( config.get("host"), diff --git a/mycroft/skills/event_scheduler.py b/mycroft/skills/event_scheduler.py index cc91a76c43ad..8d1582799fed 100644 --- a/mycroft/skills/event_scheduler.py +++ b/mycroft/skills/event_scheduler.py @@ -16,10 +16,12 @@ times. """ import json +import shutil import time from datetime import datetime, timedelta from threading import Thread, Lock from os.path import isfile, join, expanduser +from xdg import BaseDirectory from mycroft.configuration import Configuration from mycroft.messagebus.message import Message @@ -54,14 +56,19 @@ class EventScheduler(Thread): """ def __init__(self, bus, schedule_file='schedule.json'): super().__init__() - data_dir = expanduser(Configuration.get()['data_dir']) self.events = {} self.event_lock = Lock() self.bus = bus self.is_running = True - self.schedule_file = join(data_dir, schedule_file) + old_schedule_path = join(expanduser(Configuration.get()['data_dir']), + schedule_file) + new_schedule_path = join( + BaseDirectory.load_first_config('mycroft'), schedule_file) + if isfile(old_schedule_path): + shutil.move(old_schedule_path, new_schedule_path) + self.schedule_file = new_schedule_path if self.schedule_file: self.load() diff --git a/mycroft/skills/skill_updater.py b/mycroft/skills/skill_updater.py index b856739a1cb1..013411ce50eb 100644 --- a/mycroft/skills/skill_updater.py +++ b/mycroft/skills/skill_updater.py @@ -17,6 +17,7 @@ import sys from datetime import datetime from time import time +from xdg import BaseDirectory from msm import MsmException @@ -97,9 +98,8 @@ def installed_skills_file_path(self): '.mycroft-skills' ) else: - self._installed_skills_file_path = os.path.expanduser( - '~/.mycroft/.mycroft-skills' - ) + self._installed_skills_file_path = os.path.join( + BaseDirectory.save_data_path('mycroft'), '.mycroft-skills') return self._installed_skills_file_path diff --git a/mycroft/util/file_utils.py b/mycroft/util/file_utils.py index 279c7d16a3e6..cfe4589d40ca 100644 --- a/mycroft/util/file_utils.py +++ b/mycroft/util/file_utils.py @@ -22,6 +22,7 @@ import psutil from stat import S_ISREG, ST_MTIME, ST_MODE, ST_SIZE import tempfile +from xdg import BaseDirectory import mycroft.configuration from .log import LOG @@ -33,10 +34,10 @@ def resolve_resource_file(res_name): Resource names are in the form: 'filename.ext' or 'path/filename.ext' - The system wil look for ~/.mycroft/res_name first, and - if not found will look at /opt/mycroft/res_name, - then finally it will look for res_name in the 'mycroft/res' - folder of the source code package. + The system wil look for $XDG_DATA_DIRS/mycroft/res_name first + (defaults to ~/.local/share/mycroft/res_name), and if not found will + look at /opt/mycroft/res_name, then finally it will look for res_name + in the 'mycroft/res' folder of the source code package. Example: With mycroft running as the user 'bob', if you called @@ -60,8 +61,14 @@ def resolve_resource_file(res_name): if os.path.isfile(res_name): return res_name - # Now look for ~/.mycroft/res_name (in user folder) - filename = os.path.expanduser("~/.mycroft/" + res_name) + # Now look for XDG_DATA_DIRS + for dir in BaseDirectory.load_data_paths('mycroft'): + filename = os.path.join(dir, res_name) + if os.path.isfile(filename): + return filename + + # Now look in the old user location + filename = os.path.join(os.path.expanduser('~'), '.mycroft', res_name) if os.path.isfile(filename): return filename diff --git a/mycroft/util/log.py b/mycroft/util/log.py index 0ea391f8f79c..788f6df1c37c 100644 --- a/mycroft/util/log.py +++ b/mycroft/util/log.py @@ -21,7 +21,7 @@ for use. The default log level of the logger created here can ONLY be set in -/etc/mycroft/mycroft.conf or ~/.mycroft/mycroft.conf +/etc/mycroft/mycroft.conf or ~/.config/mycroft/mycroft.conf The default log level can also be programatically be changed by setting the LOG.level parameter. @@ -31,10 +31,10 @@ import logging import sys -from os.path import isfile +from os.path import isfile, join +from xdg import BaseDirectory from mycroft.util.json_helper import load_commented_json, merge_dict -from mycroft.configuration.locations import SYSTEM_CONFIG, USER_CONFIG def getLogger(name="MYCROFT"): @@ -84,7 +84,10 @@ def init(cls): # Check configs manually, the Mycroft configuration system can't be # used since it uses the LOG system and would cause horrible cyclic # dependencies. - confs = [SYSTEM_CONFIG, USER_CONFIG] + confs = [] + for dir in BaseDirectory.load_config_paths('mycroft'): + confs.append(join(dir, 'mycroft.conf')) + confs.append('/etc/mycroft/mycroft.conf') config = {} for conf in confs: try: diff --git a/test/unittests/skills/test_skill_updater.py b/test/unittests/skills/test_skill_updater.py index 38d4083d7f2e..d7cc72e3e675 100644 --- a/test/unittests/skills/test_skill_updater.py +++ b/test/unittests/skills/test_skill_updater.py @@ -13,8 +13,9 @@ # limitations under the License. # """Unit tests for the SkillUpdater class.""" -from os import path +import os from time import sleep +from xdg import BaseDirectory from unittest.mock import Mock, patch, PropertyMock from mycroft.skills.skill_updater import SkillUpdater @@ -143,7 +144,8 @@ def test_installed_skills_path_not_virtual_env(self): os_patch.return_value = False updater = SkillUpdater(self.message_bus_mock) self.assertEqual( - path.expanduser('~/.mycroft/.mycroft-skills'), + os.path.join(BaseDirectory.save_data_path('mycroft'), + '.mycroft-skills'), updater.installed_skills_file_path ) diff --git a/test/unittests/util/commented.json b/test/unittests/util/commented.json index 87c7d3e2c2db..00a581f6df47 100644 --- a/test/unittests/util/commented.json +++ b/test/unittests/util/commented.json @@ -54,7 +54,7 @@ } }, "skills": { - "directory": "~/.mycroft/skills" + "directory": "~/.local/share/mycroft/skills" }, "server": { "url": "https://api.mycroft.ai", diff --git a/test/unittests/util/plain.json b/test/unittests/util/plain.json index f8c474139ce8..eec84631d886 100644 --- a/test/unittests/util/plain.json +++ b/test/unittests/util/plain.json @@ -36,7 +36,7 @@ } }, "skills": { - "directory": "~/.mycroft/skills" + "directory": "~/.local/share/mycroft/skills" }, "server": { "url": "https://api.mycroft.ai", From faf101bfcd0ab5784a88e9a62bb633ff8b72b3ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Sun, 31 Jan 2021 08:56:10 +0100 Subject: [PATCH 3/9] Increase Jenkins timeout --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 30b69a6e7a76..4f67ec06813f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,7 +52,7 @@ pipeline { --label build=${JOB_NAME} \ -t voight-kampff-mark-1:${BRANCH_ALIAS} .' echo 'Running Mark I Voight-Kampff Test Suite' - timeout(time: 60, unit: 'MINUTES') + timeout(time: 90, unit: 'MINUTES') { sh 'mkdir -p $HOME/core/$BRANCH_ALIAS/allure' sh 'mkdir -p $HOME/core/$BRANCH_ALIAS/mycroft-logs' From 28017993c5fae3fedb0d8b8bd08d7db3de76faff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Sun, 21 Feb 2021 18:08:46 +0100 Subject: [PATCH 4/9] Restore system locations to mycroft.configuration Fixes issue with Timer skill --- mycroft/configuration/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycroft/configuration/__init__.py b/mycroft/configuration/__init__.py index 8f7d1a4ed607..c7981df30958 100644 --- a/mycroft/configuration/__init__.py +++ b/mycroft/configuration/__init__.py @@ -14,3 +14,4 @@ # from .config import Configuration, LocalConf, RemoteConf from .locale import set_default_lf_lang +from .locations import SYSTEM_CONFIG, USER_CONFIG From 8e69d4616d7052e512ff5f90fe72fe62c62f3361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Wed, 5 May 2021 20:32:19 +0200 Subject: [PATCH 5/9] Correct resolution order for resolving log configs --- mycroft/util/log.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mycroft/util/log.py b/mycroft/util/log.py index 788f6df1c37c..460b349d889e 100644 --- a/mycroft/util/log.py +++ b/mycroft/util/log.py @@ -88,6 +88,7 @@ def init(cls): for dir in BaseDirectory.load_config_paths('mycroft'): confs.append(join(dir, 'mycroft.conf')) confs.append('/etc/mycroft/mycroft.conf') + confs = reversed(confs) config = {} for conf in confs: try: From 9029dc1f41d89d86c629ba45c750330b39755b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Mon, 5 Jul 2021 07:22:52 +0200 Subject: [PATCH 6/9] Slight cleanup - Make XDG usage more visible by using the xdg module prefix - fix overloaded python keywords - remove unused imports --- mycroft/client/enclosure/mark1/__init__.py | 5 +---- mycroft/client/speech/hotword_factory.py | 13 +++++++------ mycroft/client/text/text_client.py | 14 +++++++------- mycroft/configuration/config.py | 12 ++++++------ mycroft/configuration/locations.py | 4 ++-- mycroft/filesystem/__init__.py | 4 ++-- mycroft/skills/event_scheduler.py | 5 +++-- mycroft/skills/skill_updater.py | 5 +++-- mycroft/util/file_utils.py | 7 ++++--- mycroft/util/log.py | 6 +++--- 10 files changed, 38 insertions(+), 37 deletions(-) diff --git a/mycroft/client/enclosure/mark1/__init__.py b/mycroft/client/enclosure/mark1/__init__.py index 2095945705d0..5758ecdb694b 100644 --- a/mycroft/client/enclosure/mark1/__init__.py +++ b/mycroft/client/enclosure/mark1/__init__.py @@ -14,11 +14,8 @@ # import subprocess import time -import sys -import os from alsaaudio import Mixer from threading import Thread, Timer -from xdg import BaseDirectory import serial @@ -31,7 +28,7 @@ from mycroft.client.enclosure.mark1.mouth import EnclosureMouth from mycroft.enclosure.display_manager import \ init_display_manager_bus_connection -from mycroft.configuration import Configuration, LocalConf, USER_CONFIG +from mycroft.configuration import LocalConf, USER_CONFIG from mycroft.messagebus.message import Message from mycroft.util import play_wav, create_signal, connected, check_for_signal from mycroft.util.audio_test import record diff --git a/mycroft/client/speech/hotword_factory.py b/mycroft/client/speech/hotword_factory.py index 72a5e0ee3775..c6fb57a9bae6 100644 --- a/mycroft/client/speech/hotword_factory.py +++ b/mycroft/client/speech/hotword_factory.py @@ -26,7 +26,7 @@ from threading import Timer, Thread from time import time, sleep from urllib.error import HTTPError -from xdg import BaseDirectory +import xdg.BaseDirectory from petact import install_package import requests @@ -200,11 +200,12 @@ def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"): # might be stored in a different, unwriteable, location # Make sure we pick the key we need from wherever it's located, # but save to a writeable location only - local_conf = LocalConf(join(BaseDirectory.save_config_path('mycroft'), - 'mycroft.conf')) + local_conf = LocalConf( + join(xdg.BaseDirectory.save_config_path('mycroft'), 'mycroft.conf') + ) - for dir in BaseDirectory.load_config_paths('mycroft'): - conf = LocalConf(join(dir, 'mycroft.conf')) + for conf_dir in xdg.BaseDirectory.load_config_paths('mycroft'): + conf = LocalConf(join(conf_dir, 'mycroft.conf')) # If the current config contains the precise key use it, # otherwise continue to the next file if conf.get('precise', None) is not None: @@ -279,7 +280,7 @@ def folder(self): old_path = join(expanduser('~'), '.mycroft', 'precise') if os.path.isdir(old_path): return old_path - return join(BaseDirectory.save_data_path('mycroft', 'precise')) + return join(xdg.BaseDirectory.save_data_path('mycroft', 'precise')) @property def install_destination(self): diff --git a/mycroft/client/text/text_client.py b/mycroft/client/text/text_client.py index 8b2ca3d3ea4d..acb097d96a38 100644 --- a/mycroft/client/text/text_client.py +++ b/mycroft/client/text/text_client.py @@ -15,7 +15,7 @@ import sys import io from math import ceil -from xdg import BaseDirectory +import xdg.BaseDirectory from .gui_server import start_qml_gui @@ -185,16 +185,16 @@ def load_settings(): LOG.warning(" Note that this location is deprecated and will" + " not be used in the future") LOG.warning(" Please move it to " + - os.path.join(BaseDirectory.save_config_path('mycroft'), + os.path.join(xdg.BaseDirectory.save_config_path('mycroft'), filename)) config_file = path # Check XDG_CONFIG_DIR if config_file is None: - for dir in BaseDirectory.load_config_paths('mycroft'): - file = os.path.join(dir, filename) - if os.path.isfile(file): - config_file = file + for conf_dir in xdg.BaseDirectory.load_config_paths('mycroft'): + xdg_file = os.path.join(conf_dir, filename) + if os.path.isfile(xdg_file): + config_file = xdg_file break # Check /etc/mycroft @@ -228,7 +228,7 @@ def save_settings(): config["show_meter"] = show_meter config_file = os.path.join( - BaseDirectory.save_config_path("mycroft"), filename) + xdg.BaseDirectory.save_config_path("mycroft"), filename) with io.open(config_file, 'w') as f: f.write(str(json.dumps(config, ensure_ascii=False))) diff --git a/mycroft/configuration/config.py b/mycroft/configuration/config.py index 029c19bdb11d..9919ff3945ad 100644 --- a/mycroft/configuration/config.py +++ b/mycroft/configuration/config.py @@ -17,9 +17,9 @@ import re import json import inflection -from os.path import exists, isfile, join, expanduser, dirname +from os.path import exists, isfile, join from requests import RequestException -from xdg import BaseDirectory +import xdg.BaseDirectory from mycroft.util.json_helper import load_commented_json, merge_dict from mycroft.util.log import LOG @@ -128,7 +128,7 @@ class RemoteConf(LocalConf): def __init__(self, cache=None): super(RemoteConf, self).__init__(None) - cache = cache or join(BaseDirectory.save_cache_path('mycroft'), + cache = cache or join(xdg.BaseDirectory.save_cache_path('mycroft'), 'web_cache.json') from mycroft.api import is_paired if not is_paired(): @@ -218,8 +218,8 @@ def load_config_stack(configs=None, cache=False, remote=True): # Then use XDG config # This includes both the user config and # /etc/xdg/mycroft/mycroft.conf - for dir in BaseDirectory.load_config_paths('mycroft'): - configs.append(LocalConf(join(dir, 'mycroft.conf'))) + for conf_dir in xdg.BaseDirectory.load_config_paths('mycroft'): + configs.append(LocalConf(join(conf_dir, 'mycroft.conf'))) # Then check the old user config if isfile(OLD_USER_CONFIG): @@ -231,7 +231,7 @@ def load_config_stack(configs=None, cache=False, remote=True): LOG.warning(" Note that this location is deprecated and will" + " not be used in the future") LOG.warning(" Please move it to " + - BaseDirectory.save_config_path('mycroft')) + xdg.BaseDirectory.save_config_path('mycroft')) configs.append(LocalConf(OLD_USER_CONFIG)) # Then check the XDG user config diff --git a/mycroft/configuration/locations.py b/mycroft/configuration/locations.py index 77177736b244..b17e8b50864f 100644 --- a/mycroft/configuration/locations.py +++ b/mycroft/configuration/locations.py @@ -13,7 +13,7 @@ # limitations under the License. import os from os.path import join, dirname, expanduser, exists -from xdg import BaseDirectory +import xdg.BaseDirectory DEFAULT_CONFIG = join(dirname(__file__), 'mycroft.conf') SYSTEM_CONFIG = os.environ.get('MYCROFT_SYSTEM_CONFIG', @@ -21,7 +21,7 @@ # Make sure we support the old location still # Deprecated and will be removed eventually OLD_USER_CONFIG = join(expanduser('~'), '.mycroft/mycroft.conf') -USER_CONFIG = join(BaseDirectory.save_config_path('mycroft'), +USER_CONFIG = join(xdg.BaseDirectory.save_config_path('mycroft'), 'mycroft.conf') REMOTE_CONFIG = "mycroft.ai" WEB_CONFIG_CACHE = os.environ.get('MYCROFT_WEB_CACHE', diff --git a/mycroft/filesystem/__init__.py b/mycroft/filesystem/__init__.py index c0e5c4ae1cdc..27f84509ded6 100644 --- a/mycroft/filesystem/__init__.py +++ b/mycroft/filesystem/__init__.py @@ -15,7 +15,7 @@ import os import shutil from os.path import join, expanduser, isdir -from xdg import BaseDirectory +import xdg.BaseDirectory class FileSystemAccess: @@ -35,7 +35,7 @@ def __init_path(path): raise ValueError("path must be initialized as a non empty string") old_path = join(expanduser('~'), '.mycroft', path) - path = join(BaseDirectory.save_config_path('mycroft'), path) + path = join(xdg.BaseDirectory.save_config_path('mycroft'), path) # Migrate from the old location if it still exists if isdir(old_path): diff --git a/mycroft/skills/event_scheduler.py b/mycroft/skills/event_scheduler.py index 8d1582799fed..79f29de24ead 100644 --- a/mycroft/skills/event_scheduler.py +++ b/mycroft/skills/event_scheduler.py @@ -21,7 +21,7 @@ from datetime import datetime, timedelta from threading import Thread, Lock from os.path import isfile, join, expanduser -from xdg import BaseDirectory +import xdg.BaseDirectory from mycroft.configuration import Configuration from mycroft.messagebus.message import Message @@ -65,7 +65,8 @@ def __init__(self, bus, schedule_file='schedule.json'): old_schedule_path = join(expanduser(Configuration.get()['data_dir']), schedule_file) new_schedule_path = join( - BaseDirectory.load_first_config('mycroft'), schedule_file) + xdg.BaseDirectory.load_first_config('mycroft'), schedule_file + ) if isfile(old_schedule_path): shutil.move(old_schedule_path, new_schedule_path) self.schedule_file = new_schedule_path diff --git a/mycroft/skills/skill_updater.py b/mycroft/skills/skill_updater.py index 013411ce50eb..c4eafdfcef8c 100644 --- a/mycroft/skills/skill_updater.py +++ b/mycroft/skills/skill_updater.py @@ -17,7 +17,7 @@ import sys from datetime import datetime from time import time -from xdg import BaseDirectory +import xdg.BaseDirectory from msm import MsmException @@ -99,7 +99,8 @@ def installed_skills_file_path(self): ) else: self._installed_skills_file_path = os.path.join( - BaseDirectory.save_data_path('mycroft'), '.mycroft-skills') + xdg.BaseDirectory.save_data_path('mycroft'), + '.mycroft-skills') return self._installed_skills_file_path diff --git a/mycroft/util/file_utils.py b/mycroft/util/file_utils.py index cfe4589d40ca..ee441286f5a8 100644 --- a/mycroft/util/file_utils.py +++ b/mycroft/util/file_utils.py @@ -22,7 +22,7 @@ import psutil from stat import S_ISREG, ST_MTIME, ST_MODE, ST_SIZE import tempfile -from xdg import BaseDirectory +import xdg.BaseDirectory import mycroft.configuration from .log import LOG @@ -43,6 +43,7 @@ def resolve_resource_file(res_name): With mycroft running as the user 'bob', if you called ``resolve_resource_file('snd/beep.wav')`` it would return either: + '$XDG_DATA_DIRS/mycroft/beep.wav', '/home/bob/.mycroft/snd/beep.wav' or '/opt/mycroft/snd/beep.wav' or '.../mycroft/res/snd/beep.wav' @@ -62,8 +63,8 @@ def resolve_resource_file(res_name): return res_name # Now look for XDG_DATA_DIRS - for dir in BaseDirectory.load_data_paths('mycroft'): - filename = os.path.join(dir, res_name) + for conf_dir in xdg.BaseDirectory.load_data_paths('mycroft'): + filename = os.path.join(conf_dir, res_name) if os.path.isfile(filename): return filename diff --git a/mycroft/util/log.py b/mycroft/util/log.py index 460b349d889e..8c6c2fa010b9 100644 --- a/mycroft/util/log.py +++ b/mycroft/util/log.py @@ -32,7 +32,7 @@ import sys from os.path import isfile, join -from xdg import BaseDirectory +import xdg.BaseDirectory from mycroft.util.json_helper import load_commented_json, merge_dict @@ -85,8 +85,8 @@ def init(cls): # used since it uses the LOG system and would cause horrible cyclic # dependencies. confs = [] - for dir in BaseDirectory.load_config_paths('mycroft'): - confs.append(join(dir, 'mycroft.conf')) + for conf_dir in xdg.BaseDirectory.load_config_paths('mycroft'): + confs.append(join(conf_dir, 'mycroft.conf')) confs.append('/etc/mycroft/mycroft.conf') confs = reversed(confs) config = {} From 3fd96cf71b25aed79310cb4e9a0edf4fb185609d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Sat, 10 Jul 2021 17:40:34 +0200 Subject: [PATCH 7/9] WIP Review comments - Add TODO for 22.02 to remove the compatibility code - Make Warning a single Log statement - mycroft-config script now uses XDG-environment variable - Remove redundant code - Replace hard coded references to ~/.config - Explicitly remove new path before move of "filesystem" (if needed) --- bin/mycroft-config | 4 +-- mycroft/client/enclosure/mark1/__init__.py | 5 +++- mycroft/client/speech/hotword_factory.py | 2 +- mycroft/configuration/config.py | 33 ++++++++++------------ mycroft/configuration/locations.py | 2 ++ mycroft/filesystem/__init__.py | 3 ++ 6 files changed, 27 insertions(+), 22 deletions(-) diff --git a/bin/mycroft-config b/bin/mycroft-config index 981f2c5d0c65..6598c08cc94c 100755 --- a/bin/mycroft-config +++ b/bin/mycroft-config @@ -95,11 +95,11 @@ function validate_config_file() { return $result } -_conf_file="~/.config/mycroft/mycroft.conf" +_conf_file="${XDG_CONFIG_HOME:-$HOME/.config}/mycroft/mycroft.conf" function name_to_path() { case ${1} in "system") _conf_file="/etc/mycroft/mycroft.conf" ;; - "user") _conf_file=$(readlink -f ~/.config/mycroft/mycroft.conf) ;; + "user") _conf_file=$(readlink -f ${XDG_CONFIG_HOME:-$HOME/.config}/mycroft/mycroft.conf) ;; "default") _conf_file="$DIR/../mycroft/configuration/mycroft.conf" ;; "remote") _conf_file="$HOME/.cache/mycroft/web_cache.json" ;; diff --git a/mycroft/client/enclosure/mark1/__init__.py b/mycroft/client/enclosure/mark1/__init__.py index 5758ecdb694b..e5be8d633749 100644 --- a/mycroft/client/enclosure/mark1/__init__.py +++ b/mycroft/client/enclosure/mark1/__init__.py @@ -19,6 +19,8 @@ import serial +import xdg.BaseDirectory + import mycroft.dialog from mycroft.client.enclosure.base import Enclosure from mycroft.api import has_been_paired @@ -164,7 +166,8 @@ def process(self, data): self.bus.emit(Message("speak", { 'utterance': mycroft.dialog.get("reset to factory defaults")})) subprocess.call( - 'rm ~/.config/mycroft/identity/identity2.json', + (f'rm {xdg.BaseDirectory.save_config_path("mycroft")}' + '/mycroft/identity/identity2.json'), shell=True) subprocess.call( 'rm ~/.mycroft/identity/identity2.json', diff --git a/mycroft/client/speech/hotword_factory.py b/mycroft/client/speech/hotword_factory.py index c6fb57a9bae6..a059dfb9d2e4 100644 --- a/mycroft/client/speech/hotword_factory.py +++ b/mycroft/client/speech/hotword_factory.py @@ -280,7 +280,7 @@ def folder(self): old_path = join(expanduser('~'), '.mycroft', 'precise') if os.path.isdir(old_path): return old_path - return join(xdg.BaseDirectory.save_data_path('mycroft', 'precise')) + return xdg.BaseDirectory.save_data_path('mycroft', 'precise') @property def install_destination(self): diff --git a/mycroft/configuration/config.py b/mycroft/configuration/config.py index 9919ff3945ad..82ac29b56df6 100644 --- a/mycroft/configuration/config.py +++ b/mycroft/configuration/config.py @@ -24,7 +24,7 @@ from mycroft.util.json_helper import load_commented_json, merge_dict from mycroft.util.log import LOG -from .locations import DEFAULT_CONFIG, OLD_USER_CONFIG, USER_CONFIG +from .locations import DEFAULT_CONFIG, USER_CONFIG, OLD_USER_CONFIG from .locations import SYSTEM_CONFIG @@ -170,6 +170,17 @@ def __init__(self, cache=None): self.load_local(cache) +def _log_old_location_deprecation(): + LOG.warning("\n ===============================================\n" + " == DEPRECATION WARNING ==\n" + " ===============================================\n" + f" You still have a config file at {OLD_USER_CONFIG}\n" + " Note that this location is deprecated and will" + " not be used in the future\n" + " Please move it to " + f"{xdg.BaseDirectory.save_config_path('mycroft')}") + + class Configuration: """Namespace for operations on the configuration singleton.""" __config = {} # Cached config @@ -185,9 +196,7 @@ def get(configs=None, cache=True, remote=True): Args: configs (list): List of configuration dicts cache (boolean): True if the result should be cached - remote (boolean): False if the Mycroft Home settings shouldn't - be loaded - + remote (boolean): False if the Remote settings shouldn't be loaded Returns: (dict) configuration dictionary. @@ -210,7 +219,7 @@ def load_config_stack(configs=None, cache=False, remote=True): (dict) merged dict of all configuration files """ if not configs: - configs = configs or [] + configs = [] # First use the patched config configs.append(Configuration.__patch) @@ -223,21 +232,9 @@ def load_config_stack(configs=None, cache=False, remote=True): # Then check the old user config if isfile(OLD_USER_CONFIG): - LOG.warning(" ===============================================") - LOG.warning(" == DEPRECATION WARNING ==") - LOG.warning(" ===============================================") - LOG.warning(" You still have a config file at " + - OLD_USER_CONFIG) - LOG.warning(" Note that this location is deprecated and will" + - " not be used in the future") - LOG.warning(" Please move it to " + - xdg.BaseDirectory.save_config_path('mycroft')) + _log_old_location_deprecation() configs.append(LocalConf(OLD_USER_CONFIG)) - # Then check the XDG user config - if isfile(USER_CONFIG): - configs.append(LocalConf(USER_CONFIG)) - # Then use remote config if remote: configs.append(RemoteConf()) diff --git a/mycroft/configuration/locations.py b/mycroft/configuration/locations.py index b17e8b50864f..98c5b78f37b2 100644 --- a/mycroft/configuration/locations.py +++ b/mycroft/configuration/locations.py @@ -18,11 +18,13 @@ DEFAULT_CONFIG = join(dirname(__file__), 'mycroft.conf') SYSTEM_CONFIG = os.environ.get('MYCROFT_SYSTEM_CONFIG', '/etc/mycroft/mycroft.conf') +# TODO: remove in 22.02 # Make sure we support the old location still # Deprecated and will be removed eventually OLD_USER_CONFIG = join(expanduser('~'), '.mycroft/mycroft.conf') USER_CONFIG = join(xdg.BaseDirectory.save_config_path('mycroft'), 'mycroft.conf') + REMOTE_CONFIG = "mycroft.ai" WEB_CONFIG_CACHE = os.environ.get('MYCROFT_WEB_CACHE', '/var/tmp/mycroft_web_cache.json') diff --git a/mycroft/filesystem/__init__.py b/mycroft/filesystem/__init__.py index 27f84509ded6..c63a17f46a8f 100644 --- a/mycroft/filesystem/__init__.py +++ b/mycroft/filesystem/__init__.py @@ -38,7 +38,10 @@ def __init_path(path): path = join(xdg.BaseDirectory.save_config_path('mycroft'), path) # Migrate from the old location if it still exists + # TODO: remove in 22.02 if isdir(old_path): + if isdir(path): + shutil.rmtree(path) shutil.move(old_path, path) if not isdir(path): From 87d22d30a49891056b5659a488c37da121865f2c Mon Sep 17 00:00:00 2001 From: jarbasal Date: Wed, 7 Jul 2021 15:38:40 +0100 Subject: [PATCH 8/9] fix cyclic imports with LOG The mycroft.util.log module called LOG.init() which needs to read the config, the configuration module imports the log system to log causing cyclic import issues. This moves the LOG.init() call out of the log module making it possible to use the normal config system to init the logs and does a slight re-arrangement so that the uninited log can be used. --- mycroft/__init__.py | 3 +++ mycroft/util/log.py | 31 +++++-------------------------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/mycroft/__init__.py b/mycroft/__init__.py index 90e3494c3e03..9a6e4581fedc 100644 --- a/mycroft/__init__.py +++ b/mycroft/__init__.py @@ -20,6 +20,7 @@ from mycroft.skills import (MycroftSkill, FallbackSkill, intent_handler, intent_file_handler) from mycroft.skills.intent_service import AdaptIntent +from mycroft.util.log import LOG MYCROFT_ROOT_PATH = abspath(join(dirname(__file__), '..')) @@ -33,3 +34,5 @@ 'intent_handler', 'intent_file_handler', 'AdaptIntent'] + +LOG.init() # read log level from config diff --git a/mycroft/util/log.py b/mycroft/util/log.py index 8c6c2fa010b9..b000f0ff0192 100644 --- a/mycroft/util/log.py +++ b/mycroft/util/log.py @@ -31,10 +31,7 @@ import logging import sys -from os.path import isfile, join -import xdg.BaseDirectory - -from mycroft.util.json_helper import load_commented_json, merge_dict +import mycroft def getLogger(name="MYCROFT"): @@ -65,7 +62,7 @@ class LOG: _custom_name = None handler = None - level = None + level = logging.getLevelName('INFO') # Copy actual logging methods from logging.Logger # Usage: LOG.debug(message) @@ -80,24 +77,6 @@ def init(cls): """ Initializes the class, sets the default log level and creates the required handlers. """ - - # Check configs manually, the Mycroft configuration system can't be - # used since it uses the LOG system and would cause horrible cyclic - # dependencies. - confs = [] - for conf_dir in xdg.BaseDirectory.load_config_paths('mycroft'): - confs.append(join(conf_dir, 'mycroft.conf')) - confs.append('/etc/mycroft/mycroft.conf') - confs = reversed(confs) - config = {} - for conf in confs: - try: - merge_dict(config, - load_commented_json(conf) if isfile(conf) else {}) - except Exception as e: - print('couldn\'t load {}: {}'.format(conf, str(e))) - - cls.level = logging.getLevelName(config.get('log_level', 'INFO')) log_message_format = ( '{asctime} | {levelname:8} | {process:5} | {name} | {message}' ) @@ -107,6 +86,9 @@ def init(cls): cls.handler = logging.StreamHandler(sys.stdout) cls.handler.setFormatter(formatter) + config = mycroft.configuration.Configuration.get(remote=False) + cls.level = logging.getLevelName(config.get('log_level', 'INFO')) + # Enable logging in external modules cls.create_logger('').setLevel(cls.level) @@ -148,6 +130,3 @@ def _log(cls, func, *args, **kwargs): name = 'Mycroft' func(cls.create_logger(name), *args, **kwargs) - - -LOG.init() From 4258e2f2ed7fbe8691a62ae4ad916a75999c3c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Sun, 18 Jul 2021 21:13:28 +0200 Subject: [PATCH 9/9] Reorder imports of mycroft.configuration --- mycroft/configuration/config.py | 5 +++-- mycroft/configuration/locations.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mycroft/configuration/config.py b/mycroft/configuration/config.py index 82ac29b56df6..ddfc98733b2d 100644 --- a/mycroft/configuration/config.py +++ b/mycroft/configuration/config.py @@ -14,10 +14,11 @@ # limitations under the License. # -import re -import json import inflection +import json from os.path import exists, isfile, join +import re + from requests import RequestException import xdg.BaseDirectory diff --git a/mycroft/configuration/locations.py b/mycroft/configuration/locations.py index 98c5b78f37b2..8b9590094078 100644 --- a/mycroft/configuration/locations.py +++ b/mycroft/configuration/locations.py @@ -13,6 +13,7 @@ # limitations under the License. import os from os.path import join, dirname, expanduser, exists + import xdg.BaseDirectory DEFAULT_CONFIG = join(dirname(__file__), 'mycroft.conf')