refactoring drop mara-config integration

mara_cli/_mara_modules.py

@@ -0,0 +1,68 @@
+"""Internal functions interacting with mara modules"""
+
+import copy
+from logging import Logger
+import sys
+from types import ModuleType
+from typing import Callable, Dict, Iterable
+
+import click
+
+
+_mara_modules_imported = False
+
+def import_mara_modules(log: Logger):
+    """
+    Import all installed mara modules
+
+    Args:
+        log: The application logger.
+    """
+    global _mara_modules_imported
+    if _mara_modules_imported:
+        return
+
+    import pkg_resources
+    import importlib
+
+    for i in pkg_resources.working_set:
+        package_name = i.key
+        #version = i.version
+        if package_name.startswith('mara-'):
+            log.debug(f"Import module {package_name}")
+            importlib.import_module(name=package_name.replace('-', '_'), package=package_name)
+
+    _mara_modules_imported = True
+
+
+def module_functionalities(module: ModuleType, MARA_XXX: str, type) -> []:
+    """
+    Returns some functionalities of a module that is declared in a MARA_XXX variable or function
+
+    `module.MARA_XXX` can be
+    - a function that returns a list or dict
+    - a list
+    - a dict
+    """
+    if MARA_XXX in dir(module):
+        functionalities = getattr(module, MARA_XXX)
+        if isinstance(functionalities, Callable):
+            functionalities = functionalities()
+        if isinstance(functionalities, Dict):
+            functionalities = functionalities.values()
+        if not isinstance(functionalities, Iterable):
+            raise TypeError(
+                f'{module.__name__}.{MARA_XXX} should be or return a list or dict of {type.__name__}. Got "{functionalities}".')
+        for functionality in functionalities:
+            if not isinstance(functionality, type):
+                raise TypeError(f'In {module.__name__}.{MARA_XXX}: Expected a {type.__name__}, got "{functionality}"')
+        return functionalities
+    else:
+        return []
+
+
+def get_contributed_functionality(name: str, type) -> Dict[ModuleType, object]:
+    """Gets the contributed functionality for one MARA_ variable"""
+    for module in copy.copy(sys.modules).values():
+        for obj in module_functionalities(module, name, click.Command):
+            yield (module, obj)

mara_cli/cli.py

@@ -7,15 +7,14 @@
 
 log = logging.getLogger(__name__)
 
+RED = '\033[31m'
+RESET = '\033[0m'
 
-@click.group(help="""\
-Runs contributed commandline commands
 
-Contributed functionality (ETL runners, downloader,...) are available as subcommands.
+@click.group(help="""
+The Mara ETL Framework is a Python framework to build data pipelines.
 
-To run the flask webapp, use 'flask run'.
-
-""")
+Contributed functionality (ETL runners, downloader,...) are available as subcommands.""")
 @click.option('--debug', default=False, is_flag=True, help="Show debug output")
 @click.option('--log-stderr', default=False, is_flag=True, help="Send log output to stderr")
 def cli(debug: bool, log_stderr: bool):
@@ -26,6 +25,7 @@ def cli(debug: bool, log_stderr: bool):
 
 def setup_commandline_commands():
     """Needs to be run before click itself is run so the config which contributes click commands is available"""
+    from ._mara_modules import import_mara_modules, get_contributed_functionality
     commandline_debug = '--debug' in sys.argv
     # makefiles expect all log in stdout. Send to stderr only if asked to
     log_stream = sys.stderr if '--log-stderr' in sys.argv else sys.stdout
@@ -38,33 +38,20 @@ def setup_commandline_commands():
         logging.root.setLevel(logging.DEBUG)
         log.debug("Enabled debug output via commandline")
 
-    # Initialize the config system
-    from mara_config import init_mara_config_once
-    init_mara_config_once()
-
-    # The order basically means that the we only get information about the config system startup
-    # when --debug is given on the commandline, but not when mara_config.config.debug() is configured
-    # in the config system itself.
-    # I think we can live with that...
-    from mara_config.config import debug as configured_debug
-    if configured_debug():
-        logging.root.setLevel(logging.DEBUG)
-        log.debug("Enabled debug output via config")
-
-    # overwrite any config system with commandline debug switch
-    if commandline_debug and not configured_debug():
-        from mara_config.config_system import set_config
-        set_config('debug', function=lambda: True)
+    # Import all installed mara packages
+    import_mara_modules(log)
 
-    from mara_config import get_contributed_functionality
     known_names = []
-    for module, command in get_contributed_functionality('MARA_CLICK_COMMANDS'):
+    for module, command in get_contributed_functionality('MARA_CLICK_COMMANDS', click.Command):
         if command and 'callback' in command.__dict__ and command.__dict__['callback']:
             package = command.__dict__['callback'].__module__.rpartition('.')[0]
             # Give a package a chance to put all their commands as subcommands of the main package name.
             # For that to work we have to make sure we do not add multiple commands with the same name
-            if isinstance(command, click.Group):
-                name = command.name
+            if isinstance(command, click.MultiCommand):
+                if command.name.startswith('mara-'):
+                    name = command.name[5:]
+                else:
+                    name = command.name
             else:
                 name = package + '.' + command.name
             if name in known_names:
@@ -74,6 +61,16 @@ def setup_commandline_commands():
             known_names.append(name)
             command.name = name
             cli.add_command(command)
+
+    if not cli.commands:
+        # Could not find any command in the installed modules
+        print(RED + "No mara package is installed which provide commands" + RESET, file=sys.stderr)
+        print("""
+Please install the packages you want to use, e.g. by calling
+
+    pip install mara-pipelines
+""", file=sys.stderr)
+        sys.exit(1)
 
 
 def main():

setup.cfg

@@ -11,9 +11,8 @@ license = MIT
 [options]
 packages = mara_cli
 install_requires =
-    mara-config>=0.2.0
     click
-dependency_links = git+https://github.com/mara/mara-config.git@main#egg=mara-config
+    setuptools
 
 [options.extras_require]
 test =

tests/test_mara_main.py

@@ -1,9 +1,6 @@
 from mara_cli.cli import cli
 import re
-import os
 
-# needed workaorund because mara expects a MARA_APP be importable
-os.environ['MARA_APP'] = 'mara_cli'
 
 def test_without_argument(cli_runner):
 
@@ -12,4 +9,3 @@ def test_without_argument(cli_runner):
     # here we get the name as 'cli' instead of 'mara'
     assert 'Usage: cli [OPTIONS] COMMAND [ARGS]' in result.output
     assert re.search(r'--debug\s+Show debug output',result.output) is not None
-