diff --git a/.travis.yml b/.travis.yml index f4f2c788..2662fdb9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,9 +23,9 @@ before_install: script: - | pip3 install --editable ./cli/ && - minipresto --help || exit 1 && echo -e '\n' && + minitrino --help || exit 1 && echo -e '\n' && source ~/virtualenv/python3.8/bin/activate && - python ./cli/minipresto/test/runner.py + python ./cli/minitrino/test/runner.py after_failure: - "Tests failed." diff --git a/cli/minipresto/cmd/cmd_version.py b/cli/minipresto/cmd/cmd_version.py deleted file mode 100644 index e88d3d4d..00000000 --- a/cli/minipresto/cmd/cmd_version.py +++ /dev/null @@ -1,19 +0,0 @@ -#!usr/bin/env/python3 -# -*- coding: utf-8 -*- - -import click -from minipresto.cli import pass_environment -from minipresto import utils - - -@click.command( - "version", - help=("""Display the Minipresto version."""), -) -@utils.exception_handler -@pass_environment -def cli(ctx): - """Version command for Minipresto.""" - - version = utils.get_cli_ver() - ctx.logger.log(f"Minipresto version: {version}") diff --git a/cli/minipresto/test/runner.py b/cli/minipresto/test/runner.py deleted file mode 100644 index acac1e1d..00000000 --- a/cli/minipresto/test/runner.py +++ /dev/null @@ -1,30 +0,0 @@ -#!usr/bin/env/python3 -# -*- coding: utf-8 -*- - -import minipresto.test.test_misc as test_misc -import minipresto.test.test_cmd_config as test_config -import minipresto.test.test_cmd_down as test_down -import minipresto.test.test_cmd_provision as test_provision -import minipresto.test.test_cmd_remove as test_remove -import minipresto.test.test_cmd_snapshot as test_snapshot -import minipresto.test.test_cmd_version as test_version -import minipresto.test.test_cmd_modules as test_modules -import minipresto.test.test_cmd_lib_install as test_cmd_lib_install - - -def main(): - """Minipresto unit test runner.""" - - test_misc.main() - # test_config.main() - test_down.main() - test_provision.main() - test_remove.main() - test_snapshot.main() - test_version.main() - test_modules.main() - test_cmd_lib_install.main() - - -if __name__ == "__main__": - main() diff --git a/cli/minipresto/__init__.py b/cli/minitrino/__init__.py similarity index 100% rename from cli/minipresto/__init__.py rename to cli/minitrino/__init__.py diff --git a/cli/minipresto/cli.py b/cli/minitrino/cli.py similarity index 73% rename from cli/minipresto/cli.py rename to cli/minitrino/cli.py index c202e04c..a7e20e42 100644 --- a/cli/minipresto/cli.py +++ b/cli/minitrino/cli.py @@ -4,12 +4,12 @@ import os import click -from minipresto import settings -from minipresto import components +from minitrino import settings +from minitrino import components from pathlib import Path -CONTEXT_SETTINGS = {"auto_envvar_prefix": "MINIPRESTO"} +CONTEXT_SETTINGS = {"auto_envvar_prefix": "MINITRINO"} pass_environment = click.make_pass_decorator(components.Environment, ensure=True) @@ -26,7 +26,7 @@ def list_commands(self, ctx): def get_command(self, ctx, name): try: - mod = __import__(f"minipresto.cmd.cmd_{name}", None, None, ["cli"]) + mod = __import__(f"minitrino.cmd.cmd_{name}", None, None, ["cli"]) except ImportError: return return mod.cli @@ -49,9 +49,9 @@ def get_command(self, ctx, name): help=( """Add or override environment variables. - Environment variables are sourced from the Minipresto library's root - 'minipresto.env' file as well as the user config file in - '~/.minipresto/minipresto.cfg'. Variables supplied by this option will + Environment variables are sourced from the Minitrino library's root + 'minitrino.env' file as well as the user config file in + '~/.minitrino/minitrino.cfg'. Variables supplied by this option will override values from either of those sources. The variables will also be passed to the environment of the shell executing commands during the 'provision' command.""" @@ -59,11 +59,11 @@ def get_command(self, ctx, name): ) @pass_environment def cli(ctx, verbose, env): - """Welcome to the Minipresto command line interface. + """Welcome to the Minitrino command line interface. To report issues and ask questions, please file a GitHub issue and apply a descriptive label at the GitHub repository: - https://github.com/jefflester/minipresto + https://github.com/jefflester/minitrino """ ctx._user_init(verbose, env) diff --git a/cli/minipresto/cmd/__init__.py b/cli/minitrino/cmd/__init__.py similarity index 100% rename from cli/minipresto/cmd/__init__.py rename to cli/minitrino/cmd/__init__.py diff --git a/cli/minipresto/cmd/cmd_config.py b/cli/minitrino/cmd/cmd_config.py similarity index 76% rename from cli/minipresto/cmd/cmd_config.py rename to cli/minitrino/cmd/cmd_config.py index 65aff7eb..18431f4c 100644 --- a/cli/minipresto/cmd/cmd_config.py +++ b/cli/minitrino/cmd/cmd_config.py @@ -4,16 +4,16 @@ import os import click -from minipresto.cli import pass_environment -from minipresto import utils -from minipresto.settings import CONFIG_TEMPLATE +from minitrino.cli import pass_environment +from minitrino import utils +from minitrino.settings import CONFIG_TEMPLATE from shutil import rmtree @click.command( "config", - help=("""Edit the Minipresto user configuration file."""), + help=("""Edit the Minitrino user configuration file."""), ) @click.option( "-r", @@ -21,7 +21,7 @@ is_flag=True, default=False, help=( - """Reset the Minipresto user configuration file and create a new config + """Reset the Minitrino user configuration file and create a new config file from a template. WARNING: This will remove your configuration file (if it exists) and @@ -31,11 +31,11 @@ @utils.exception_handler @pass_environment def cli(ctx, reset): - """Config command for Minipresto.""" + """Config command for Minitrino.""" - if not os.path.isdir(ctx.minipresto_user_dir): - ctx.logger.log(f"No {ctx.minipresto_user_dir} directory found. Creating...") - os.mkdir(ctx.minipresto_user_dir) + if not os.path.isdir(ctx.minitrino_user_dir): + ctx.logger.log(f"No {ctx.minitrino_user_dir} directory found. Creating...") + os.mkdir(ctx.minitrino_user_dir) if os.path.isfile(ctx.config_file) and not reset: ctx.logger.log( @@ -49,7 +49,7 @@ def cli(ctx, reset): write_template() edit_file() else: - ctx.logger.log(f"Opted out of recreating {ctx.minipresto_user_dir} file.") + ctx.logger.log(f"Opted out of recreating {ctx.minitrino_user_dir} file.") else: ctx.logger.log( f"No config file found at path: {ctx.config_file}. " diff --git a/cli/minipresto/cmd/cmd_down.py b/cli/minitrino/cmd/cmd_down.py similarity index 75% rename from cli/minipresto/cmd/cmd_down.py rename to cli/minitrino/cmd/cmd_down.py index f17d6a93..591a2f97 100644 --- a/cli/minipresto/cmd/cmd_down.py +++ b/cli/minitrino/cmd/cmd_down.py @@ -4,16 +4,16 @@ import sys import click -from minipresto.cli import pass_environment -from minipresto import utils -from minipresto import errors as err -from minipresto.settings import RESOURCE_LABEL +from minitrino.cli import pass_environment +from minitrino import utils +from minitrino import errors as err +from minitrino.settings import RESOURCE_LABEL @click.command( "down", help=( - """Bring down running Minipresto containers. This command follows the + """Bring down running Minitrino containers. This command follows the behavior of `docker-compose down` where containers are both stopped and removed.""" ), @@ -32,13 +32,13 @@ "--sig-kill", is_flag=True, default=False, - help=("""Stop Minipresto containers without a grace period."""), + help=("""Stop Minitrino containers without a grace period."""), ) @utils.exception_handler @pass_environment def cli(ctx, sig_kill, keep): - """Down command for Minipresto. Exits with a 0 status code if there are no - running minipresto containers.""" + """Down command for Minitrino. Exits with a 0 status code if there are no + running minitrino containers.""" utils.check_daemon(ctx.docker_client) utils.check_lib(ctx) @@ -54,7 +54,7 @@ def cli(ctx, sig_kill, keep): if sig_kill: stop_timeout = 1 ctx.logger.log( - "Stopping Minipresto containers with sig-kill...", + "Stopping Minitrino containers with sig-kill...", level=ctx.logger.verbose, ) else: @@ -78,4 +78,4 @@ def cli(ctx, sig_kill, keep): container.remove() ctx.logger.log(f"Removed container: {identifier}", level=ctx.logger.verbose) - ctx.logger.log("Brought down all Minipresto containers.") + ctx.logger.log("Brought down all Minitrino containers.") diff --git a/cli/minipresto/cmd/cmd_lib_install.py b/cli/minitrino/cmd/cmd_lib_install.py similarity index 64% rename from cli/minipresto/cmd/cmd_lib_install.py rename to cli/minitrino/cmd/cmd_lib_install.py index cae885df..9c96f162 100644 --- a/cli/minipresto/cmd/cmd_lib_install.py +++ b/cli/minitrino/cmd/cmd_lib_install.py @@ -6,14 +6,14 @@ import click import shutil -from minipresto.cli import pass_environment -from minipresto import errors as err -from minipresto import utils +from minitrino.cli import pass_environment +from minitrino import errors as err +from minitrino import utils @click.command( "lib_install", - help=("""Install the Minipresto library."""), + help=("""Install the Minitrino library."""), ) @click.option( "-v", @@ -25,15 +25,15 @@ @utils.exception_handler @pass_environment def cli(ctx, version): - """Library installation command for Minipresto.""" + """Library installation command for Minitrino.""" if not version: version = utils.get_cli_ver() - lib_dir = os.path.join(ctx.minipresto_user_dir, "lib") + lib_dir = os.path.join(ctx.minitrino_user_dir, "lib") if os.path.isdir(lib_dir): response = ctx.logger.prompt_msg( - f"The Minipresto library at {lib_dir} will be overwritten. " + f"The Minitrino library at {lib_dir} will be overwritten. " f"Continue? [Y/N]" ) if utils.validate_yes(response): @@ -52,18 +52,18 @@ def cli(ctx, version): @pass_environment def download_and_extract(ctx, version=""): - github_uri = f"https://github.com/jefflester/minipresto/archive/{version}.tar.gz" - tarball = os.path.join(ctx.minipresto_user_dir, f"{version}.tar.gz") + github_uri = f"https://github.com/jefflester/minitrino/archive/{version}.tar.gz" + tarball = os.path.join(ctx.minitrino_user_dir, f"{version}.tar.gz") file_basename = f"minitrino-{version}" # filename after unpacking - lib_dir = os.path.join(ctx.minipresto_user_dir, file_basename, "lib") + lib_dir = os.path.join(ctx.minitrino_user_dir, file_basename, "lib") try: # Download the release tarball cmd = f"curl -fsSL {github_uri} > {tarball}" ctx.cmd_executor.execute_commands(cmd) if not os.path.isfile(tarball): - raise err.MiniprestoError( - f"Failed to download Minipresto library ({tarball} not found)." + raise err.MinitrinoError( + f"Failed to download Minitrino library ({tarball} not found)." ) # Unpack tarball and copy lib @@ -72,14 +72,14 @@ def download_and_extract(ctx, version=""): level=ctx.logger.verbose, ) ctx.cmd_executor.execute_commands( - f"tar -xzvf {tarball} -C {ctx.minipresto_user_dir}", - f"mv {lib_dir} {ctx.minipresto_user_dir}", + f"tar -xzvf {tarball} -C {ctx.minitrino_user_dir}", + f"mv {lib_dir} {ctx.minitrino_user_dir}", ) # Check that the library is present - lib_dir = os.path.join(ctx.minipresto_user_dir, "lib") + lib_dir = os.path.join(ctx.minitrino_user_dir, "lib") if not os.path.isdir(lib_dir): - raise err.MiniprestoError( + raise err.MinitrinoError( f"Library failed to install (not found at {lib_dir})" ) @@ -88,13 +88,13 @@ def download_and_extract(ctx, version=""): except Exception as e: cleanup(tarball, file_basename, False) - raise err.MiniprestoError(str(e)) + raise err.MinitrinoError(str(e)) @pass_environment def cleanup(ctx, tarball="", file_basename="", trigger_error=True): ctx.cmd_executor.execute_commands( - f"rm -rf {tarball} {os.path.join(ctx.minipresto_user_dir, file_basename)}", + f"rm -rf {tarball} {os.path.join(ctx.minitrino_user_dir, file_basename)}", trigger_error=trigger_error, ) diff --git a/cli/minipresto/cmd/cmd_modules.py b/cli/minitrino/cmd/cmd_modules.py similarity index 81% rename from cli/minipresto/cmd/cmd_modules.py rename to cli/minitrino/cmd/cmd_modules.py index 1dedad51..2ca05384 100644 --- a/cli/minipresto/cmd/cmd_modules.py +++ b/cli/minitrino/cmd/cmd_modules.py @@ -4,9 +4,9 @@ import click import json -from minipresto.cli import pass_environment -from minipresto import errors as err -from minipresto import utils +from minitrino.cli import pass_environment +from minitrino import errors as err +from minitrino import utils @click.command( @@ -43,7 +43,7 @@ @utils.exception_handler @pass_environment def cli(ctx, modules, json_format, running): - """Version command for Minipresto.""" + """Version command for Minitrino.""" utils.check_lib(ctx) @@ -55,7 +55,7 @@ def cli(ctx, modules, json_format, running): if not module_dict: raise err.UserError( f"Invalid module: {module}", - "Ensure the module you're referencing is in the Minipresto library.", + "Ensure the module you're referencing is in the Minitrino library.", ) log_info(module, module_dict, json_format) else: @@ -81,10 +81,10 @@ def log_info(ctx, module_name="", module_dict={}, json_format=False): module_dict = {module_name: module_dict} ctx.logger.log(json.dumps(module_dict, indent=2)) else: - description = module_dict.get("description", "") - incompatible_modules = module_dict.get("incompatible_modules", []) - ctx.logger.log( - f"Module: {module_name}\n" - f"Description: {description}\n" - f"Incompatible Modules: {incompatible_modules}", - ) + log_msg = [f"Module: {module_name}\n"] + keys = ["description", "incompatibleModules", "enterprise"] + for k, v in module_dict.items(): + if k in keys: + log_msg.extend(f"{k.title()}: {v}\n") + + ctx.logger.log("".join(log_msg)) diff --git a/cli/minipresto/cmd/cmd_provision.py b/cli/minitrino/cmd/cmd_provision.py similarity index 76% rename from cli/minipresto/cmd/cmd_provision.py rename to cli/minitrino/cmd/cmd_provision.py index 78ebcd58..3b57e653 100644 --- a/cli/minipresto/cmd/cmd_provision.py +++ b/cli/minitrino/cmd/cmd_provision.py @@ -4,24 +4,25 @@ # Instead of using the Docker SDK, this script will form a Docker Compose # command string to execute straight through the docker-compose CLI. This is # required because the Docker SDK does not support communication with Compose -# files, and Minipresto benefits hugely from Docker Compose. +# files, and Minitrino benefits hugely from Docker Compose. import os import stat import hashlib import time import click - -from minipresto.cli import pass_environment -from minipresto import utils -from minipresto import errors as err -from minipresto.settings import RESOURCE_LABEL -from minipresto.settings import MODULE_ROOT -from minipresto.settings import MODULE_CATALOG -from minipresto.settings import MODULE_SECURITY -from minipresto.settings import ETC_PRESTO -from minipresto.settings import PRESTO_CONFIG -from minipresto.settings import PRESTO_JVM_CONFIG +import yaml + +from minitrino.cli import pass_environment +from minitrino import utils +from minitrino import errors as err +from minitrino.settings import RESOURCE_LABEL +from minitrino.settings import MODULE_ROOT +from minitrino.settings import MODULE_CATALOG +from minitrino.settings import MODULE_SECURITY +from minitrino.settings import ETC_TRINO +from minitrino.settings import TRINO_CONFIG +from minitrino.settings import TRINO_JVM_CONFIG from docker.errors import NotFound @@ -62,34 +63,35 @@ docker-compose shell command. Run `docker-compose up --help` to see all available options. - Example: minipresto provision --docker-native --build + Example: minitrino provision --docker-native --build - Example: minipresto provision --docker-native '--remove-orphans + Example: minitrino provision --docker-native '--remove-orphans --force-recreate'""" ), ) @utils.exception_handler @pass_environment def cli(ctx, modules, no_rollback, docker_native): - """Provision command for Minipresto. If the resulting docker-compose command + """Provision command for Minitrino. If the resulting docker-compose command is unsuccessful, the function exits with a non-zero status code.""" utils.check_daemon(ctx.docker_client) utils.check_lib(ctx) modules = append_running_modules(modules) check_compatibility(modules) + check_enterprise(modules) if not modules: ctx.logger.log( f"No catalog or security options received. Provisioning " - f"standalone Presto container..." + f"standalone Trino container..." ) else: for module in modules: if not ctx.modules.data.get(module, False): raise err.UserError( f"Invalid module: '{module}'. It was not found " - f"in the Minipresto library at {ctx.minipresto_lib_dir}" + f"in the Minitrino library at {ctx.minitrino_lib_dir}" ) try: @@ -147,7 +149,7 @@ def check_compatibility(ctx, modules=[]): other. If they are, a user error is raised.""" for module in modules: - incompatible = ctx.modules.data.get(module, {}).get("incompatible_modules", []) + incompatible = ctx.modules.data.get(module, {}).get("incompatibleModules", []) if not incompatible: continue for module_inner in modules: @@ -160,7 +162,41 @@ def check_compatibility(ctx, modules=[]): f"with module '{module}'. Incompatible modules listed for module " f"'{module}' are: {incompatible}", f"You can see which modules are incompatible with this module by " - f"running 'minipresto modules -m {module}'", + f"running 'minitrino modules -m {module}'", + ) + + +@pass_environment +def check_enterprise(ctx, modules=[]): + """Checks if any of the provided modules are Starburst Enterprise features. + If they are, we check that a pointer to an SEP license is provided.""" + + ctx.logger.log( + "Checking for SEP license for enterprise modules...", + level=ctx.logger.verbose, + ) + + for module in modules: + enterprise = ctx.modules.data.get(module, {}).get("enterprise", False) + if enterprise: + yaml_path = os.path.join(ctx.minitrino_lib_dir, "docker-compose.yml") + with open(yaml_path) as f: + yaml_file = yaml.load(f, Loader=yaml.FullLoader) + if ( + not yaml_file.get("services", False) + .get("trino", False) + .get("volumes", False) + ): + raise err.UserError( + f"Module {module} requires a Starburst license. " + f"The license volume in the library's docker-compose.yml " + f"file must be uncommented at: {yaml_path}" + ) + if not ctx.env.get_var("STARBURST_LIC_PATH", False): + raise err.UserError( + f"Module {module} requires a Starburst license. " + f"You must provide a path to a Starburst license via the " + f"STARBURST_LIC_PATH environment variable" ) @@ -191,7 +227,7 @@ def build_command(ctx, docker_native="", compose_env={}, chunk=""): compose_env_string, "\\\n", "docker-compose -f ", - os.path.join(ctx.minipresto_lib_dir, "docker-compose.yml"), + os.path.join(ctx.minitrino_lib_dir, "docker-compose.yml"), " \\\n", chunk, # Module YAML paths "up -d", @@ -225,7 +261,7 @@ def execute_bootstraps(ctx, modules=[]): ctx.modules.data.get(module, {}).get("yaml_dict", {}).get("services", {}) ) if not module_services: - raise err.MiniprestoError( + raise err.MinitrinoError( f"Invalid Docker Compose YAML file (no 'services' section found): {yaml_file}" ) # Get all services defined in YAML file @@ -235,7 +271,7 @@ def execute_bootstraps(ctx, modules=[]): containers = [] # Get all container names for each service for service in services: - bootstrap = service[1].get("environment", {}).get("MINIPRESTO_BOOTSTRAP") + bootstrap = service[1].get("environment", {}).get("MINITRINO_BOOTSTRAP") if bootstrap is None: continue container_name = service[1].get("container_name") @@ -251,7 +287,7 @@ def execute_bootstraps(ctx, modules=[]): @pass_environment def execute_container_bootstrap(ctx, bootstrap="", container_name="", yaml_file=""): """Executes a single bootstrap inside a container. If the - `/opt/minipresto/bootstrap_status.txt` file has the same checksum as the + `/opt/minitrino/bootstrap_status.txt` file has the same checksum as the bootstrap script that is about to be executed, the boostrap script is skipped. @@ -281,7 +317,7 @@ def execute_container_bootstrap(ctx, bootstrap="", container_name="", yaml_file= # Check if this script has already been executed output = ctx.cmd_executor.execute_commands( - "cat /opt/minipresto/bootstrap_status.txt", + "cat /opt/minitrino/bootstrap_status.txt", suppress_output=True, container=container, trigger_error=False, @@ -306,7 +342,7 @@ def execute_container_bootstrap(ctx, bootstrap="", container_name="", yaml_file= # Record executed file checksum ctx.cmd_executor.execute_commands( f"/tmp/{os.path.basename(bootstrap_file)}", - f'bash -c "echo {bootstrap_checksum} >> /opt/minipresto/bootstrap_status.txt"', + f'bash -c "echo {bootstrap_checksum} >> /opt/minitrino/bootstrap_status.txt"', container=container, ) @@ -320,7 +356,7 @@ def execute_container_bootstrap(ctx, bootstrap="", container_name="", yaml_file= @pass_environment def check_dup_configs(ctx): - """Checks for duplicate configs in Presto config files (jvm.config and + """Checks for duplicate configs in Trino config files (jvm.config and config.properties). This is a safety check for modules that may improperly modify these files. @@ -328,30 +364,30 @@ def check_dup_configs(ctx): identical. For config.properties, duplicates will be registered if there are multiple overlapping property keys.""" - check_files = [PRESTO_CONFIG, PRESTO_JVM_CONFIG] + check_files = [TRINO_CONFIG, TRINO_JVM_CONFIG] for check_file in check_files: ctx.logger.log( - f"Checking Presto {check_file} for duplicate configs...", + f"Checking Trino {check_file} for duplicate configs...", level=ctx.logger.verbose, ) - container = ctx.docker_client.containers.get("presto") + container = ctx.docker_client.containers.get("trino") output = ctx.cmd_executor.execute_commands( - f"cat {ETC_PRESTO}/{check_file}", + f"cat {ETC_TRINO}/{check_file}", suppress_output=True, container=container, ) configs = output[0].get("output", "") if not configs: - raise err.MiniprestoError( - f"Presto {check_file} file unable to be read from Presto container." + raise err.MinitrinoError( + f"Trino {check_file} file unable to be read from Trino container." ) configs = configs.strip().split("\n") configs.sort() duplicates = [] - if check_file == PRESTO_CONFIG: + if check_file == TRINO_CONFIG: for i, config in enumerate(configs): if config.startswith("#"): continue @@ -372,7 +408,7 @@ def check_dup_configs(ctx): duplicates = set(duplicates) duplicates_string = "\n".join(duplicates) ctx.logger.log( - f"Duplicate Presto configuration properties detected in " + f"Duplicate Trino configuration properties detected in " f"{check_file} file:\n{duplicates_string}", level=ctx.logger.warn, ) @@ -391,7 +427,7 @@ def check_dup_configs(ctx): elif duplicates: duplicates_string = "\n".join(duplicates) ctx.logger.log( - f"Duplicate Presto configuration properties detected in " + f"Duplicate Trino configuration properties detected in " f"{check_file} file:\n{duplicates_string}", level=ctx.logger.warn, ) @@ -400,65 +436,65 @@ def check_dup_configs(ctx): @pass_environment def append_user_config(ctx, containers_to_restart=[]): - """Appends Presto config from minipresto.cfg file. If the config is not + """Appends Trino config from minitrino.cfg file. If the config is not present, it is added. If it exists, it is replaced. If anything changes in - the Presto config, the Presto container is added to the restart list if it's + the Trino config, the Trino container is added to the restart list if it's not already in the list.""" - user_presto_config = ctx.env.get_var("CONFIG", "") - if user_presto_config: - user_presto_config = user_presto_config.strip().split("\n") + user_trino_config = ctx.env.get_var("CONFIG", "") + if user_trino_config: + user_trino_config = user_trino_config.strip().split("\n") user_jvm_config = ctx.env.get_var("JVM_CONFIG", "") if user_jvm_config: user_jvm_config = user_jvm_config.strip().split("\n") - if not user_presto_config and not user_jvm_config: + if not user_trino_config and not user_jvm_config: return containers_to_restart ctx.logger.log( - "Appending user-defined Presto config to Presto container config...", + "Appending user-defined Trino config to Trino container config...", level=ctx.logger.verbose, ) - presto_container = ctx.docker_client.containers.get("presto") - if not presto_container: - raise err.MiniprestoError( - f"Attempting to append Presto configuration in Presto container, " - f"but no running Presto container was found." + trino_container = ctx.docker_client.containers.get("trino") + if not trino_container: + raise err.MinitrinoError( + f"Attempting to append Trino configuration in Trino container, " + f"but no running Trino container was found." ) ctx.logger.log( - "Checking Presto server status before updating configs...", + "Checking Trino server status before updating configs...", level=ctx.logger.verbose, ) retry = 0 while retry <= 30: - logs = presto_container.logs().decode() + logs = trino_container.logs().decode() if "======== SERVER STARTED ========" in logs: break - elif presto_container.status != "running": - raise err.MiniprestoError( - f"Presto container stopped running. Inspect the container logs if the " + elif trino_container.status != "running": + raise err.MinitrinoError( + f"Trino container stopped running. Inspect the container logs if the " f"container is still available. If the container was rolled back, rerun " f"the command with the '--no-rollback' option, then inspect the logs." ) else: ctx.logger.log( - "Presto server has not started. Waiting one second and trying again...", + "Trino server has not started. Waiting one second and trying again...", level=ctx.logger.verbose, ) time.sleep(1) retry += 1 current_configs = ctx.cmd_executor.execute_commands( - f"cat {ETC_PRESTO}/{PRESTO_CONFIG}", - f"cat {ETC_PRESTO}/{PRESTO_JVM_CONFIG}", - container=presto_container, + f"cat {ETC_TRINO}/{TRINO_CONFIG}", + f"cat {ETC_TRINO}/{TRINO_JVM_CONFIG}", + container=trino_container, suppress_output=True, ) - current_presto_config = current_configs[0].get("output", "").strip().split("\n") + current_trino_config = current_configs[0].get("output", "").strip().split("\n") current_jvm_config = current_configs[1].get("output", "").strip().split("\n") def append_configs(user_configs, current_configs, filename): @@ -467,7 +503,7 @@ def append_configs(user_configs, current_configs, filename): # config. If there is not overlapping config key, append it to the # current config list. - if filename == PRESTO_CONFIG: + if filename == TRINO_CONFIG: for user_config in user_configs: user_config = utils.parse_key_value_pair( user_config, err_type=err.UserError @@ -512,20 +548,20 @@ def append_configs(user_configs, current_configs, filename): # Replace existing file with new values ctx.cmd_executor.execute_commands( - f"rm {ETC_PRESTO}/{filename}", container=presto_container + f"rm {ETC_TRINO}/{filename}", container=trino_container ) for current_config in current_configs: append_config = ( - f'bash -c "cat <> {ETC_PRESTO}/{filename}\n{current_config}\nEOT"' + f'bash -c "cat <> {ETC_TRINO}/{filename}\n{current_config}\nEOT"' ) - ctx.cmd_executor.execute_commands(append_config, container=presto_container) + ctx.cmd_executor.execute_commands(append_config, container=trino_container) - append_configs(user_presto_config, current_presto_config, PRESTO_CONFIG) - append_configs(user_jvm_config, current_jvm_config, PRESTO_JVM_CONFIG) + append_configs(user_trino_config, current_trino_config, TRINO_CONFIG) + append_configs(user_jvm_config, current_jvm_config, TRINO_JVM_CONFIG) - if not "presto" in containers_to_restart: - containers_to_restart.append("presto") + if not "trino" in containers_to_restart: + containers_to_restart.append("trino") return containers_to_restart @@ -548,19 +584,19 @@ def restart_containers(ctx, containers_to_restart=[]): ) container.restart() except NotFound: - raise err.MiniprestoError( + raise err.MinitrinoError( f"Attempting to restart container '{container.name}', but the container was not found." ) @pass_environment def initialize_containers(ctx): - """Initializes each container with /opt/minipresto/bootstrap_status.txt.""" + """Initializes each container with /opt/minitrino/bootstrap_status.txt.""" containers = ctx.docker_client.containers.list(filters={"label": RESOURCE_LABEL}) for container in containers: output = ctx.cmd_executor.execute_commands( - "cat /opt/minipresto/bootstrap_status.txt", + "cat /opt/minitrino/bootstrap_status.txt", suppress_output=True, container=container, trigger_error=False, @@ -568,14 +604,14 @@ def initialize_containers(ctx): output_string = output[0].get("output", "").strip() if "no such file or directory" in output_string.lower(): ctx.cmd_executor.execute_commands( - "mkdir -p /opt/minipresto/", - "touch /opt/minipresto/bootstrap_status.txt", + "mkdir -p /opt/minitrino/", + "touch /opt/minitrino/bootstrap_status.txt", container=container, ) elif output[0].get("return_code", None) == 0: continue else: - raise err.MiniprestoError( + raise err.MinitrinoError( f"Command failed.\n" f"Output: {output_string}\n" f"Exit code: {output[0].get('return_code', None)}" diff --git a/cli/minipresto/cmd/cmd_remove.py b/cli/minitrino/cmd/cmd_remove.py similarity index 86% rename from cli/minipresto/cmd/cmd_remove.py rename to cli/minitrino/cmd/cmd_remove.py index 4f5a8a7d..4e9e2971 100644 --- a/cli/minipresto/cmd/cmd_remove.py +++ b/cli/minitrino/cmd/cmd_remove.py @@ -4,31 +4,31 @@ import sys import click -from minipresto.cli import pass_environment -from minipresto import utils -from minipresto.settings import IMAGE -from minipresto.settings import VOLUME -from minipresto.settings import RESOURCE_LABEL +from minitrino.cli import pass_environment +from minitrino import utils +from minitrino.settings import IMAGE +from minitrino.settings import VOLUME +from minitrino.settings import RESOURCE_LABEL from docker.errors import APIError @click.command( "remove", - help=("""Remove Minipresto resources."""), + help=("""Remove Minitrino resources."""), ) @click.option( "-i", "--images", is_flag=True, default=False, - help=("""Remove Minipresto images."""), + help=("""Remove Minitrino images."""), ) @click.option( "-v", "--volumes", is_flag=True, default=False, - help=("""Remove Minipresto container volumes."""), + help=("""Remove Minitrino container volumes."""), ) @click.option( "-l", @@ -48,20 +48,20 @@ is_flag=True, default=False, help=( - """Force the removal of Minipresto resources. Normal Docker removal + """Force the removal of Minitrino resources. Normal Docker removal restrictions apply.""" ), ) @utils.exception_handler @pass_environment def cli(ctx, images, volumes, labels, force): - """Remove command for Minipresto.""" + """Remove command for Minitrino.""" utils.check_daemon(ctx.docker_client) if all((not images, not volumes, not labels)) or all((images, volumes, not labels)): response = ctx.logger.prompt_msg( - "You are about to all remove minipresto images and volumes. Continue? [Y/N]" + "You are about to all remove minitrino images and volumes. Continue? [Y/N]" ) if utils.validate_yes(response): remove_items(IMAGE, force) @@ -80,7 +80,7 @@ def cli(ctx, images, volumes, labels, force): @pass_environment def remove_items(ctx, item_type, force, labels=[]): - """Removes Docker items. If no labels are passed in, all Minipresto + """Removes Docker items. If no labels are passed in, all Minitrino resources are removed. If label(s) are passed in, the removal is limited to the passed in labels.""" diff --git a/cli/minipresto/cmd/cmd_snapshot.py b/cli/minitrino/cmd/cmd_snapshot.py similarity index 87% rename from cli/minipresto/cmd/cmd_snapshot.py rename to cli/minitrino/cmd/cmd_snapshot.py index 38140e7a..bcecfbbf 100644 --- a/cli/minipresto/cmd/cmd_snapshot.py +++ b/cli/minitrino/cmd/cmd_snapshot.py @@ -9,24 +9,24 @@ import tarfile import fileinput -from minipresto.cli import pass_environment -from minipresto import utils -from minipresto import errors as err -from minipresto.settings import SNAPSHOT_ROOT_FILES -from minipresto.settings import PROVISION_SNAPSHOT_TEMPLATE -from minipresto.settings import LIB -from minipresto.settings import MODULE_ROOT -from minipresto.settings import MODULE_CATALOG -from minipresto.settings import MODULE_SECURITY -from minipresto.settings import MODULE_RESOURCES -from minipresto.settings import SCRUB_KEYS +from minitrino.cli import pass_environment +from minitrino import utils +from minitrino import errors as err +from minitrino.settings import SNAPSHOT_ROOT_FILES +from minitrino.settings import PROVISION_SNAPSHOT_TEMPLATE +from minitrino.settings import LIB +from minitrino.settings import MODULE_ROOT +from minitrino.settings import MODULE_CATALOG +from minitrino.settings import MODULE_SECURITY +from minitrino.settings import MODULE_RESOURCES +from minitrino.settings import SCRUB_KEYS @click.command( "snapshot", help=( - """Create a snapshot of a Minipresto environment. A tarball is placed in - the Minipresto `lib/snapshots/` directory. + """Create a snapshot of a Minitrino environment. A tarball is placed in + the Minitrino `lib/snapshots/` directory. To take a snapshot of an active environment, leave the `--module` and option out of the command. @@ -60,7 +60,7 @@ type=click.Path(), help=( """Directory to save the resulting snapshot file in. Defaults to the - snapshots directory in the Minipresto library.""" + snapshots directory in the Minitrino library.""" ), ) @click.option( @@ -85,9 +85,9 @@ @utils.exception_handler @pass_environment def cli(ctx, modules, name, directory, force, no_scrub): - """Snapshot command for Minipresto.""" + """Snapshot command for Minitrino.""" - # The snapshot temp files are saved in ~/.minipresto/snapshots/ + # The snapshot temp files are saved in ~/.minitrino/snapshots/ # regardless of the directory provided. The artifact (tarball) will go # to either the default directory or the user-provided directory. @@ -100,7 +100,7 @@ def cli(ctx, modules, name, directory, force, no_scrub): ) if not directory: - directory = os.path.join(ctx.minipresto_lib_dir, "snapshots") + directory = os.path.join(ctx.minitrino_lib_dir, "snapshots") validate_name(name) check_exists(name, directory, force) @@ -112,8 +112,8 @@ def cli(ctx, modules, name, directory, force, no_scrub): modules = ctx.modules.get_running_modules() if not modules: ctx.logger.log( - f"No running Minipresto modules to snapshot. Snapshotting " - f"Presto resources only.", + f"No running Minitrino modules to snapshot. Snapshotting " + f"Trino resources only.", level=ctx.logger.verbose, ) else: @@ -189,7 +189,7 @@ def prepare_snapshot_dir(ctx, name, active, no_scrub, modules): @pass_environment def build_snapshot_command(ctx, snapshot_name_dir, modules=[], active=True): """Builds a basic shell command that can be used to provision an environment - with the minipresto CLI. Used for snapshot purposes.""" + with the minitrino CLI. Used for snapshot purposes.""" command_string = build_command_string(modules) create_snapshot_command_file(command_string, snapshot_name_dir) @@ -210,7 +210,7 @@ def build_command_string(ctx, modules=[]): bash_source = '"${BASH_SOURCE%/*}"' command_string = ( - f"minipresto -v --env LIB_PATH={bash_source}/lib provision {option_string}\n\n" + f"minitrino -v --env LIB_PATH={bash_source}/lib provision {option_string}\n\n" ) return command_string.replace(" ", " ") @@ -218,7 +218,7 @@ def build_command_string(ctx, modules=[]): @pass_environment def create_snapshot_command_file(ctx, command_string="", snapshot_name_dir=""): - """Creates an .sh file in the minipresto directory for usage by the snapshot + """Creates an .sh file in the minitrino directory for usage by the snapshot command. This way, a similar command used to provision the environment is preserved.""" @@ -239,7 +239,7 @@ def create_snapshot_command_file(ctx, command_string="", snapshot_name_dir=""): st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, ) except Exception as e: - minipresto.utils.handle_exception(e, ctx.verbose) + utils.handle_exception(e, ctx.verbose) with open(file_dest, "a") as provision_snapshot_file: provision_snapshot_file.write(command_string) @@ -258,13 +258,13 @@ def clone_lib_dir(ctx, name): os.mkdir(os.path.join(snapshot_name_dir, LIB, MODULE_ROOT, MODULE_RESOURCES)) # Copy root lib files to snapshot - for filename in os.listdir(ctx.minipresto_lib_dir): + for filename in os.listdir(ctx.minitrino_lib_dir): if filename in SNAPSHOT_ROOT_FILES: - file_path = os.path.join(ctx.minipresto_lib_dir, filename) + file_path = os.path.join(ctx.minitrino_lib_dir, filename) shutil.copy(file_path, os.path.join(snapshot_name_dir, LIB)) # Copy everything from lib/modules/resources - resources_dir = os.path.join(ctx.minipresto_lib_dir, MODULE_ROOT, MODULE_RESOURCES) + resources_dir = os.path.join(ctx.minitrino_lib_dir, MODULE_ROOT, MODULE_RESOURCES) for filename in os.listdir(resources_dir): file_path = os.path.join(resources_dir, filename) shutil.copy( @@ -314,7 +314,7 @@ def copy_config_file(ctx, snapshot_name_dir, no_scrub=False): def scrub_config_file(ctx, snapshot_name_dir): """Scrubs the user config file of sensitive data.""" - snapshot_config_file = os.path.join(snapshot_name_dir, "minipresto.cfg") + snapshot_config_file = os.path.join(snapshot_name_dir, "minitrino.cfg") if os.path.isfile(snapshot_config_file): for line in fileinput.input(snapshot_config_file, inplace=True): if "=" in line: @@ -379,6 +379,4 @@ def check_complete(ctx, name, directory): snapshot_file = os.path.join(directory, f"{name}.tar.gz") if not os.path.isfile(snapshot_file): - raise err.MiniprestoError( - f"Snapshot tarball failed to write to {snapshot_file}" - ) + raise err.MinitrinoError(f"Snapshot tarball failed to write to {snapshot_file}") diff --git a/cli/minitrino/cmd/cmd_version.py b/cli/minitrino/cmd/cmd_version.py new file mode 100644 index 00000000..5f65b3d1 --- /dev/null +++ b/cli/minitrino/cmd/cmd_version.py @@ -0,0 +1,27 @@ +#!usr/bin/env/python3 +# -*- coding: utf-8 -*- + +import os +import sys +import click +from minitrino.cli import pass_environment +from minitrino import utils + + +@click.command( + "version", + help=("""Display Minitrino CLI and library versions."""), +) +@utils.exception_handler +@pass_environment +def cli(ctx): + """Version command for Minitrino.""" + + cli_version = utils.get_cli_ver() + ctx.logger.log(f"Minitrino version: {cli_version}") + + try: + lib_version = utils.get_lib_ver(ctx.minitrino_lib_dir) + ctx.logger.log(f"Library version: {lib_version}") + except: + ctx.logger.log("Library version: NOT INSTALLED") diff --git a/cli/minipresto/components.py b/cli/minitrino/components.py similarity index 84% rename from cli/minipresto/components.py rename to cli/minitrino/components.py index 4872caf3..bf5ccdbf 100644 --- a/cli/minipresto/components.py +++ b/cli/minitrino/components.py @@ -11,13 +11,13 @@ from pathlib import Path from configparser import ConfigParser -from minipresto import utils -from minipresto import errors as err -from minipresto.settings import RESOURCE_LABEL -from minipresto.settings import MODULE_LABEL_KEY_ROOT -from minipresto.settings import MODULE_ROOT -from minipresto.settings import MODULE_SECURITY -from minipresto.settings import MODULE_CATALOG +from minitrino import utils +from minitrino import errors as err +from minitrino.settings import RESOURCE_LABEL +from minitrino.settings import MODULE_LABEL_KEY_ROOT +from minitrino.settings import MODULE_ROOT +from minitrino.settings import MODULE_SECURITY +from minitrino.settings import MODULE_CATALOG class Environment: @@ -26,10 +26,10 @@ class Environment: CLI's entrypoint, as it depends on user-provided inputs. ### Public Attributes (Interactive) - - `logger`: A `minipresto.utils.Logger` object. + - `logger`: A `minitrino.utils.Logger` object. - `env`: An `EnvironmentVariables` object containing all CLI environment variables, subdivided by sections when possible. - - `modules`: A `Modules` object containing metadata about Minipresto + - `modules`: A `Modules` object containing metadata about Minitrino modules. - `cmd_executor`: A `CommandExecutor` object to execute shell commands in the host shell and inside containers. @@ -39,13 +39,13 @@ class Environment: ### Public Attributes (Static) - `verbose`: If `True`, logs flagged as verbose to are sent to stdout. - `user_home_dir`: The home directory of the current user. - - `minipresto_user_dir`: The location of the Minipresto directory relative - to the user home directory (~/.minipresto/). - - `config_file`: The location of the user's minipresto.cfg file. + - `minitrino_user_dir`: The location of the Minitrino directory relative + to the user home directory (~/.minitrino/). + - `config_file`: The location of the user's minitrino.cfg file. - `snapshot_dir`: The location of the user's snapshot directory (this is essentially a temporary directory, as 'permanent' snapshot tarballs are written to the library or user-specified directory). - - `minipresto_lib_dir`: The location of the Minipresto library.""" + - `minitrino_lib_dir`: The location of the Minitrino library.""" @utils.exception_handler def __init__(self): @@ -63,21 +63,21 @@ def __init__(self): # Paths self.user_home_dir = os.path.expanduser("~") - self.minipresto_user_dir = self._handle_minipresto_user_dir() + self.minitrino_user_dir = self._handle_minitrino_user_dir() self.config_file = self._get_config_file() - self.snapshot_dir = os.path.join(self.minipresto_user_dir, "snapshots") + self.snapshot_dir = os.path.join(self.minitrino_user_dir, "snapshots") @property - def minipresto_lib_dir(self): - """The directory of the Minipresto library. The directory can be + def minitrino_lib_dir(self): + """The directory of the Minitrino library. The directory can be determined in four ways (this is the order of precedence): 1. Passing `LIB_PATH` to the CLI's `--env` option sets the library directory for the current command. - 2. The `minipresto.cfg` file's `LIB_PATH` variable sets the library + 2. The `minitrino.cfg` file's `LIB_PATH` variable sets the library directory if present. - 3. The path `~/.minipresto/lib/` is used as the default lib path if the + 3. The path `~/.minitrino/lib/` is used as the default lib path if the `LIB_PATH` var is not found. - 4. As a last resort, Minipresto will check to see if the library exists + 4. As a last resort, Minitrino will check to see if the library exists in relation to the positioning of the `components.py` file and assumes the project is being run out of a cloned repository.""" @@ -89,26 +89,26 @@ def minipresto_lib_dir(self): except: pass - if not lib_dir and os.path.isdir(os.path.join(self.minipresto_user_dir, "lib")): - lib_dir = os.path.join(self.minipresto_user_dir, "lib") + if not lib_dir and os.path.isdir(os.path.join(self.minitrino_user_dir, "lib")): + lib_dir = os.path.join(self.minitrino_user_dir, "lib") elif not lib_dir: # Use repo root, fail if this doesn't exist lib_dir = Path(os.path.abspath(__file__)).resolve().parents[2] lib_dir = os.path.join(lib_dir, "lib") if not os.path.isdir(lib_dir) or not os.path.isfile( - os.path.join(lib_dir, "minipresto.env") + os.path.join(lib_dir, "minitrino.env") ): raise err.UserError( - "You must provide a path to a compatible Minipresto library.", - f"You can point to a Minipresto library a few different " + "You must provide a path to a compatible Minitrino library.", + f"You can point to a Minitrino library a few different " f"ways:\n(1) You can set the 'LIB_PATH' variable in your " - f"Minipresto config via the command 'minipresto config'--this " + f"Minitrino config via the command 'minitrino config'--this " f"should be placed under the '[CLI]' section.\n(2) You can " f"pass in 'LIB_PATH' as an environment variable for the current " - f"command, e.g. 'minipresto -e LIB_PATH= ...'\n" - f"(3) If the above variable is not found, Minipresto will check " - f"if '~/.minipresto/lib/' is a valid directory.\n(4) " - f"If you are running Minipresto out of a cloned repo, the library " + f"command, e.g. 'minitrino -e LIB_PATH= ...'\n" + f"(3) If the above variable is not found, Minitrino will check " + f"if '~/.minitrino/lib/' is a valid directory.\n(4) " + f"If you are running Minitrino out of a cloned repo, the library " f"path will be automatically detected without the need to perform " f"any of the above.", ) @@ -128,9 +128,9 @@ def _user_init(self, verbose=False, user_env=[]): # Skip the library-related procedures if the library is not found try: - if self.minipresto_lib_dir: + if self.minitrino_lib_dir: self.logger.log( - f"Library path set to: {self.minipresto_lib_dir}", + f"Library path set to: {self.minitrino_lib_dir}", level=self.logger.verbose, ) @@ -141,12 +141,12 @@ def _user_init(self, verbose=False, user_env=[]): # Warn the user if the library and CLI vers don't match cli_ver = utils.get_cli_ver() - lib_ver = utils.get_lib_ver(self.minipresto_lib_dir) + lib_ver = utils.get_lib_ver(self.minitrino_lib_dir) if cli_ver != lib_ver: self.logger.log( f"CLI version {cli_ver} and library version {lib_ver} " - f"do not match. You can update the Minipresto library " - f"version to match the CLI version by running 'minipresto " + f"do not match. You can update the Minitrino library " + f"version to match the CLI version by running 'minitrino " f"lib_install'.", level=self.logger.warn, ) @@ -157,35 +157,35 @@ def _user_init(self, verbose=False, user_env=[]): self.cmd_executor = CommandExecutor(self) self._get_docker_clients() - def _handle_minipresto_user_dir(self): - """Checks if a Minipresto directory exists in the user home directory. - If it does not, it is created. The path to the Minipresto user home + def _handle_minitrino_user_dir(self): + """Checks if a Minitrino directory exists in the user home directory. + If it does not, it is created. The path to the Minitrino user home directory is returned.""" - minipresto_user_dir = os.path.abspath( - os.path.join(self.user_home_dir, ".minipresto") + minitrino_user_dir = os.path.abspath( + os.path.join(self.user_home_dir, ".minitrino") ) - if not os.path.isdir(minipresto_user_dir): - os.mkdir(minipresto_user_dir) - return minipresto_user_dir + if not os.path.isdir(minitrino_user_dir): + os.mkdir(minitrino_user_dir) + return minitrino_user_dir def _get_config_file(self): - """Returns the correct filepath for the minipresto.cfg file. Adds to + """Returns the correct filepath for the minitrino.cfg file. Adds to initialization warnings if the file does not exist, but will return the path regardless.""" - config_file = os.path.join(self.minipresto_user_dir, "minipresto.cfg") + config_file = os.path.join(self.minitrino_user_dir, "minitrino.cfg") if not os.path.isfile(config_file): self.logger.log( - f"No minipresto.cfg file found at {config_file}. " - f"Run 'minipresto config' to reconfigure this file and directory.", + f"No minitrino.cfg file found at {config_file}. " + f"Run 'minitrino config' to reconfigure this file and directory.", level=self.logger.warn, ) return config_file def _get_docker_clients(self): """Gets DockerClient and APIClient objects. References the DOCKER_HOST - variable in `minipresto.cfg` and uses for clients if present. Returns a + variable in `minitrino.cfg` and uses for clients if present. Returns a tuple of DockerClient and APIClient objects, respectiveley. If there is an error fetching the clients, None types will be returned @@ -203,7 +203,7 @@ def _get_docker_clients(self): class EnvironmentVariables: - """Gathers all Minipresto variables into a single source of truth. + """Gathers all Minitrino variables into a single source of truth. ### Parameters - `ctx`: Instantiated Environment object (with user input already accounted @@ -219,7 +219,7 @@ class EnvironmentVariables: ### Usage ```python # ctx object has an instantiated EnvironmentVariables object - env_variable = ctx.env.get_var("STARBURST_VER", "338-e") + env_variable = ctx.env.get_var("STARBURST_VER", "354-e") env_section = ctx.env.get_section("MODULES") ```""" @@ -232,7 +232,7 @@ def __init__(self, ctx=None): self.env = {} self._ctx = ctx - self._parse_minipresto_config() + self._parse_minitrino_config() self._parse_user_env() def get_var(self, key="", default=None): @@ -266,8 +266,8 @@ def get_section(self, section=""): return self.env.get(section.upper(), {}) - def _parse_minipresto_config(self): - """Parses the Minipresto config file and adds it to the env + def _parse_minitrino_config(self): + """Parses the Minitrino config file and adds it to the env dictionary.""" if not os.path.isfile(self._ctx.config_file): @@ -295,20 +295,20 @@ def _parse_minipresto_config(self): ) def _parse_library_env(self): - """Parses the Minipresto library's root `minipresto.env` file. All config from + """Parses the Minitrino library's root `minitrino.env` file. All config from this file is added to the 'MODULES' section of the environment dictionary since this file explicitly defines the versions of the module services.""" - env_file = os.path.join(self._ctx.minipresto_lib_dir, "minipresto.env") + env_file = os.path.join(self._ctx.minitrino_lib_dir, "minitrino.env") if not os.path.isfile(env_file): raise err.UserError( - f"Library 'minipresto.env' file does not exist at path: {env_file}", - f"Are you pointing to a valid library, and is the minipresto.env file " + f"Library 'minitrino.env' file does not exist at path: {env_file}", + f"Are you pointing to a valid library, and is the minitrino.env file " f"present in that library?", ) - # Check if modules section was added from Minipresto config file parsing + # Check if modules section was added from Minitrino config file parsing section = self.env.get("MODULES", None) if not isinstance(section, dict): self.env["MODULES"] = {} @@ -349,7 +349,7 @@ def _parse_user_env(self): new_dict = {} for section_k, section_v in self.env.items(): if not isinstance(section_v, dict): - raise err.MiniprestoError( + raise err.MinitrinoError( f"Invalid environment dictionary. Expected nested dictionaries. " f"Received dictionary:\n" f"{json.dumps(self.env, indent=2)}" @@ -382,7 +382,7 @@ def _log_env_vars(self): class Modules: - """Contains information about all valid Minipresto modules. + """Contains information about all valid Minitrino modules. ### Parameters - `ctx`: Instantiated Environment object (with user input already accounted @@ -418,9 +418,9 @@ def get_running_modules(self): if not containers: return {} - # Remove Presto container since it isn't a module + # Remove Trino container since it isn't a module for i, container in enumerate(containers): - if container.name == "presto": + if container.name == "trino": del containers[i] names = [] @@ -435,9 +435,9 @@ def get_running_modules(self): else: continue label_set[k] = v - if not label_set and container.name != "presto": + if not label_set and container.name != "trino": raise err.UserError( - f"Missing Minipresto labels for container '{container.name}'.", + f"Missing Minitrino labels for container '{container.name}'.", f"Check this module's 'docker-compose.yml' file and ensure you are " f"following the documentation on labels.", ) @@ -449,7 +449,7 @@ def get_running_modules(self): raise err.UserError( f"Module '{name}' is running, but it is not found " f"in the library. Was it deleted, or are you pointing " - f"Minipresto to the wrong location?" + f"Minitrino to the wrong location?" ) if not running.get(name, False): running[name] = self.data[name] @@ -467,11 +467,11 @@ def _load_modules(self): self._ctx.logger.log("Loading modules...", level=self._ctx.logger.verbose) - modules_dir = os.path.join(self._ctx.minipresto_lib_dir, MODULE_ROOT) + modules_dir = os.path.join(self._ctx.minitrino_lib_dir, MODULE_ROOT) if not os.path.isdir(modules_dir): - raise err.MiniprestoError( + raise err.MinitrinoError( f"Path is not a directory: {modules_dir}. " - f"Are you pointing to a compatible Minipresto library?" + f"Are you pointing to a compatible Minitrino library?" ) # Loop through both catalog and security modules @@ -532,13 +532,8 @@ def _load_modules(self): f"Will not load metadata for module.", level=self._ctx.logger.verbose, ) - - self.data[module_name]["description"] = metadata.get( - "description", "No module description provided." - ) - self.data[module_name]["incompatible_modules"] = metadata.get( - "incompatible_modules", [] - ) + for k, v in metadata.items(): + self.data[module_name][k] = v class CommandExecutor: @@ -643,7 +638,7 @@ def _execute_in_shell(self, command="", **kwargs): output, _ = process.communicate() # Get full output (stdout + stderr) if process.returncode != 0 and kwargs.get("trigger_error", True): - raise err.MiniprestoError( + raise err.MinitrinoError( f"Failed to execute shell command:\n{command}\n" f"Exit code: {process.returncode}" ) @@ -660,7 +655,7 @@ def _execute_in_container(self, command="", **kwargs): container = kwargs.get("container", None) if container is None: - raise err.MiniprestoError( + raise err.MinitrinoError( f"Attempted to execute a command inside of a " f"container, but a container object was not provided." ) @@ -723,7 +718,7 @@ def _execute_in_container(self, command="", **kwargs): ) if return_code != 0 and kwargs.get("trigger_error", True): - raise err.MiniprestoError( + raise err.MinitrinoError( f"Failed to execute command in container '{container.name}':\n{command}\n" f"Exit code: {return_code}" ) @@ -737,7 +732,7 @@ def _construct_environment(self, environment={}, container=None): host environment will be set to the container's existing environment variables.""" - # Remove conflicting keys from host environment; Minipresto environment + # Remove conflicting keys from host environment; Minitrino environment # variables take precendance if not container: diff --git a/cli/minipresto/errors.py b/cli/minitrino/errors.py similarity index 78% rename from cli/minipresto/errors.py rename to cli/minitrino/errors.py index ee3ca28b..42103a0c 100644 --- a/cli/minipresto/errors.py +++ b/cli/minitrino/errors.py @@ -1,11 +1,11 @@ #!usr/bin/env/python3 # -*- coding: utf-8 -*- -from minipresto import utils +from minitrino import utils -class MiniprestoError(Exception): - """Generic Minipresto exception class.""" +class MinitrinoError(Exception): + """Generic Minitrino exception class.""" exit_code = 1 @@ -19,8 +19,8 @@ def __str__(self): return self.msg -class UserError(MiniprestoError): - """An exception that Minipresto can handle and show to the user. +class UserError(MinitrinoError): + """An exception that Minitrino can handle and show to the user. ### Parameters - `msg`: The message to log, and the raised exception's message. diff --git a/cli/minipresto/settings.py b/cli/minitrino/settings.py similarity index 75% rename from cli/minipresto/settings.py rename to cli/minitrino/settings.py index 6d83ed41..d9749f3c 100644 --- a/cli/minipresto/settings.py +++ b/cli/minitrino/settings.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # Docker labels -RESOURCE_LABEL = "com.starburst.tests=minipresto" +RESOURCE_LABEL = "com.starburst.tests=minitrino" MODULE_LABEL_KEY_ROOT = "com.starburst.tests.module" # Generic Constants @@ -14,13 +14,13 @@ MODULE_CATALOG = "catalog" MODULE_SECURITY = "security" MODULE_RESOURCES = "resources" -ETC_PRESTO = "/usr/lib/presto/etc" -PRESTO_CONFIG = "config.properties" -PRESTO_JVM_CONFIG = "jvm.config" +ETC_TRINO = "/etc/starburst" +TRINO_CONFIG = "config.properties" +TRINO_JVM_CONFIG = "jvm.config" LIB_INDEPENDENT_CMDS = ["lib_install"] # Snapshots -SNAPSHOT_ROOT_FILES = ["docker-compose.yml", "minipresto.env", "Dockerfile"] +SNAPSHOT_ROOT_FILES = ["docker-compose.yml", "minitrino.env", "Dockerfile", "version"] # Terminal DEFAULT_INDENT = " " * 5 @@ -44,7 +44,7 @@ [DOCKER] DOCKER_HOST= -[PRESTO] +[TRINO] CONFIG= JVM_CONFIG= @@ -78,9 +78,9 @@ # Below is the exact command used to provision the snapshotted environment. Run this # command in your terminal to reproduce the exact state of the environment. # -# If you need config data from the snapshot's 'minipresto.cfg' file, you will either -# need to copy it from the snapshot directory to '~./minipresto/minipresto.cfg' or -# individually copy the needed configs to your existing 'minipresto.cfg' file. +# If you need config data from the snapshot's 'minitrino.cfg' file, you will either +# need to copy it from the snapshot directory to '~./minitrino/minitrino.cfg' or +# individually copy the needed configs to your existing 'minitrino.cfg' file. # ------------------------------------------------------------------------------------ diff --git a/cli/minipresto/test/__init__.py b/cli/minitrino/test/__init__.py similarity index 100% rename from cli/minipresto/test/__init__.py rename to cli/minitrino/test/__init__.py diff --git a/cli/minipresto/test/helpers.py b/cli/minitrino/test/helpers.py similarity index 81% rename from cli/minipresto/test/helpers.py rename to cli/minitrino/test/helpers.py index cccc4d83..b84c57f4 100644 --- a/cli/minipresto/test/helpers.py +++ b/cli/minitrino/test/helpers.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import os +import re import docker import time import sys @@ -11,23 +12,23 @@ from datetime import datetime from click.testing import CliRunner from pathlib import Path -from minipresto.cli import cli +from minitrino.cli import cli # Path references # ----------------------------------------------------------------------------------- USER_HOME_DIR = os.path.expanduser("~") -MINIPRESTO_USER_DIR = os.path.abspath(os.path.join(USER_HOME_DIR, ".minipresto")) -CONFIG_FILE = os.path.abspath(os.path.join(MINIPRESTO_USER_DIR, "minipresto.cfg")) -MINIPRESTO_LIB_DIR = Path(os.path.abspath(__file__)).resolve().parents[3] -SNAPSHOT_DIR = os.path.join(MINIPRESTO_LIB_DIR, "lib", "snapshots") +MINITRINO_USER_DIR = os.path.abspath(os.path.join(USER_HOME_DIR, ".minitrino")) +CONFIG_FILE = os.path.abspath(os.path.join(MINITRINO_USER_DIR, "minitrino.cfg")) +MINITRINO_LIB_DIR = Path(os.path.abspath(__file__)).resolve().parents[3] +SNAPSHOT_DIR = os.path.join(MINITRINO_LIB_DIR, "lib", "snapshots") SNAPSHOT_FILE = os.path.join(SNAPSHOT_DIR, "test.tar.gz") -MINIPRESTO_USER_SNAPSHOTS_DIR = os.path.join(MINIPRESTO_USER_DIR, "snapshots") +MINITRINO_USER_SNAPSHOTS_DIR = os.path.join(MINITRINO_USER_DIR, "snapshots") # ----------------------------------------------------------------------------------- -class MiniprestoResult: +class MinitrinoResult: def __init__(self, click_result, output, exit_code): - """Result class containing information about the result of a Minipresto + """Result class containing information about the result of a Minitrino command. Attributes @@ -50,10 +51,12 @@ def execute_command(command=[], print_output=True, command_input=""): else: result = runner.invoke(cli, command, input=command_input) if print_output: - print(f"Output of command [minipresto {' '.join(command)}]:\n{result.output}") + print(f"Output of command [minitrino {' '.join(command)}]:\n{result.output}") - # Remove newlines for string assertion consistency - return MiniprestoResult(result, result.output.replace("\n", " "), result.exit_code) + # Remove newlines and extra spaces for string assertion consistency + output = result.output.replace("\n", "") + output = re.sub(" +", " ", output) + return MinitrinoResult(result, output, result.exit_code) def log_success(msg): @@ -96,12 +99,12 @@ def start_docker_daemon(): if return_code != 0: raise Exception("Failed to start Docker daemon.") - docker_client = docker.from_env() counter = 0 while counter < 61: if counter == 61: raise Exception("Docker daemon failed to start after one minute.") try: + docker_client = docker.from_env() docker_client.ping() break except: diff --git a/cli/minitrino/test/runner.py b/cli/minitrino/test/runner.py new file mode 100644 index 00000000..629b3bd0 --- /dev/null +++ b/cli/minitrino/test/runner.py @@ -0,0 +1,30 @@ +#!usr/bin/env/python3 +# -*- coding: utf-8 -*- + +import minitrino.test.test_misc as test_misc +import minitrino.test.test_cmd_config as test_config +import minitrino.test.test_cmd_down as test_down +import minitrino.test.test_cmd_provision as test_provision +import minitrino.test.test_cmd_remove as test_remove +import minitrino.test.test_cmd_snapshot as test_snapshot +import minitrino.test.test_cmd_version as test_version +import minitrino.test.test_cmd_modules as test_modules +import minitrino.test.test_cmd_lib_install as test_cmd_lib_install + + +def main(): + """Minitrino unit test runner.""" + + test_misc.main() + # test_config.main() + test_down.main() + test_provision.main() + test_remove.main() + test_snapshot.main() + test_version.main() + test_modules.main() + test_cmd_lib_install.main() + + +if __name__ == "__main__": + main() diff --git a/cli/minipresto/test/test_cmd_config.py b/cli/minitrino/test/test_cmd_config.py similarity index 76% rename from cli/minipresto/test/test_cmd_config.py rename to cli/minitrino/test/test_cmd_config.py index f17918ec..5af07e5b 100644 --- a/cli/minipresto/test/test_cmd_config.py +++ b/cli/minitrino/test/test_cmd_config.py @@ -1,14 +1,14 @@ #!usr/bin/env/python3 # -*- coding: utf-8 -*- -# 1. Test with no Minipresto directory -# 2. Test with no Minipresto config file (ensure template is created) +# 1. Test with no Minitrino directory +# 2. Test with no Minitrino config file (ensure template is created) # 3. Test reset w/ existing config dir and file (ensure template is created) # 4. Test editing an invalid config file import os import subprocess -import minipresto.test.helpers as helpers +import minitrino.test.helpers as helpers from inspect import currentframe from types import FrameType @@ -32,11 +32,11 @@ def test_no_directory(): helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) - subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_DIR}", shell=True) - return_code = subprocess.call("minipresto config", shell=True) + subprocess.call(f"rm -rf {helpers.MINITRINO_USER_DIR}", shell=True) + return_code = subprocess.call("minitrino config", shell=True) assert return_code == 0 - assert os.path.isdir(helpers.MINIPRESTO_USER_DIR) + assert os.path.isdir(helpers.MINITRINO_USER_DIR) assert os.path.isfile(helpers.CONFIG_FILE) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) @@ -52,7 +52,7 @@ def test_reset_with_directory(): helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) subprocess.call( - f"mkdir {helpers.MINIPRESTO_USER_DIR}", shell=True, stdout=subprocess.DEVNULL + f"mkdir {helpers.MINITRINO_USER_DIR}", shell=True, stdout=subprocess.DEVNULL ) start_time = time.time() @@ -60,7 +60,7 @@ def test_reset_with_directory(): output = "" while time.time() - start_time <= end_time: process = subprocess.Popen( - "minipresto -v config --reset", + "minitrino -v config --reset", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -76,7 +76,7 @@ def test_reset_with_directory(): break process = subprocess.Popen( - "minipresto -v config --reset", + "minitrino -v config --reset", stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -90,11 +90,11 @@ def test_reset_with_directory(): assert all( ( "Configuration directory exists" in output, - "Created Minipresto configuration directory" in output, + "Created Minitrino configuration directory" in output, "Opening existing config file at path" in output, ) ) - assert os.path.isdir(helpers.MINIPRESTO_USER_DIR) + assert os.path.isdir(helpers.MINITRINO_USER_DIR) assert os.path.isfile(helpers.CONFIG_FILE) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) @@ -108,10 +108,10 @@ def test_edit_invalid_config(): helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) - subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_DIR}", shell=True) - subprocess.call(f"mkdir {helpers.MINIPRESTO_USER_DIR}", shell=True) + subprocess.call(f"rm -rf {helpers.MINITRINO_USER_DIR}", shell=True) + subprocess.call(f"mkdir {helpers.MINITRINO_USER_DIR}", shell=True) subprocess.call(f"touch {helpers.CONFIG_FILE}", shell=True) - return_code = subprocess.call(f"minipresto config", shell=True) + return_code = subprocess.call(f"minitrino config", shell=True) assert return_code == 0 helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) @@ -121,10 +121,10 @@ def test_edit_valid_config(): helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) - subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_DIR}", shell=True) - subprocess.call(f"mkdir {helpers.MINIPRESTO_USER_DIR}", shell=True) + subprocess.call(f"rm -rf {helpers.MINITRINO_USER_DIR}", shell=True) + subprocess.call(f"mkdir {helpers.MINITRINO_USER_DIR}", shell=True) helpers.make_sample_config() - return_code = subprocess.call(f"minipresto config", shell=True) + return_code = subprocess.call(f"minitrino config", shell=True) assert return_code == 0 helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) diff --git a/cli/minipresto/test/test_cmd_down.py b/cli/minitrino/test/test_cmd_down.py similarity index 92% rename from cli/minipresto/test/test_cmd_down.py rename to cli/minitrino/test/test_cmd_down.py index ed8706cd..16826e09 100644 --- a/cli/minipresto/test/test_cmd_down.py +++ b/cli/minitrino/test/test_cmd_down.py @@ -2,12 +2,12 @@ # -*- coding: utf-8 -*- import docker -import minipresto.test.helpers as helpers +import minitrino.test.helpers as helpers from inspect import currentframe from types import FrameType from typing import cast -from minipresto.settings import RESOURCE_LABEL +from minitrino.settings import RESOURCE_LABEL def main(): @@ -53,7 +53,7 @@ def test_running_containers(): "Stopped container" in result.output, "Removed container" in result.output, "test" in result.output, - "presto" in result.output, + "trino" in result.output, ) ) @@ -82,7 +82,7 @@ def test_keep(): ) for container in containers: - assert container.name.lower() == "presto" or container.name.lower() == "test" + assert container.name.lower() == "trino" or container.name.lower() == "test" helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup() diff --git a/cli/minipresto/test/test_cmd_lib_install.py b/cli/minitrino/test/test_cmd_lib_install.py similarity index 75% rename from cli/minipresto/test/test_cmd_lib_install.py rename to cli/minitrino/test/test_cmd_lib_install.py index 55cfa6af..057dc85b 100644 --- a/cli/minipresto/test/test_cmd_lib_install.py +++ b/cli/minitrino/test/test_cmd_lib_install.py @@ -3,7 +3,7 @@ import os import subprocess -import minipresto.test.helpers as helpers +import minitrino.test.helpers as helpers from inspect import currentframe from types import FrameType @@ -18,7 +18,7 @@ def main(): def test_install(): - """Verifies that the Minipresto library can be installed.""" + """Verifies that the Minitrino library can be installed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() @@ -27,13 +27,13 @@ def test_install(): result = helpers.execute_command(["-v", "lib_install", "--version", "0.0.0"]) assert result.exit_code == 0 - assert os.path.isdir(os.path.join(helpers.MINIPRESTO_USER_DIR, "lib")) + assert os.path.isdir(os.path.join(helpers.MINITRINO_USER_DIR, "lib")) helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) def test_install_overwrite(): - """Verifies that the Minipresto library can be installed over an existing + """Verifies that the Minitrino library can be installed over an existing library.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -43,7 +43,7 @@ def test_install_overwrite(): ) assert result.exit_code == 0 - assert os.path.isdir(os.path.join(helpers.MINIPRESTO_USER_DIR, "lib")) + assert os.path.isdir(os.path.join(helpers.MINITRINO_USER_DIR, "lib")) assert "Removing existing library directory" in result.output cleanup() @@ -57,10 +57,10 @@ def test_invalid_ver(): helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) - result = helpers.execute_command(["-v", "lib_install", "--version", "YEE-PRESTO"]) + result = helpers.execute_command(["-v", "lib_install", "--version", "YEE-TRINO"]) assert result.exit_code == 1 - assert not os.path.isdir(os.path.join(helpers.MINIPRESTO_USER_DIR, "lib")) + assert not os.path.isdir(os.path.join(helpers.MINITRINO_USER_DIR, "lib")) cleanup() @@ -68,7 +68,7 @@ def test_invalid_ver(): def cleanup(): - subprocess.call("rm -rf ~/.minipresto/", shell=True) + subprocess.call("rm -rf ~/.minitrino/", shell=True) if __name__ == "__main__": diff --git a/cli/minipresto/test/test_cmd_modules.py b/cli/minitrino/test/test_cmd_modules.py similarity index 94% rename from cli/minipresto/test/test_cmd_modules.py rename to cli/minitrino/test/test_cmd_modules.py index 2d6e83de..2388528d 100644 --- a/cli/minipresto/test/test_cmd_modules.py +++ b/cli/minitrino/test/test_cmd_modules.py @@ -3,7 +3,7 @@ import os import subprocess -import minipresto.test.helpers as helpers +import minitrino.test.helpers as helpers from inspect import currentframe from types import FrameType @@ -20,7 +20,7 @@ def main(): def test_invalid_module(): - """Ensures Minipresto exists with a user error if an invalid module name is + """Ensures Minitrino exists with a user error if an invalid module name is provided.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -59,7 +59,7 @@ def test_all_modules(): ( "Module: test" in result.output, "Description:" in result.output, - "Incompatible Modules:" in result.output, + "Incompatiblemodules:" in result.output, ) ) diff --git a/cli/minipresto/test/test_cmd_provision.py b/cli/minitrino/test/test_cmd_provision.py similarity index 81% rename from cli/minipresto/test/test_cmd_provision.py rename to cli/minitrino/test/test_cmd_provision.py index 9b877529..ed99bc77 100644 --- a/cli/minipresto/test/test_cmd_provision.py +++ b/cli/minitrino/test/test_cmd_provision.py @@ -2,18 +2,18 @@ # -*- coding: utf-8 -*- # TODO: Test no rollback -# TODO: Test invalid user config (Presto/JVM) +# TODO: Test invalid user config (Trino/JVM) import os import docker import time import subprocess -import minipresto.test.helpers as helpers +import minitrino.test.helpers as helpers from inspect import currentframe from types import FrameType from typing import cast -from minipresto.settings import RESOURCE_LABEL +from minitrino.settings import RESOURCE_LABEL def main(): @@ -30,7 +30,7 @@ def main(): def test_standalone(): - """Verifies that a standalone Presto container is provisioned when no + """Verifies that a standalone Trino container is provisioned when no options are passed in.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -44,7 +44,7 @@ def test_standalone(): assert len(containers) == 1 for container in containers: - assert container.name == "presto" + assert container.name == "trino" helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup() @@ -100,13 +100,13 @@ def test_bootstrap_script(result): assert all( ( - "Successfully executed bootstrap script in container: 'presto'", + "Successfully executed bootstrap script in container: 'trino'", "Successfully executed bootstrap script in container: 'test'", ) ) - presto_bootstrap_check = subprocess.Popen( - f"docker exec -i presto ls /usr/lib/presto/etc/", + trino_bootstrap_check = subprocess.Popen( + f"docker exec -i trino ls /etc/starburst/", shell=True, stdout=subprocess.PIPE, universal_newlines=True, @@ -118,10 +118,10 @@ def test_bootstrap_script(result): universal_newlines=True, ) - presto_bootstrap_check, _ = presto_bootstrap_check.communicate() + trino_bootstrap_check, _ = trino_bootstrap_check.communicate() test_bootstrap_check, _ = test_bootstrap_check.communicate() - assert "test_bootstrap.txt" in presto_bootstrap_check + assert "test_bootstrap.txt" in trino_bootstrap_check assert "hello world" in test_bootstrap_check helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) @@ -138,7 +138,7 @@ def test_bootstrap_re_execute(): assert result.exit_code == 0 assert all( ( - "Bootstrap already executed in container 'presto'. Skipping.", + "Bootstrap already executed in container 'trino'. Skipping.", "Bootstrap already executed in container 'test'. Skipping.", ) ) @@ -147,8 +147,8 @@ def test_bootstrap_re_execute(): def test_valid_user_config(): - """Ensures that valid, user-defined Presto/JVM config can be successfully - appended to Presto config files.""" + """Ensures that valid, user-defined Trino/JVM config can be successfully + appended to Trino config files.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -167,31 +167,30 @@ def test_valid_user_config(): assert result.exit_code == 0 assert ( - "Appending user-defined Presto config to Presto container config" - in result.output + "Appending user-defined Trino config to Trino container config" in result.output ) jvm_config = subprocess.Popen( - f"docker exec -i presto cat /usr/lib/presto/etc/jvm.config", + f"docker exec -i trino cat /etc/starburst/jvm.config", shell=True, stdout=subprocess.PIPE, universal_newlines=True, ) - presto_config = subprocess.Popen( - f"docker exec -i presto cat /usr/lib/presto/etc/config.properties", + trino_config = subprocess.Popen( + f"docker exec -i trino cat /etc/starburst/config.properties", shell=True, stdout=subprocess.PIPE, universal_newlines=True, ) jvm_config, _ = jvm_config.communicate() - presto_config, _ = presto_config.communicate() + trino_config, _ = trino_config.communicate() assert all(("-Xmx2G" in jvm_config, "-Xms1G" in jvm_config)) assert all( ( - "query.max-stage-count=85" in presto_config, - "query.max-execution-time=1h" in presto_config, + "query.max-stage-count=85" in trino_config, + "query.max-execution-time=1h" in trino_config, ) ) @@ -200,7 +199,7 @@ def test_valid_user_config(): def test_duplicate_config_props(): - """Ensures that duplicate configuration properties in Presto are logged as a + """Ensures that duplicate configuration properties in Trino are logged as a warning to the user.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -212,7 +211,7 @@ def test_duplicate_config_props(): f"\nquery.max-execution-time=1h\nquery.max-execution-time=2h'" ) subprocess.Popen( - f'docker exec -i presto sh -c "echo {cmd_chunk} >> /usr/lib/presto/etc/config.properties"', + f'docker exec -i trino sh -c "echo {cmd_chunk} >> /etc/starburst/config.properties"', shell=True, stdout=subprocess.PIPE, universal_newlines=True, @@ -220,7 +219,7 @@ def test_duplicate_config_props(): cmd_chunk = "$'-Xms1G\n-Xms1G'" subprocess.Popen( - f'docker exec -i presto sh -c "echo {cmd_chunk} >> /usr/lib/presto/etc/jvm.config"', + f'docker exec -i trino sh -c "echo {cmd_chunk} >> /etc/starburst/jvm.config"', shell=True, stdout=subprocess.PIPE, universal_newlines=True, @@ -234,13 +233,13 @@ def test_duplicate_config_props(): assert all( ( - "Duplicate Presto configuration properties detected in config.properties" + "Duplicate Trino configuration properties detected in config.properties" in result.output, "query.max-stage-count=85" in result.output, "query.max-stage-count=100" in result.output, "query.max-execution-time=1h" in result.output, "query.max-execution-time=2h" in result.output, - "Duplicate Presto configuration properties detected in jvm.config" + "Duplicate Trino configuration properties detected in jvm.config" in result.output, "-Xms1G" in result.output, "-Xms1G" in result.output, @@ -284,14 +283,14 @@ def test_provision_append(): assert result.exit_code == 0 assert "Identified the following running modules" in result.output - assert len(containers) == 3 # presto, test, and postgres + assert len(containers) == 3 # trino, test, and postgres helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) cleanup() def get_containers(): - """Returns all running minipresto containers.""" + """Returns all running minitrino containers.""" docker_client = docker.from_env() return docker_client.containers.list(filters={"label": RESOURCE_LABEL}) diff --git a/cli/minipresto/test/test_cmd_remove.py b/cli/minitrino/test/test_cmd_remove.py similarity index 95% rename from cli/minipresto/test/test_cmd_remove.py rename to cli/minitrino/test/test_cmd_remove.py index c26137fd..c60347e8 100644 --- a/cli/minipresto/test/test_cmd_remove.py +++ b/cli/minitrino/test/test_cmd_remove.py @@ -3,9 +3,9 @@ import docker import subprocess -import minipresto.test.helpers as helpers +import minitrino.test.helpers as helpers -from minipresto.settings import RESOURCE_LABEL +from minitrino.settings import RESOURCE_LABEL from inspect import currentframe from types import FrameType from typing import cast @@ -29,7 +29,7 @@ def main(): def test_images(): - """Verifies that images with the standard Minipresto label applied to them + """Verifies that images with the standard Minitrino label applied to them are removed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -54,7 +54,7 @@ def test_images(): def test_volumes(): - """Verifies that volumes with the standard Minipresto label applied to them + """Verifies that volumes with the standard Minitrino label applied to them are removed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -106,7 +106,7 @@ def test_label(): }, { "resource_type": docker_client.images, - "label": "com.starburst.tests.module=presto", + "label": "com.starburst.tests.module=trino", "expected_count": 1, }, ) @@ -155,7 +155,7 @@ def test_multiple_labels(): def test_invalid_label(): - """Verifies that images with the Minipresto label applied to them are + """Verifies that images with the Minitrino label applied to them are removed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -182,7 +182,7 @@ def test_invalid_label(): def test_all(): - """Verifies that all Minipresto resources are removed.""" + """Verifies that all Minitrino resources are removed.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) diff --git a/cli/minipresto/test/test_cmd_snapshot.py b/cli/minitrino/test/test_cmd_snapshot.py similarity index 91% rename from cli/minipresto/test/test_cmd_snapshot.py rename to cli/minitrino/test/test_cmd_snapshot.py index 6a1090cb..9ceb80e3 100644 --- a/cli/minipresto/test/test_cmd_snapshot.py +++ b/cli/minitrino/test/test_cmd_snapshot.py @@ -5,7 +5,7 @@ import docker import pathlib import subprocess -import minipresto.test.helpers as helpers +import minitrino.test.helpers as helpers from inspect import currentframe from types import FrameType @@ -14,7 +14,7 @@ def snapshot_test_yaml_file(snapshot_name="test"): return os.path.join( - helpers.MINIPRESTO_USER_SNAPSHOTS_DIR, + helpers.MINITRINO_USER_SNAPSHOTS_DIR, snapshot_name, "lib", "modules", @@ -26,7 +26,7 @@ def snapshot_test_yaml_file(snapshot_name="test"): def snapshot_config_file(snapshot_name="test"): return os.path.join( - helpers.MINIPRESTO_USER_SNAPSHOTS_DIR, snapshot_name, "minipresto.cfg" + helpers.MINITRINO_USER_SNAPSHOTS_DIR, snapshot_name, "minitrino.cfg" ) @@ -49,12 +49,12 @@ def main(): def test_snapshot_no_directory(): """Verifies that a snapshot can be created when there is no existing - snapshots directory in the Minipresto user home directory.""" + snapshots directory in the Minitrino user home directory.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) cleanup() - subprocess.call(f"rm -rf {helpers.MINIPRESTO_USER_SNAPSHOTS_DIR}", shell=True) + subprocess.call(f"rm -rf {helpers.MINITRINO_USER_SNAPSHOTS_DIR}", shell=True) result = helpers.execute_command( ["-v", "snapshot", "--name", "test", "--module", "test"], command_input="y\n", @@ -85,7 +85,7 @@ def test_snapshot_active_env(): def test_snapshot_standalone(): - """Verifies that a the standlone Presto module can be snapshotted.""" + """Verifies that a the standlone Trino module can be snapshotted.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -96,7 +96,7 @@ def test_snapshot_standalone(): ) run_assertions(result, False) - assert "Snapshotting Presto resources only" in result.output + assert "Snapshotting Trino resources only" in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) @@ -218,7 +218,7 @@ def test_command_snapshot_file(): helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) command_snapshot_file = os.path.join( - helpers.MINIPRESTO_USER_SNAPSHOTS_DIR, "test", "provision-snapshot.sh" + helpers.MINITRINO_USER_SNAPSHOTS_DIR, "test", "provision-snapshot.sh" ) process = subprocess.Popen( command_snapshot_file, @@ -306,11 +306,11 @@ def run_assertions( assert "Snapshot complete" in result.output assert result.exit_code == 0 - if os.path.isfile(os.path.join(helpers.MINIPRESTO_USER_DIR, "minipresto.cfg")): + if os.path.isfile(os.path.join(helpers.MINITRINO_USER_DIR, "minitrino.cfg")): assert os.path.isfile(snapshot_config_file(snapshot_name)) command_snapshot_file = os.path.join( - helpers.MINIPRESTO_USER_SNAPSHOTS_DIR, snapshot_name, "provision-snapshot.sh" + helpers.MINITRINO_USER_SNAPSHOTS_DIR, snapshot_name, "provision-snapshot.sh" ) assert os.path.isfile(command_snapshot_file) @@ -318,7 +318,7 @@ def run_assertions( assert os.path.isfile(os.path.join(check_path, f"{snapshot_name}.tar.gz")) with open(command_snapshot_file) as f: - assert "minipresto -v --env LIB_PATH=" in f.read() + assert "minitrino -v --env LIB_PATH=" in f.read() def cleanup(snapshot_name="test"): @@ -326,7 +326,7 @@ def cleanup(snapshot_name="test"): if not snapshot_name == "test": subprocess.call( - f"rm -rf {os.path.join(helpers.MINIPRESTO_USER_SNAPSHOTS_DIR, snapshot_name)}.tar.gz", + f"rm -rf {os.path.join(helpers.MINITRINO_USER_SNAPSHOTS_DIR, snapshot_name)}.tar.gz", shell=True, ) else: diff --git a/cli/minipresto/test/test_cmd_version.py b/cli/minitrino/test/test_cmd_version.py similarity index 81% rename from cli/minipresto/test/test_cmd_version.py rename to cli/minitrino/test/test_cmd_version.py index df57aaa2..8cd62ec5 100644 --- a/cli/minipresto/test/test_cmd_version.py +++ b/cli/minitrino/test/test_cmd_version.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- -import minipresto.test.helpers as helpers +import minitrino.test.helpers as helpers import pkg_resources from inspect import currentframe @@ -21,7 +21,7 @@ def test_version(): helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) result = helpers.execute_command(["version"]) - assert pkg_resources.require("Minipresto")[0].version in result.output + assert pkg_resources.require("Minitrino")[0].version in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) diff --git a/cli/minipresto/test/test_misc.py b/cli/minitrino/test/test_misc.py similarity index 86% rename from cli/minipresto/test/test_misc.py rename to cli/minitrino/test/test_misc.py index 7a0215e7..357c536d 100644 --- a/cli/minipresto/test/test_misc.py +++ b/cli/minitrino/test/test_misc.py @@ -4,7 +4,7 @@ # TODO: Test docker host # TODO: Test symlink paths (should work with os.environ() registered in subproc) -import minipresto.test.helpers as helpers +import minitrino.test.helpers as helpers from inspect import currentframe from types import FrameType @@ -32,7 +32,7 @@ def main(): def test_daemon_off_all(*args): - """Verifies that each Minipresto command properly exits properly if the + """Verifies that each Minitrino command properly exits properly if the Docker daemon is off or unresponsive.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -82,9 +82,9 @@ def test_multiple_env(): "--env", "COMPOSE_PROJECT_NAME=test", "--env", - "STARBURST_VER=338-e.1", + "STARBURST_VER=354-e", "--env", - "PRESTO=is=awesome", + "TRINO=is=awesome", "version", ] ) @@ -93,8 +93,8 @@ def test_multiple_env(): assert all( ( '"COMPOSE_PROJECT_NAME": "test"' in result.output, - '"STARBURST_VER": "338-e.1"' in result.output, - '"PRESTO": "is=awesome"' in result.output, + '"STARBURST_VER": "354-e"' in result.output, + '"TRINO": "is=awesome"' in result.output, ) ) @@ -123,7 +123,7 @@ def test_invalid_env(): def test_invalid_lib(): - """Verifies that Minipresto exists with a user error if pointing to an + """Verifies that Minitrino exists with a user error if pointing to an invalid library.""" helpers.log_status(cast(FrameType, currentframe()).f_code.co_name) @@ -132,7 +132,7 @@ def test_invalid_lib(): result = helpers.execute_command(["-v", "--env", "LIB_PATH=/tmp/", "modules"]) assert result.exit_code == 2 - assert "You must provide a path to a compatible Minipresto library" in result.output + assert "You must provide a path to a compatible Minitrino library" in result.output # Fake directory result = helpers.execute_command( @@ -140,7 +140,7 @@ def test_invalid_lib(): ) assert result.exit_code == 2 - assert "You must provide a path to a compatible Minipresto library" in result.output + assert "You must provide a path to a compatible Minitrino library" in result.output helpers.log_success(cast(FrameType, currentframe()).f_code.co_name) diff --git a/cli/minipresto/utils.py b/cli/minitrino/utils.py similarity index 89% rename from cli/minipresto/utils.py rename to cli/minitrino/utils.py index ed6f689f..6bbcbcc8 100644 --- a/cli/minipresto/utils.py +++ b/cli/minitrino/utils.py @@ -6,8 +6,8 @@ import traceback import pkg_resources -from minipresto import errors as err -from minipresto.settings import DEFAULT_INDENT +from minitrino import errors as err +from minitrino.settings import DEFAULT_INDENT from click import echo, style, prompt from textwrap import fill @@ -16,7 +16,7 @@ class Logger: - """Minipresto logging class. The logger does not log to a file; it uses + """Minitrino logging class. The logger does not log to a file; it uses `click.echo()` to print text to the user's terminal. The log level will affect the prefix color (i.e. the '[i]' in info messages @@ -69,7 +69,7 @@ def log(self, *args, level=None, stream=False): try: msg = str(msg) except: - raise err.MiniprestoError( + raise err.MinitrinoError( f"A string is required for {self.log.__name__}." ) msgs = msg.replace("\r", "\n").split("\n") @@ -101,7 +101,7 @@ def prompt_msg(self, msg="", input_type=str): try: msg = str(msg) except: - raise err.MiniprestoError(f"A string is required for {self.log.__name__}.") + raise err.MinitrinoError(f"A string is required for {self.log.__name__}.") msg = self._format(msg) styled_prefix = style( @@ -124,11 +124,11 @@ def _format(self, msg): msg = msg.replace("\n", f"\n{DEFAULT_INDENT}") msg = fill( msg, - terminal_width, + terminal_width - 4, subsequent_indent=DEFAULT_INDENT, replace_whitespace=False, break_on_hyphens=False, - break_long_words=False, + break_long_words=True, ) return msg @@ -153,14 +153,14 @@ def handle_exception(error=Exception, additional_msg="", skip_traceback=False): error_msg = error.msg exit_code = error.exit_code skip_traceback = True - elif isinstance(error, err.MiniprestoError): + elif isinstance(error, err.MinitrinoError): error_msg = error.msg exit_code = error.exit_code elif isinstance(error, Exception): error_msg = str(error) exit_code = 1 else: - raise err.MiniprestoError( + raise err.MinitrinoError( f"Invalid type given to 'e' parameter of {handle_exception.__name__}. " f"Expected an Exception type, but got type {type(error).__name__}" ) @@ -217,7 +217,7 @@ def handle_missing_param(params=[]): if not params: raise handle_missing_param(list(locals().keys())) - return err.MiniprestoError(f"Parameters {params} required to execute function.") + return err.MinitrinoError(f"Parameters {params} required to execute function.") def check_daemon(docker_client): @@ -235,14 +235,14 @@ def check_daemon(docker_client): def check_lib(ctx): - """Checks if a Minipresto library exists.""" + """Checks if a Minitrino library exists.""" - ctx.minipresto_lib_dir + ctx.minitrino_lib_dir def generate_identifier(identifiers=None): """Returns an 'object identifier' string used for creating log messages, - e.g. '[ID: 12345] [Name: presto]'. + e.g. '[ID: 12345] [Name: trino]'. ### Parameters - `identifiers`: Dictionary of "identifier_value": "identifier_key" pairs. @@ -251,7 +251,7 @@ def generate_identifier(identifiers=None): ```python identifier = generate_identifier( {"ID": container.short_id, "Name": container.name} - ) # Will Spit out -> "[ID: 12345] [Name: presto]" + ) # Will Spit out -> "[ID: 12345] [Name: trino]" ```""" if not identifiers: @@ -263,16 +263,16 @@ def generate_identifier(identifiers=None): return " ".join(identifier) -def parse_key_value_pair(key_value_pair, err_type=err.MiniprestoError): +def parse_key_value_pair(key_value_pair, err_type=err.MinitrinoError): """Parses a key-value pair in string form and returns the resulting pair as both a 2-element list. If the string cannot be split by "=", a - MiniprestoError is raised. + MinitrinoError is raised. ### Parameters - `key_value_pair`: A string formatted as a key-value pair, i.e. - `"PRESTO=338-e.0"`. + `"TRINO=354-e"`. - `err_type`: The exception to raise if an "=" delimiter is not in the - key-value pair. Defaults to `MiniprestoError`. + key-value pair. Defaults to `MinitrinoError`. ### Return Values - A list `[k, v]`, but will return `None` if the stripped input is an empty @@ -305,16 +305,16 @@ def parse_key_value_pair(key_value_pair, err_type=err.MiniprestoError): def get_cli_ver(): - """Returns the version of the Minipresto CLI.""" + """Returns the version of the Minitrino CLI.""" - return pkg_resources.require("Minipresto")[0].version + return pkg_resources.require("Minitrino")[0].version def get_lib_ver(library_path=""): - """Returns the version of the Minipresto library. + """Returns the version of the Minitrino library. ### Parameters - - `library_path`: The Minipresto library directory.""" + - `library_path`: The Minitrino library directory.""" version_file = os.path.join(library_path, "version") try: @@ -323,9 +323,9 @@ def get_lib_ver(library_path=""): line = line.strip() if line: return line - return "NOT FOUND" + return "NOT INSTALLED" except: - return "NOT FOUND" + return "NOT INSTALLED" def validate_yes(response=""): diff --git a/cli/setup.py b/cli/setup.py index ed3b532b..758a31e9 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -9,12 +9,12 @@ README = (HERE / "readme.md").read_text() setup( - name="minipresto", - version="1.0.2", - description="A command line tool that makes it easy to run modular Presto environments locally.", + name="minitrino", + version="2.0.0", + description="A command line tool that makes it easy to run modular Trino environments locally.", long_description=README, long_description_content_type="text/markdown", - url="https://github.com/jefflester/minipresto", + url="https://github.com/jefflester/minitrino", author="Jeff Lester", author_email="jeff.lester.dev@gmail.com", license="Apache-2.0", @@ -23,9 +23,9 @@ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ], - keyword="presto, docker, minipresto", + keyword="trino, docker, minitrino", python_requires=">=3.8", - packages=["minipresto", "minipresto.cmd"], + packages=["minitrino", "minitrino.cmd"], include_package_data=True, install_requires=[ "click==7.1.2", @@ -33,5 +33,5 @@ "docker==5.0.0", "PyYAML", ], - entry_points={"console_scripts": ["minipresto=minipresto.cli:cli"]}, + entry_points={"console_scripts": ["minitrino=minitrino.cli:cli"]}, ) diff --git a/install.sh b/install.sh index 80ed5948..ec46057f 100755 --- a/install.sh +++ b/install.sh @@ -6,26 +6,26 @@ function install() { elif pip3 --version 2>&1 | grep -q -e "python3.[6-9]" -e "python 3.[6-9]"; then PIP=pip3 else - echo "Minipresto requires Python 3.6+. Please install a compatible Python version and ensure Pip points to it." + echo "Minitrino requires Python 3.6+. Please install a compatible Python version and ensure Pip points to it." exit 1 fi if [[ $1 == "-v" ]]; then set -ex - echo "Installing minipresto CLI..." + echo "Installing minitrino CLI..." "${PIP}" install --editable "${BASH_SOURCE%/*}"/cli/ else set -e - echo "Installing minipresto CLI..." + echo "Installing minitrino CLI..." "${PIP}" install -q --editable "${BASH_SOURCE%/*}"/cli/ fi } time install $1 -echo -e "\nInstallation complete! Start with the CLI by configuring it running 'minipresto config' \ -(you can do this later). Alternatively, get started immediately with 'minipresto provision'.\n" +echo -e "\nInstallation complete! Start with the CLI by configuring it running 'minitrino config' \ +(you can do this later). Alternatively, get started immediately with 'minitrino provision'.\n" -minipresto +minitrino echo -e "\n" \ No newline at end of file diff --git a/lib/Dockerfile b/lib/Dockerfile index 34560c9b..bc3ecc52 100644 --- a/lib/Dockerfile +++ b/lib/Dockerfile @@ -1,26 +1,26 @@ ARG STARBURST_VER -FROM starburstdata/presto:${STARBURST_VER} +FROM starburstdata/starburst-enterprise:${STARBURST_VER} ARG STARBURST_VER ENV STARBURST_VER=${STARBURST_VER} -ENV PRESTO_CLI_PATH=/usr/local/bin/presto-cli +ENV TRINO_CLI_PATH=/usr/local/bin/trino-cli -ADD ./modules/resources/wait-for-it.sh /opt/minipresto/ +ADD ./modules/resources/wait-for-it.sh /opt/minitrino/ USER root RUN set -euxo pipefail \ && yum install -y wget \ sudo \ - && usermod -aG wheel presto \ - && echo presto | passwd presto --stdin \ - && echo "presto ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \ + && usermod -aG wheel starburst \ + && echo starburst | passwd starburst --stdin \ + && echo "starburst ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \ && DIST=${STARBURST_VER:0:3} \ - && CLI_URL=https://repo1.maven.org/maven2/io/prestosql/presto-cli/"${DIST}"/presto-cli-"${DIST}"-executable.jar \ - && curl -fsSL "${CLI_URL}" > "${PRESTO_CLI_PATH}" \ - && chmod -v +x "${PRESTO_CLI_PATH}" \ - && chown --reference=/usr/lib/presto/etc/config.properties "${PRESTO_CLI_PATH}" \ - && ln -vs "${PRESTO_CLI_PATH}" \ + && CLI_URL=https://repo1.maven.org/maven2/io/trino/trino-cli/"${DIST}"/trino-cli-"${DIST}"-executable.jar \ + && curl -fsSL "${CLI_URL}" > "${TRINO_CLI_PATH}" \ + && chmod -v +x "${TRINO_CLI_PATH}" \ + && chown --reference=/etc/starburst/config.properties "${TRINO_CLI_PATH}" \ + && ln -vs "${TRINO_CLI_PATH}" \ && echo OK -USER presto -WORKDIR /home/presto +USER starburst +WORKDIR /home/trino diff --git a/lib/docker-compose.yml b/lib/docker-compose.yml index bbb16c7b..b758707e 100644 --- a/lib/docker-compose.yml +++ b/lib/docker-compose.yml @@ -1,23 +1,23 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: build: context: . args: STARBURST_VER: "${STARBURST_VER}" labels: - - "com.starburst.tests=minipresto" - - "com.starburst.tests.module=presto" - image: "minipresto/presto:${STARBURST_VER}" - container_name: "presto" - hostname: "presto.minipresto.starburstdata.com" + - "com.starburst.tests=minitrino" + - "com.starburst.tests.module=trino" + image: "minitrino/trino:${STARBURST_VER}" + container_name: "trino" + hostname: "trino.minitrino.starburstdata.com" labels: - - "com.starburst.tests=minipresto" - - "com.starburst.tests.module=presto" + - "com.starburst.tests=minitrino" + - "com.starburst.tests.module=trino" # Uncomment this to enable the volume mount. The variable should point to a # valid SEP license. # volumes: - # - "${STARBURST_LIC_PATH}:/usr/lib/presto/etc/starburstdata.license:ro" + # - "${STARBURST_LIC_PATH}:/etc/starburst/starburstdata.license:ro" ports: - "8080:8080" diff --git a/lib/minipresto.env b/lib/minitrino.env similarity index 50% rename from lib/minipresto.env rename to lib/minitrino.env index 72cc3e96..18716b81 100644 --- a/lib/minipresto.env +++ b/lib/minitrino.env @@ -1,13 +1,15 @@ -COMPOSE_PROJECT_NAME=minipresto +COMPOSE_PROJECT_NAME=minitrino -STARBURST_VER=338-e.0 +STARBURST_VER=354-e ELASTICSEARCH_VER=7.6.2 MYSQL_VER=5 OPEN_LDAP_VER=1.3.0 POSTGRES_VER=11 -POSTGRES_HIVE_HMS_VER=11 +POSTGRES_DELTA_LAKE_VER=11 +POSTGRES_HIVE_MINIO_VER=11 +POSTGRES_HIVE_S3_VER=11 POSTGRES_EVENT_LOGGER_VER=11 POSTGRES_RANGER_VER=12.2 -SEP_RANGER_VER=2.0.24 +SEP_RANGER_VER=2.0.49 SQLSERVER_VER=2017-latest diff --git a/lib/modules/catalog/delta-lake/delta-lake.yml b/lib/modules/catalog/delta-lake/delta-lake.yml new file mode 100644 index 00000000..e8dd24b4 --- /dev/null +++ b/lib/modules/catalog/delta-lake/delta-lake.yml @@ -0,0 +1,61 @@ +version: "3.8" +services: + trino: + volumes: + - "./modules/catalog/delta-lake/resources/trino/delta.properties:/etc/starburst/catalog/delta.properties" + + postgres-delta-lake: + image: "postgres:${POSTGRES_DELTA_LAKE_VER}" + container_name: "postgres-delta-lake" + labels: + - "com.starburst.tests=minitrino" + - "com.starburst.tests.module.delta-lake=catalog-delta-lake" + env_file: + - "./modules/catalog/delta-lake/resources/postgres/postgres.env" + volumes: + - "postgres-delta-lake-data:/var/lib/postgresql/data" + + metastore-delta-lake: + image: "starburstdata/hive-metastore-unsecured:0cc95b" + container_name: "metastore-delta-lake" + labels: + - "com.starburst.tests=minitrino" + - "com.starburst.tests.module.delta-lake=catalog-delta-lake" + depends_on: + - "postgres-delta-lake" + env_file: + - "./modules/catalog/delta-lake/resources/hms/hms.env" + volumes: + - "./modules/resources/wait-for-it.sh:/etc/boot/wait-for-it.sh" + command: + [ + "./etc/boot/wait-for-it.sh", + "postgres-delta-lake:5432", + "--strict", + "--timeout=60", + "--", + "./opt/bin/start-hive-metastore.sh", + ] + ports: + - "9083:9084" + + minio-delta-lake: + image: "minio/minio" + container_name: "minio-delta-lake" + ports: + - "9000:9001" + volumes: + - "./modules/catalog/delta-lake/resources/minio/data/:/data" + labels: + - "com.starburst.tests=minitrino" + - "com.starburst.tests.module.delta-lake=catalog-delta-lake" + environment: + MINIO_ACCESS_KEY: access-key + MINIO_SECRET_KEY: secret-key + command: server /data + +volumes: + postgres-delta-lake-data: + labels: + - "com.starburst.tests=minitrino" + - "com.starburst.tests.module.delta-lake=catalog-delta-lake" diff --git a/lib/modules/catalog/delta-lake/metadata.json b/lib/modules/catalog/delta-lake/metadata.json new file mode 100644 index 00000000..095d9958 --- /dev/null +++ b/lib/modules/catalog/delta-lake/metadata.json @@ -0,0 +1,5 @@ +{ + "description": "Creates a Delta Lake catalog using the enterprise Delta connector.", + "incompatibleModules": [], + "enterprise": true +} \ No newline at end of file diff --git a/lib/modules/catalog/delta-lake/readme.md b/lib/modules/catalog/delta-lake/readme.md new file mode 100644 index 00000000..47b36490 --- /dev/null +++ b/lib/modules/catalog/delta-lake/readme.md @@ -0,0 +1,14 @@ +# Delta-Lake Module +This module uses the Delta Lake connector. There is no Spark backend, so tables +need to be created via CTAS queries from Trino. Example: + +``` +CREATE TABLE delta.default.customer +WITH ( + location = 's3a://sample-bucket/default/' +) +AS SELECT * FROM tpch.tiny.customer; +``` + +This will create the table `delta.default.customer` and a corresponding +`_delta_log` directory in the backing MinIO object storage. diff --git a/lib/modules/catalog/delta-lake/resources/hms/hms.env b/lib/modules/catalog/delta-lake/resources/hms/hms.env new file mode 100644 index 00000000..f37b0f29 --- /dev/null +++ b/lib/modules/catalog/delta-lake/resources/hms/hms.env @@ -0,0 +1,9 @@ +HIVE_METASTORE_JDBC_URL=jdbc:postgresql://postgres-delta-lake:5432/hivemetastore +HIVE_METASTORE_DRIVER=org.postgresql.Driver +HIVE_METASTORE_USER=hivemetastore +HIVE_METASTORE_PASSWORD=trinoRocks15 +HIVE_METASTORE_URI=metastore-minio +S3_PATH_STYLE_ACCESS=true +S3_ENDPOINT=http://minio-delta-lake:9000 +S3_ACCESS_KEY=access-key +S3_SECRET_KEY=secret-key \ No newline at end of file diff --git a/lib/modules/catalog/delta-lake/resources/minio/data/sample-bucket/.gitkeep b/lib/modules/catalog/delta-lake/resources/minio/data/sample-bucket/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/lib/modules/catalog/delta-lake/resources/postgres/postgres.env b/lib/modules/catalog/delta-lake/resources/postgres/postgres.env new file mode 100644 index 00000000..9c07ef07 --- /dev/null +++ b/lib/modules/catalog/delta-lake/resources/postgres/postgres.env @@ -0,0 +1,3 @@ +POSTGRES_USER=hivemetastore +POSTGRES_PASSWORD=trinoRocks15 +POSTGRES_DB=hivemetastore \ No newline at end of file diff --git a/lib/modules/catalog/delta-lake/resources/trino/delta.properties b/lib/modules/catalog/delta-lake/resources/trino/delta.properties new file mode 100644 index 00000000..3a1678da --- /dev/null +++ b/lib/modules/catalog/delta-lake/resources/trino/delta.properties @@ -0,0 +1,7 @@ +connector.name=delta-lake +hive.metastore.uri=thrift://metastore-delta-lake:9083 +hive.s3.endpoint=http://minio-delta-lake:9000/ +hive.s3.path-style-access=true +hive.s3.aws-access-key=access-key +hive.s3.aws-secret-key=secret-key +hive.allow-drop-table=true diff --git a/lib/modules/catalog/elasticsearch/elasticsearch.yml b/lib/modules/catalog/elasticsearch/elasticsearch.yml index a49aa82d..c4f90b84 100644 --- a/lib/modules/catalog/elasticsearch/elasticsearch.yml +++ b/lib/modules/catalog/elasticsearch/elasticsearch.yml @@ -1,24 +1,24 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/catalog/elasticsearch/resources/presto/elasticsearch.properties:/usr/lib/presto/etc/catalog/elasticsearch.properties" + - "./modules/catalog/elasticsearch/resources/trino/elasticsearch.properties:/etc/starburst/catalog/elasticsearch.properties" elasticsearch: image: "elasticsearch:${ELASTICSEARCH_VER}" container_name: elasticsearch volumes: - - "./modules/resources/wait-for-it.sh:/opt/minipresto/wait-for-it.sh" + - "./modules/resources/wait-for-it.sh:/opt/minitrino/wait-for-it.sh" environment: - MINIPRESTO_BOOTSTRAP: "bootstrap-elasticsearch.sh" + MINITRINO_BOOTSTRAP: "bootstrap-elasticsearch.sh" discovery.type: "single-node" bootstrap.memory_lock: "true" network.host: "0.0.0.0" network.bind_host: "0.0.0.0" ES_JAVA_OPTS: "-Xms256m -Xmx256m" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.elasticsearch=catalog-elasticsearch" ports: - 9200:9200 @@ -31,6 +31,6 @@ services: # - ELASTICSEARCH_URL=http://elasticsearch:9200 # - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 # labels: - # - "com.starburst.tests=minipresto" + # - "com.starburst.tests=minitrino" # ports: # - 5601:5601 diff --git a/lib/modules/catalog/elasticsearch/metadata.json b/lib/modules/catalog/elasticsearch/metadata.json index c91ebe0c..db8fae63 100644 --- a/lib/modules/catalog/elasticsearch/metadata.json +++ b/lib/modules/catalog/elasticsearch/metadata.json @@ -1,4 +1,5 @@ { "description": "Creates an Elasticsearch catalog using the standard Elasticsearch connector.", - "incompatible_modules": [] + "incompatibleModules": [], + "enterprise": false } \ No newline at end of file diff --git a/lib/modules/catalog/elasticsearch/resources/bootstrap/bootstrap-elasticsearch.sh b/lib/modules/catalog/elasticsearch/resources/bootstrap/bootstrap-elasticsearch.sh index b43626d1..fb13ea69 100755 --- a/lib/modules/catalog/elasticsearch/resources/bootstrap/bootstrap-elasticsearch.sh +++ b/lib/modules/catalog/elasticsearch/resources/bootstrap/bootstrap-elasticsearch.sh @@ -7,7 +7,7 @@ set -euxo pipefail echo "Waiting for Elasticsearch to come up..." -/opt/minipresto/wait-for-it.sh elasticsearch:9200 --strict --timeout=60 -- echo "Elasticsearch service is up." +/opt/minitrino/wait-for-it.sh elasticsearch:9200 --strict --timeout=60 -- echo "Elasticsearch service is up." echo "Creating user index..." curl -XPUT http://localhost:9200/user?pretty=true; @@ -65,7 +65,7 @@ curl -XPOST http://localhost:9200/user/profile/3?pretty=true -H 'Content-Type: a curl -XPOST http://localhost:9200/user/profile/4?pretty=true -H 'Content-Type: application/json' -d ' { "full_name" : "Julian Spring", - "bio" : "Starburst Presto superuser.", + "bio" : "Starburst Trino superuser.", "age" : 7, "location" : "37.7749290,-122.4194160", "enjoys_coffee" : true, diff --git a/lib/modules/catalog/elasticsearch/resources/presto/elasticsearch.properties b/lib/modules/catalog/elasticsearch/resources/trino/elasticsearch.properties similarity index 100% rename from lib/modules/catalog/elasticsearch/resources/presto/elasticsearch.properties rename to lib/modules/catalog/elasticsearch/resources/trino/elasticsearch.properties diff --git a/lib/modules/catalog/hive-minio/hive-minio.yml b/lib/modules/catalog/hive-minio/hive-minio.yml index 077a6fd9..368cd7b3 100644 --- a/lib/modules/catalog/hive-minio/hive-minio.yml +++ b/lib/modules/catalog/hive-minio/hive-minio.yml @@ -1,16 +1,15 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/catalog/hive-minio/resources/presto/hive_hms_minio.properties:/usr/lib/presto/etc/catalog/hive_hms_minio.properties" + - "./modules/catalog/hive-minio/resources/trino/hive_minio.properties:/etc/starburst/catalog/hive_minio.properties" postgres-hive-minio: - image: "postgres:${POSTGRES_HIVE_HMS_VER}" + image: "postgres:${POSTGRES_HIVE_MINIO_VER}" container_name: "postgres-hive-minio" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.hive-minio=catalog-hive-minio" - user: "postgres" env_file: - "./modules/catalog/hive-minio/resources/postgres/postgres.env" volumes: @@ -20,7 +19,7 @@ services: image: "starburstdata/hive-metastore-unsecured:0cc95b" container_name: "metastore-minio" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.hive-minio=catalog-hive-minio" depends_on: - "postgres-hive-minio" @@ -48,7 +47,7 @@ services: volumes: - "./modules/catalog/hive-minio/resources/minio/data/:/data" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.hive-minio=catalog-hive-minio" environment: MINIO_ACCESS_KEY: access-key @@ -58,5 +57,5 @@ services: volumes: postgres-hive-minio-data: labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.hive-minio=catalog-hive-minio" diff --git a/lib/modules/catalog/hive-minio/metadata.json b/lib/modules/catalog/hive-minio/metadata.json index b7f5c9f5..b1f1fc22 100644 --- a/lib/modules/catalog/hive-minio/metadata.json +++ b/lib/modules/catalog/hive-minio/metadata.json @@ -1,4 +1,5 @@ { "description": "Creates a Hive catalog using MinIO for object storage and Postgres as a backend metastore DB.", - "incompatible_modules": [] + "incompatibleModules": [], + "enterprise": false } \ No newline at end of file diff --git a/lib/modules/catalog/hive-minio/readme.md b/lib/modules/catalog/hive-minio/readme.md index 831bc924..066fb8a1 100644 --- a/lib/modules/catalog/hive-minio/readme.md +++ b/lib/modules/catalog/hive-minio/readme.md @@ -3,13 +3,13 @@ This module uses Minio as a local implementation of S3 object storage. You can w You can access the Minio UI at `http://localhost:9000` with `access-key` and `secret-key` for credentials. -You can create a table with ORC data with Presto very quickly: +You can create a table with ORC data with Trino very quickly: ``` -presto> create schema hive_hms_minio.tiny with (location='s3a://sample-bucket/tiny/'); +trino> create schema hive_minio.tiny with (location='s3a://sample-bucket/tiny/'); CREATE SCHEMA -presto> create table hive_hms_minio.tiny.customer as select * from tpch.tiny.customer; +trino> create table hive_minio.tiny.customer as select * from tpch.tiny.customer; CREATE TABLE: 1500 rows ``` diff --git a/lib/modules/catalog/hive-minio/resources/hms/hms.env b/lib/modules/catalog/hive-minio/resources/hms/hms.env index fead8c00..ed47c079 100644 --- a/lib/modules/catalog/hive-minio/resources/hms/hms.env +++ b/lib/modules/catalog/hive-minio/resources/hms/hms.env @@ -1,7 +1,7 @@ HIVE_METASTORE_JDBC_URL=jdbc:postgresql://postgres-hive-minio:5432/hivemetastore HIVE_METASTORE_DRIVER=org.postgresql.Driver HIVE_METASTORE_USER=hivemetastore -HIVE_METASTORE_PASSWORD=prestoRocks15 +HIVE_METASTORE_PASSWORD=trinoRocks15 HIVE_METASTORE_URI=metastore-minio S3_PATH_STYLE_ACCESS=true S3_ENDPOINT=http://minio:9000 diff --git a/lib/modules/catalog/hive-minio/resources/postgres/postgres.env b/lib/modules/catalog/hive-minio/resources/postgres/postgres.env index 7c7a5dc8..9c07ef07 100644 --- a/lib/modules/catalog/hive-minio/resources/postgres/postgres.env +++ b/lib/modules/catalog/hive-minio/resources/postgres/postgres.env @@ -1,3 +1,3 @@ POSTGRES_USER=hivemetastore -POSTGRES_PASSWORD=prestoRocks15 +POSTGRES_PASSWORD=trinoRocks15 POSTGRES_DB=hivemetastore \ No newline at end of file diff --git a/lib/modules/catalog/hive-minio/resources/presto/hive_hms_minio.properties b/lib/modules/catalog/hive-minio/resources/trino/hive_minio.properties similarity index 100% rename from lib/modules/catalog/hive-minio/resources/presto/hive_hms_minio.properties rename to lib/modules/catalog/hive-minio/resources/trino/hive_minio.properties diff --git a/lib/modules/catalog/hive-s3/hive-s3.yml b/lib/modules/catalog/hive-s3/hive-s3.yml index b3c01e79..930643b6 100644 --- a/lib/modules/catalog/hive-s3/hive-s3.yml +++ b/lib/modules/catalog/hive-s3/hive-s3.yml @@ -1,21 +1,20 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: environment: S3_ENDPOINT: "${S3_ENDPOINT}" S3_ACCESS_KEY: "${S3_ACCESS_KEY}" S3_SECRET_KEY: "${S3_SECRET_KEY}" REGION: "${REGION}" volumes: - - "./modules/catalog/hive-s3/resources/presto/hive_hms.properties:/usr/lib/presto/etc/catalog/hive_hms.properties" + - "./modules/catalog/hive-s3/resources/trino/hive_s3.properties:/etc/starburst/catalog/hive_s3.properties" postgres-hive-s3: - image: "postgres:${POSTGRES_HIVE_HMS_VER}" + image: "postgres:${POSTGRES_HIVE_S3_VER}" container_name: "postgres-hive-s3" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.hive-s3=catalog-hive-s3" - user: "postgres" env_file: - "./modules/catalog/hive-s3/resources/postgres/postgres.env" volumes: @@ -25,7 +24,7 @@ services: image: "starburstdata/hive-metastore-unsecured" container_name: "metastore" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.hive-s3=catalog-hive-s3" depends_on: - "postgres-hive-s3" @@ -43,10 +42,10 @@ services: "./opt/bin/start-hive-metastore.sh", ] ports: - - "9083:9083" + - "9083:9085" volumes: postgres-hive-s3-data: labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.hive-s3=catalog-hive-s3" diff --git a/lib/modules/catalog/hive-s3/metadata.json b/lib/modules/catalog/hive-s3/metadata.json index d2774eeb..60cd022f 100644 --- a/lib/modules/catalog/hive-s3/metadata.json +++ b/lib/modules/catalog/hive-s3/metadata.json @@ -1,4 +1,5 @@ { "description": "Creates a Hive catalog using AWS S3 for object storage and Postgres as a backend metastore DB.", - "incompatible_modules": [] + "incompatibleModules": [], + "enterprise": false } \ No newline at end of file diff --git a/lib/modules/catalog/hive-s3/readme.md b/lib/modules/catalog/hive-s3/readme.md index 100cc911..b4765a7b 100644 --- a/lib/modules/catalog/hive-s3/readme.md +++ b/lib/modules/catalog/hive-s3/readme.md @@ -1,2 +1,2 @@ # Hive-S3 Module -This module uses a Hive metastore container along with a Postgres container for the metastore's backend storage. Using Minipresto's configuration, you can input AWS credentials to link the module to a an S3 bucket for data storage, reading, and writing. Removing the volume associated with the Postgres container will effectively remove your metastore's database, so only remove the volume if you are prepared to lose all of your metadata. \ No newline at end of file +This module uses a Hive metastore container along with a Postgres container for the metastore's backend storage. Using Minitrino's configuration, you can input AWS credentials to link the module to a an S3 bucket for data storage, reading, and writing. Removing the volume associated with the Postgres container will effectively remove your metastore's database, so only remove the volume if you are prepared to lose all of your metadata. \ No newline at end of file diff --git a/lib/modules/catalog/hive-s3/resources/hms/hms.env b/lib/modules/catalog/hive-s3/resources/hms/hms.env index 2f8567e6..33668e30 100644 --- a/lib/modules/catalog/hive-s3/resources/hms/hms.env +++ b/lib/modules/catalog/hive-s3/resources/hms/hms.env @@ -1,7 +1,7 @@ HIVE_METASTORE_JDBC_URL=jdbc:postgresql://postgres-hive-s3:5432/hivemetastore HIVE_METASTORE_DRIVER=org.postgresql.Driver HIVE_METASTORE_USER=hivemetastore -HIVE_METASTORE_PASSWORD=prestoRocks15 +HIVE_METASTORE_PASSWORD=trinoRocks15 S3_ENDPOINT=${S3_ENDPOINT} S3_ACCESS_KEY=${S3_ACCESS_KEY} S3_SECRET_KEY=${S3_SECRET_KEY} diff --git a/lib/modules/catalog/hive-s3/resources/postgres/postgres.env b/lib/modules/catalog/hive-s3/resources/postgres/postgres.env index 7c7a5dc8..9c07ef07 100644 --- a/lib/modules/catalog/hive-s3/resources/postgres/postgres.env +++ b/lib/modules/catalog/hive-s3/resources/postgres/postgres.env @@ -1,3 +1,3 @@ POSTGRES_USER=hivemetastore -POSTGRES_PASSWORD=prestoRocks15 +POSTGRES_PASSWORD=trinoRocks15 POSTGRES_DB=hivemetastore \ No newline at end of file diff --git a/lib/modules/catalog/hive-s3/resources/presto/hive_hms.properties b/lib/modules/catalog/hive-s3/resources/trino/hive_s3.properties similarity index 100% rename from lib/modules/catalog/hive-s3/resources/presto/hive_hms.properties rename to lib/modules/catalog/hive-s3/resources/trino/hive_s3.properties diff --git a/lib/modules/catalog/mysql/metadata.json b/lib/modules/catalog/mysql/metadata.json index c6db38d1..a008b685 100644 --- a/lib/modules/catalog/mysql/metadata.json +++ b/lib/modules/catalog/mysql/metadata.json @@ -1,4 +1,5 @@ { "description": "Creates a MySQL catalog using the standard MySQL connector.", - "incompatible_modules": [] + "incompatibleModules": [], + "enterprise": false } \ No newline at end of file diff --git a/lib/modules/catalog/mysql/mysql.yml b/lib/modules/catalog/mysql/mysql.yml index aee5eca4..0968f4c4 100644 --- a/lib/modules/catalog/mysql/mysql.yml +++ b/lib/modules/catalog/mysql/mysql.yml @@ -1,17 +1,17 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/catalog/mysql/resources/presto/mysql.properties:/usr/lib/presto/etc/catalog/mysql.properties" + - "./modules/catalog/mysql/resources/trino/mysql.properties:/etc/starburst/catalog/mysql.properties" mysql: image: "mysql:${MYSQL_VER}" container_name: "mysql" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.mysql=catalog-mysql" env_file: - "./modules/catalog/mysql/resources/mysql/mysql.env" environment: - MINIPRESTO_BOOTSTRAP: "bootstrap.sh" + MINITRINO_BOOTSTRAP: "bootstrap.sh" diff --git a/lib/modules/catalog/mysql/resources/bootstrap/bootstrap.sh b/lib/modules/catalog/mysql/resources/bootstrap/bootstrap.sh index 1c4b8acc..58cac8de 100755 --- a/lib/modules/catalog/mysql/resources/bootstrap/bootstrap.sh +++ b/lib/modules/catalog/mysql/resources/bootstrap/bootstrap.sh @@ -3,7 +3,7 @@ MAX_ATTEMPTS=60 for (( ATTEMPTS=1; ATTEMPTS<=MAX_ATTEMPTS; ATTEMPTS++ )) do - mysql -pprestoRocks15 -e "GRANT ALL PRIVILEGES ON *.* TO 'admin';" + mysql -ptrinoRocks15 -e "GRANT ALL PRIVILEGES ON *.* TO 'admin';" if [ $? == 0 ]; then break fi diff --git a/lib/modules/catalog/mysql/resources/mysql/mysql.env b/lib/modules/catalog/mysql/resources/mysql/mysql.env index 342d6406..d75b0e62 100644 --- a/lib/modules/catalog/mysql/resources/mysql/mysql.env +++ b/lib/modules/catalog/mysql/resources/mysql/mysql.env @@ -1,3 +1,3 @@ -MYSQL_ROOT_PASSWORD=prestoRocks15 +MYSQL_ROOT_PASSWORD=trinoRocks15 MYSQL_USER=admin -MYSQL_PASSWORD=prestoRocks15 +MYSQL_PASSWORD=trinoRocks15 diff --git a/lib/modules/catalog/mysql/resources/presto/mysql.properties b/lib/modules/catalog/mysql/resources/trino/mysql.properties similarity index 70% rename from lib/modules/catalog/mysql/resources/presto/mysql.properties rename to lib/modules/catalog/mysql/resources/trino/mysql.properties index efe8b04d..ed5db3c6 100644 --- a/lib/modules/catalog/mysql/resources/presto/mysql.properties +++ b/lib/modules/catalog/mysql/resources/trino/mysql.properties @@ -1,4 +1,4 @@ connector.name=mysql connection-url=jdbc:mysql://mysql:3306 connection-user=admin -connection-password=prestoRocks15 +connection-password=trinoRocks15 diff --git a/lib/modules/catalog/postgres/metadata.json b/lib/modules/catalog/postgres/metadata.json index 4030994f..e42a56b6 100644 --- a/lib/modules/catalog/postgres/metadata.json +++ b/lib/modules/catalog/postgres/metadata.json @@ -1,4 +1,5 @@ { "description": "Creates a Postgres catalog using the standard Postgres connector.", - "incompatible_modules": [] + "incompatibleModules": [], + "enterprise": false } \ No newline at end of file diff --git a/lib/modules/catalog/postgres/postgres.yml b/lib/modules/catalog/postgres/postgres.yml index 2719c42f..229ab71e 100644 --- a/lib/modules/catalog/postgres/postgres.yml +++ b/lib/modules/catalog/postgres/postgres.yml @@ -1,15 +1,15 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/catalog/postgres/resources/presto/postgres.properties:/usr/lib/presto/etc/catalog/postgres.properties" + - "./modules/catalog/postgres/resources/trino/postgres.properties:/etc/starburst/catalog/postgres.properties" postgres: image: "postgres:${POSTGRES_VER}" container_name: "postgres" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.postgres=catalog-postgres" env_file: - "./modules/catalog/postgres/resources/postgres/postgres.env" diff --git a/lib/modules/catalog/postgres/readme.md b/lib/modules/catalog/postgres/readme.md index 1ab95994..66b108fb 100644 --- a/lib/modules/catalog/postgres/readme.md +++ b/lib/modules/catalog/postgres/readme.md @@ -1,2 +1,2 @@ # Postgres Connector Module -This module provisions a standalone Postgres service. It is named uniquely to avoid conflicts with other modules that may use Presto as a backend, such as the `hive-s3` module. \ No newline at end of file +This module provisions a standalone Postgres service. It is named uniquely to avoid conflicts with other modules that may use Trino as a backend, such as the `hive-s3` module. \ No newline at end of file diff --git a/lib/modules/catalog/postgres/resources/postgres/postgres.env b/lib/modules/catalog/postgres/resources/postgres/postgres.env index 6226a9e4..ee30255a 100644 --- a/lib/modules/catalog/postgres/resources/postgres/postgres.env +++ b/lib/modules/catalog/postgres/resources/postgres/postgres.env @@ -1,3 +1,3 @@ POSTGRES_USER=admin -POSTGRES_PASSWORD=prestoRocks15 -POSTGRES_DB=minipresto +POSTGRES_PASSWORD=trinoRocks15 +POSTGRES_DB=minitrino diff --git a/lib/modules/catalog/postgres/resources/presto/postgres.properties b/lib/modules/catalog/postgres/resources/presto/postgres.properties deleted file mode 100644 index d0d7da7b..00000000 --- a/lib/modules/catalog/postgres/resources/presto/postgres.properties +++ /dev/null @@ -1,4 +0,0 @@ -connector.name=postgresql -connection-url=jdbc:postgresql://postgres:5432/minipresto -connection-user=admin -connection-password=prestoRocks15 diff --git a/lib/modules/catalog/postgres/resources/trino/postgres.properties b/lib/modules/catalog/postgres/resources/trino/postgres.properties new file mode 100644 index 00000000..c5a38dab --- /dev/null +++ b/lib/modules/catalog/postgres/resources/trino/postgres.properties @@ -0,0 +1,4 @@ +connector.name=postgresql +connection-url=jdbc:postgresql://postgres:5432/minitrino +connection-user=admin +connection-password=trinoRocks15 diff --git a/lib/modules/catalog/snowflake-distributed/metadata.json b/lib/modules/catalog/snowflake-distributed/metadata.json index 91d53ad3..fe19ea0e 100644 --- a/lib/modules/catalog/snowflake-distributed/metadata.json +++ b/lib/modules/catalog/snowflake-distributed/metadata.json @@ -1,4 +1,5 @@ { "description": "Creates a Snowflake catalog leveraging the Snowflake distributed connector.", - "incompatible_modules": [] + "incompatibleModules": [], + "enterprise": true } \ No newline at end of file diff --git a/lib/modules/catalog/snowflake-distributed/resources/presto/snowflake_distributed.properties b/lib/modules/catalog/snowflake-distributed/resources/trino/snowflake_distributed.properties similarity index 100% rename from lib/modules/catalog/snowflake-distributed/resources/presto/snowflake_distributed.properties rename to lib/modules/catalog/snowflake-distributed/resources/trino/snowflake_distributed.properties diff --git a/lib/modules/catalog/snowflake-distributed/snowflake-distributed.yml b/lib/modules/catalog/snowflake-distributed/snowflake-distributed.yml index b481f42e..8498794f 100644 --- a/lib/modules/catalog/snowflake-distributed/snowflake-distributed.yml +++ b/lib/modules/catalog/snowflake-distributed/snowflake-distributed.yml @@ -1,7 +1,7 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: environment: SNOWFLAKE_DIST_CONNECT_URL: "${SNOWFLAKE_DIST_CONNECT_URL}" SNOWFLAKE_DIST_CONNECT_USER: "${SNOWFLAKE_DIST_CONNECT_USER}" @@ -12,4 +12,4 @@ services: labels: - "com.starburst.tests.module.snowflake-distributed=catalog-snowflake-distributed" volumes: - - "./modules/catalog/snowflake-distributed/resources/presto/snowflake_distributed.properties:/usr/lib/presto/etc/catalog/snowflake_distributed.properties" + - "./modules/catalog/snowflake-distributed/resources/trino/snowflake_distributed.properties:/etc/starburst/catalog/snowflake_distributed.properties" diff --git a/lib/modules/catalog/snowflake-jdbc/metadata.json b/lib/modules/catalog/snowflake-jdbc/metadata.json index 2903453a..3b5b6b59 100644 --- a/lib/modules/catalog/snowflake-jdbc/metadata.json +++ b/lib/modules/catalog/snowflake-jdbc/metadata.json @@ -1,4 +1,5 @@ { "description": "Creates a Snowflake catalog leveraging the Snowflake JDBC connector.", - "incompatible_modules": [] + "incompatibleModules": [], + "enterprise": true } \ No newline at end of file diff --git a/lib/modules/catalog/snowflake-jdbc/resources/presto/snowflake_jdbc.properties b/lib/modules/catalog/snowflake-jdbc/resources/trino/snowflake_jdbc.properties similarity index 100% rename from lib/modules/catalog/snowflake-jdbc/resources/presto/snowflake_jdbc.properties rename to lib/modules/catalog/snowflake-jdbc/resources/trino/snowflake_jdbc.properties diff --git a/lib/modules/catalog/snowflake-jdbc/snowflake-jdbc.yml b/lib/modules/catalog/snowflake-jdbc/snowflake-jdbc.yml index 9257a069..cbbe1f69 100644 --- a/lib/modules/catalog/snowflake-jdbc/snowflake-jdbc.yml +++ b/lib/modules/catalog/snowflake-jdbc/snowflake-jdbc.yml @@ -1,7 +1,7 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: environment: SNOWFLAKE_JDBC_CONNECT_URL: "${SNOWFLAKE_JDBC_CONNECT_URL}" SNOWFLAKE_JDBC_CONNECT_USER: "${SNOWFLAKE_JDBC_CONNECT_USER}" @@ -12,4 +12,4 @@ services: labels: - "com.starburst.tests.module.snowflake-jdbc=catalog-snowflake-jdbc" volumes: - - "./modules/catalog/snowflake-jdbc/resources/presto/snowflake_jdbc.properties:/usr/lib/presto/etc/catalog/snowflake_jdbc.properties" + - "./modules/catalog/snowflake-jdbc/resources/trino/snowflake_jdbc.properties:/etc/starburst/catalog/snowflake_jdbc.properties" diff --git a/lib/modules/catalog/sqlserver/metadata.json b/lib/modules/catalog/sqlserver/metadata.json index d0935717..a8486fed 100644 --- a/lib/modules/catalog/sqlserver/metadata.json +++ b/lib/modules/catalog/sqlserver/metadata.json @@ -1,4 +1,5 @@ { "description": "Creates a SQL Server catalog using the standard SQL Server connector.", - "incompatible_modules": [] + "incompatibleModules": [], + "enterprise": false } diff --git a/lib/modules/catalog/sqlserver/resources/sqlserver/sqlserver.env b/lib/modules/catalog/sqlserver/resources/sqlserver/sqlserver.env index b34bd2d3..19bf56ea 100644 --- a/lib/modules/catalog/sqlserver/resources/sqlserver/sqlserver.env +++ b/lib/modules/catalog/sqlserver/resources/sqlserver/sqlserver.env @@ -1,3 +1,3 @@ ACCEPT_EULA=Y -SA_PASSWORD=prestoRocks15 +SA_PASSWORD=trinoRocks15 MSSQL_PID=Developer diff --git a/lib/modules/catalog/sqlserver/resources/presto/sqlserver.properties b/lib/modules/catalog/sqlserver/resources/trino/sqlserver.properties similarity index 75% rename from lib/modules/catalog/sqlserver/resources/presto/sqlserver.properties rename to lib/modules/catalog/sqlserver/resources/trino/sqlserver.properties index f8170ddf..323ee097 100644 --- a/lib/modules/catalog/sqlserver/resources/presto/sqlserver.properties +++ b/lib/modules/catalog/sqlserver/resources/trino/sqlserver.properties @@ -1,4 +1,4 @@ connector.name=sqlserver connection-url=jdbc:sqlserver://sqlserver:1433;database=master connection-user=sa -connection-password=prestoRocks15 +connection-password=trinoRocks15 diff --git a/lib/modules/catalog/sqlserver/sqlserver.yml b/lib/modules/catalog/sqlserver/sqlserver.yml index d09f014e..612edd54 100644 --- a/lib/modules/catalog/sqlserver/sqlserver.yml +++ b/lib/modules/catalog/sqlserver/sqlserver.yml @@ -1,15 +1,15 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/catalog/sqlserver/resources/presto/sqlserver.properties:/usr/lib/presto/etc/catalog/sqlserver.properties" + - "./modules/catalog/sqlserver/resources/trino/sqlserver.properties:/etc/starburst/catalog/sqlserver.properties" sqlserver: image: "mcr.microsoft.com/mssql/server:${SQLSERVER_VER}" container_name: "sqlserver" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.sqlserver=catalog-sqlserver" env_file: - "./modules/catalog/sqlserver/resources/sqlserver/sqlserver.env" diff --git a/lib/modules/catalog/test/metadata.json b/lib/modules/catalog/test/metadata.json index 18e2a712..dff8e011 100644 --- a/lib/modules/catalog/test/metadata.json +++ b/lib/modules/catalog/test/metadata.json @@ -1,4 +1,5 @@ { "description": "Test module.", - "incompatible_modules": ["ldap"] + "incompatibleModules": ["ldap"], + "enterprise": false } \ No newline at end of file diff --git a/lib/modules/catalog/test/resources/bootstrap/bootstrap-presto.sh b/lib/modules/catalog/test/resources/bootstrap/bootstrap-presto.sh deleted file mode 100755 index dcb6803b..00000000 --- a/lib/modules/catalog/test/resources/bootstrap/bootstrap-presto.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -touch /usr/lib/presto/etc/test_bootstrap.txt diff --git a/lib/modules/catalog/test/resources/bootstrap/bootstrap-trino.sh b/lib/modules/catalog/test/resources/bootstrap/bootstrap-trino.sh new file mode 100755 index 00000000..c7999af1 --- /dev/null +++ b/lib/modules/catalog/test/resources/bootstrap/bootstrap-trino.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +touch /etc/starburst/test_bootstrap.txt diff --git a/lib/modules/catalog/test/test.yml b/lib/modules/catalog/test/test.yml index d7fc0f89..e63effde 100644 --- a/lib/modules/catalog/test/test.yml +++ b/lib/modules/catalog/test/test.yml @@ -1,23 +1,23 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: environment: - MINIPRESTO_BOOTSTRAP: "bootstrap-presto.sh" + MINITRINO_BOOTSTRAP: "bootstrap-trino.sh" test: build: context: ./modules/catalog/test/ labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.test=catalog-test" environment: - MINIPRESTO_BOOTSTRAP: "bootstrap-test.sh" + MINITRINO_BOOTSTRAP: "bootstrap-test.sh" command: "tail -F anything" # Keep alive - image: "minipresto:test" + image: "minitrino:test" container_name: "test" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.test=catalog-test" volumes: - "test-data:/etc/test/" @@ -25,5 +25,5 @@ services: volumes: test-data: labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.test=catalog-test" diff --git a/lib/modules/security/event-logger/event-logger.yml b/lib/modules/security/event-logger/event-logger.yml index 357876e3..25aeefe5 100644 --- a/lib/modules/security/event-logger/event-logger.yml +++ b/lib/modules/security/event-logger/event-logger.yml @@ -1,16 +1,16 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/security/event-logger/resources/presto/postgres_event_logger.properties:/usr/lib/presto/etc/catalog/postgres_event_logger.properties" - - "./modules/security/event-logger/resources/presto/event-listener.properties:/usr/lib/presto/etc/event-listener.properties" + - "./modules/security/event-logger/resources/trino/postgres_event_logger.properties:/etc/starburst/catalog/postgres_event_logger.properties" + - "./modules/security/event-logger/resources/trino/event-listener.properties:/etc/starburst/event-listener.properties" postgres-event-logger: image: "postgres:${POSTGRES_EVENT_LOGGER_VER}" container_name: "postgres-event-logger" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.event-logger=security-event-logger" env_file: - "./modules/security/event-logger/resources/event-logger/postgres.env" diff --git a/lib/modules/security/event-logger/metadata.json b/lib/modules/security/event-logger/metadata.json index 0531a6b8..2bf610eb 100644 --- a/lib/modules/security/event-logger/metadata.json +++ b/lib/modules/security/event-logger/metadata.json @@ -1,4 +1,5 @@ { "description": "Implements the query event logger, which logs all query information to a Postgres database.", - "incompatible_modules": [] + "incompatibleModules": [], + "enterprise": true } \ No newline at end of file diff --git a/lib/modules/security/event-logger/resources/event-logger/postgres.env b/lib/modules/security/event-logger/resources/event-logger/postgres.env index bae1d89c..9c8e08a3 100644 --- a/lib/modules/security/event-logger/resources/event-logger/postgres.env +++ b/lib/modules/security/event-logger/resources/event-logger/postgres.env @@ -1,3 +1,3 @@ POSTGRES_USER=admin -POSTGRES_PASSWORD=prestoRocks15 +POSTGRES_PASSWORD=trinoRocks15 POSTGRES_DB=event_logger diff --git a/lib/modules/security/event-logger/resources/presto/event-listener.properties b/lib/modules/security/event-logger/resources/trino/event-listener.properties similarity index 81% rename from lib/modules/security/event-logger/resources/presto/event-listener.properties rename to lib/modules/security/event-logger/resources/trino/event-listener.properties index c3ef0c45..59d7a211 100644 --- a/lib/modules/security/event-logger/resources/presto/event-listener.properties +++ b/lib/modules/security/event-logger/resources/trino/event-listener.properties @@ -1,4 +1,4 @@ event-listener.name=event-logger jdbc.url=jdbc:postgresql://postgres-event-logger:5432/event_logger jdbc.user=admin -jdbc.password=prestoRocks15 \ No newline at end of file +jdbc.password=trinoRocks15 \ No newline at end of file diff --git a/lib/modules/security/event-logger/resources/presto/postgres_event_logger.properties b/lib/modules/security/event-logger/resources/trino/postgres_event_logger.properties similarity index 78% rename from lib/modules/security/event-logger/resources/presto/postgres_event_logger.properties rename to lib/modules/security/event-logger/resources/trino/postgres_event_logger.properties index 1df98011..6fcd53ae 100644 --- a/lib/modules/security/event-logger/resources/presto/postgres_event_logger.properties +++ b/lib/modules/security/event-logger/resources/trino/postgres_event_logger.properties @@ -1,4 +1,4 @@ connector.name=postgresql connection-url=jdbc:postgresql://postgres-event-logger:5432/event_logger connection-user=admin -connection-password=prestoRocks15 \ No newline at end of file +connection-password=trinoRocks15 \ No newline at end of file diff --git a/lib/modules/security/file-access-control/file-access-control.yml b/lib/modules/security/file-access-control/file-access-control.yml new file mode 100644 index 00000000..923905ec --- /dev/null +++ b/lib/modules/security/file-access-control/file-access-control.yml @@ -0,0 +1,11 @@ +version: "3.8" +services: + + trino: + volumes: + - "./modules/security/file-access-control/resources/trino/access-control.properties:/etc/starburst/access-control.properties" + - "./modules/security/file-access-control/resources/trino/group-provider.properties:/etc/starburst/group-provider.properties" + - "./modules/security/file-access-control/resources/trino/rules.json:/etc/starburst/rules.json" + - "./modules/security/file-access-control/resources/trino/group.txt:/etc/starburst/group.txt" + labels: + - "com.starburst.tests.module.file-access-control=security-file-access-control" \ No newline at end of file diff --git a/lib/modules/security/file-access-control/metadata.json b/lib/modules/security/file-access-control/metadata.json new file mode 100644 index 00000000..0bc15c5c --- /dev/null +++ b/lib/modules/security/file-access-control/metadata.json @@ -0,0 +1,5 @@ +{ + "description": "Secures Trino with file-based access control.", + "incompatibleModules": ["system-ranger"], + "enterprise": false +} \ No newline at end of file diff --git a/lib/modules/security/file-access-control/readme.md b/lib/modules/security/file-access-control/readme.md new file mode 100644 index 00000000..a4f896ae --- /dev/null +++ b/lib/modules/security/file-access-control/readme.md @@ -0,0 +1,35 @@ +# File Access Control Module +A module which utilizes Trino's [file-based system access control +plugin](https://docs.starburst.io/latest/security/file-system-access-control.html). +This also makes used of the [file-based group +provider](https://docs.starburst.io/latest/security/group-file.html). + +## Sample Usage +To provision this module, run: + +```shell +minitrino provision --module file-access-control +``` + +You will need to supply a username to the Trino CLI in order to map to a group +(see `group.txt` for which users belong to which groups). Example: + +```shell +trino-cli --user admin-2 +trino-cli --user metadata-1 +trino-cli --user platform +``` + +## Policies +The access policy is located in the `rules.json` file which defines groups of +users that map to a certain access control permission. The users for the groups +are defined in the `group.txt` file. + +- Users in the `platform-admins` group have full access to all objects within Trino +- Users in the `metadata-users` group only have access to the tables within the + `system.metadata` schema +- Users in the `platform-users` group only have access to the tables within the + `system.runtime` schema + +You can modify this module to further specify access control permissions to +other catalogs provisioned with other Minitrino modules. diff --git a/lib/modules/security/file-access-control/resources/trino/access-control.properties b/lib/modules/security/file-access-control/resources/trino/access-control.properties new file mode 100644 index 00000000..e8e9c7ce --- /dev/null +++ b/lib/modules/security/file-access-control/resources/trino/access-control.properties @@ -0,0 +1,3 @@ +access-control.name=file +security.config-file=/etc/starburst/rules.json +security.refresh-period=5s diff --git a/lib/modules/security/file-access-control/resources/trino/group-provider.properties b/lib/modules/security/file-access-control/resources/trino/group-provider.properties new file mode 100644 index 00000000..caf83835 --- /dev/null +++ b/lib/modules/security/file-access-control/resources/trino/group-provider.properties @@ -0,0 +1,2 @@ +group-provider.name=file +file.group-file=/etc/starburst/group.txt diff --git a/lib/modules/security/file-access-control/resources/trino/group.txt b/lib/modules/security/file-access-control/resources/trino/group.txt new file mode 100644 index 00000000..05fd7fdd --- /dev/null +++ b/lib/modules/security/file-access-control/resources/trino/group.txt @@ -0,0 +1,3 @@ +platform-admins:admin,admin-1,admin-2 +metadata-users:metadata,metadata-1,metadata-2 +platform-users:platform,platform-1,platform-2 diff --git a/lib/modules/security/file-access-control/resources/trino/rules.json b/lib/modules/security/file-access-control/resources/trino/rules.json new file mode 100644 index 00000000..a9ef0ea3 --- /dev/null +++ b/lib/modules/security/file-access-control/resources/trino/rules.json @@ -0,0 +1,90 @@ +{ + "catalogs":[ + { + "group":"platform-admins", + "catalog":".*", + "allow":"all" + }, + { + "group":"metadata-users|platform-users", + "catalog":"system", + "allow":"all" + }, + { + "catalog":"system", + "allow":"none" + }, + { + "catalog":".*", + "allow":"none" + } + ], + "schemas":[ + { + "group":"platform-admins", + "schema":".*", + "owner":true + }, + { + "group":"metadata-users", + "schema":"system.metadata", + "owner":true + }, + { + "group":"platform-users", + "schema":"system.runtime", + "owner":true + }, + { + "catalog":".*", + "schema":".*", + "owner":false + } + ], + "tables":[ + { + "group":"platform-admins", + "catalog":".*", + "schema":".*", + "privileges":[ + "SELECT", + "INSERT", + "DELETE", + "OWNERSHIP", + "GRANT_SELECT" + ] + }, + { + "group":"metadata-users", + "catalog":"system", + "schema":"metadata", + "privileges":[ + "SELECT", + "INSERT", + "DELETE", + "OWNERSHIP", + "GRANT_SELECT" + ] + }, + { + "group":"platform-users", + "catalog":"system", + "schema":"runtime", + "privileges":[ + "SELECT", + "INSERT", + "DELETE", + "OWNERSHIP", + "GRANT_SELECT" + ] + }, + { + "group":".*", + "catalog":".*", + "schema":".*", + "privileges":[ + + ] + } + ] +} \ No newline at end of file diff --git a/lib/modules/security/ldap/ldap.yml b/lib/modules/security/ldap/ldap.yml index 0ee25780..8e89a52a 100755 --- a/lib/modules/security/ldap/ldap.yml +++ b/lib/modules/security/ldap/ldap.yml @@ -1,30 +1,30 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/security/ldap/resources/presto/password-authenticator.properties:/usr/lib/presto/etc/password-authenticator.properties" - - "./modules/security/ldap/resources/ldap/ldap-users:/usr/lib/presto/etc/ldap-users" - - "~/.minipresto/ssl:/usr/lib/presto/etc/ssl" + - "./modules/security/ldap/resources/trino/password-authenticator.properties:/etc/starburst/password-authenticator.properties" + - "./modules/security/ldap/resources/ldap/ldap-users:/etc/starburst/ldap-users" + - "~/.minitrino/ssl:/etc/starburst/ssl" environment: - MINIPRESTO_BOOTSTRAP: "bootstrap-presto.sh" + MINITRINO_BOOTSTRAP: "bootstrap-trino.sh" links: - - "ldap:ldap.minipresto.starburstdata.com" + - "ldap:ldap.minitrino.starburstdata.com" ports: - "8443:8443" ldap: image: "osixia/openldap:${OPEN_LDAP_VER}" container_name: "ldap" - hostname: "ldap.minipresto.starburstdata.com" + hostname: "ldap.minitrino.starburstdata.com" environment: LDAP_ORGANISATION: "Example Inc." LDAP_DOMAIN: "example.com" - LDAP_ADMIN_PASSWORD: "prestoRocks15" + LDAP_ADMIN_PASSWORD: "trinoRocks15" LDAP_TLS_VERIFY_CLIENT: "try" LDAPTLS_REQCERT: "never" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.ldap=security-ldap" ports: - "636:636" diff --git a/lib/modules/security/ldap/metadata.json b/lib/modules/security/ldap/metadata.json index 91d9477d..5e45b01a 100644 --- a/lib/modules/security/ldap/metadata.json +++ b/lib/modules/security/ldap/metadata.json @@ -1,4 +1,5 @@ { - "description": "Secures Presto with LDAP authentication.", - "incompatible_modules": ["password-file"] + "description": "Secures Trino with LDAP authentication.", + "incompatibleModules": ["password-file"], + "enterprise": false } \ No newline at end of file diff --git a/lib/modules/security/ldap/readme.md b/lib/modules/security/ldap/readme.md index c838c12f..d22547f0 100644 --- a/lib/modules/security/ldap/readme.md +++ b/lib/modules/security/ldap/readme.md @@ -1,6 +1,6 @@ # LDAP Module -This module provisions an LDAP server for authenticating users in Presto. This -also enables SSL / TLS between the LDAP server and Presto, and between Presto +This module provisions an LDAP server for authenticating users in Trino. This +also enables SSL / TLS between the LDAP server and Trino, and between Trino and clients. It is compatible with other security modules like **system-ranger** and **event-logger**, but is mutually-exclusive of the **password-file** module. @@ -11,42 +11,42 @@ and **event-logger**, but is mutually-exclusive of the **password-file** module. To provision this module, run: ```shell -minipresto provision --module ldap +minitrino provision --module ldap ``` ## Default Usernames and Passwords -- alice / prestoRocks15 -- bob / prestoRocks15 +- alice / trinoRocks15 +- bob / trinoRocks15 ## Client Keystore and Truststore The Java keystore and truststore needed for clients and drivers to securely -connect to Presto are located in a volume mount `~/.minipresto/ssl`. These two -files are transient and will be automatically replaced whenever Minipresto is +connect to Trino are located in a volume mount `~/.minitrino/ssl`. These two +files are transient and will be automatically replaced whenever Minitrino is provisioned with a security module that enables SSL. -## Accessing Presto with the CLI +## Accessing Trino with the CLI Via Docker: ``` -docker exec -it presto presto-cli --server https://presto:8443 \ - --truststore-path /usr/lib/presto/etc/ssl/truststore.jks --truststore-password prestoRocks15 \ - --keystore-path /usr/lib/presto/etc/ssl/keystore.jks --keystore-password prestoRocks15 \ +docker exec -it trino trino-cli --server https://trino:8443 \ + --truststore-path /etc/starburst/ssl/truststore.jks --truststore-password trinoRocks15 \ + --keystore-path /etc/starburst/ssl/keystore.jks --keystore-password trinoRocks15 \ --user bob --password ``` Via Host Machine: ``` -presto-cli-xxx-executable.jar --server https://localhost:8443 \ - --truststore-path ~/.minipresto/ssl/truststore.jks --truststore-password prestoRocks15 \ - --keystore-path ~/.minipresto/ssl/keystore.jks --keystore-password prestoRocks15 \ +trino-cli-xxx-executable.jar --server https://localhost:8443 \ + --truststore-path ~/.minitrino/ssl/truststore.jks --truststore-password trinoRocks15 \ + --keystore-path ~/.minitrino/ssl/keystore.jks --keystore-password trinoRocks15 \ --user bob --password ``` Note that the CLI will prompt you for the password. -## Accessing the Presto Web UI +## Accessing the Trino Web UI Open a web browser and go to https://localhost:8443 and log in with a valid LDAP username and password. @@ -81,12 +81,12 @@ objectClass: top cn: jeff sn: jeff mail: jeff@example.com -userPassword: prestoRocks15 +userPassword: trinoRocks15 EOF ``` 3. Use the **ldapmodify** tool to add the new user ``` -ldapmodify -x -D "cn=admin,dc=example,dc=com" -w prestoRocks15 -H ldaps://ldap:636 -f jeff.ldif +ldapmodify -x -D "cn=admin,dc=example,dc=com" -w trinoRocks15 -H ldaps://ldap:636 -f jeff.ldif ``` diff --git a/lib/modules/security/ldap/resources/bootstrap/bootstrap-presto.sh b/lib/modules/security/ldap/resources/bootstrap/bootstrap-trino.sh similarity index 60% rename from lib/modules/security/ldap/resources/bootstrap/bootstrap-presto.sh rename to lib/modules/security/ldap/resources/bootstrap/bootstrap-trino.sh index c98e2a0f..0f858e54 100755 --- a/lib/modules/security/ldap/resources/bootstrap/bootstrap-presto.sh +++ b/lib/modules/security/ldap/resources/bootstrap/bootstrap-trino.sh @@ -4,24 +4,24 @@ set -euxo pipefail export LDAPTLS_REQCERT=never echo "Waiting for LDAP to come up..." -/opt/minipresto/wait-for-it.sh ldap:636 --strict --timeout=60 -- echo "LDAP service is up." +/opt/minitrino/wait-for-it.sh ldap:636 --strict --timeout=60 -- echo "LDAP service is up." echo "Creating certs directory..." -PRESTO_CERTS=/usr/lib/presto/etc/certs -if [ ! -d "${PRESTO_CERTS}" ]; then - mkdir "${PRESTO_CERTS}" +TRINO_CERTS=/etc/starburst/certs +if [ ! -d "${TRINO_CERTS}" ]; then + mkdir "${TRINO_CERTS}" fi echo "Setting variables..." TRUSTSTORE_PATH=/etc/pki/java/cacerts TRUSTSTORE_DEFAULT_PASS=changeit -TRUSTSTORE_PASS=prestoRocks15 -KEYSTORE_PASS=prestoRocks15 -SSL_DIR=/usr/lib/presto/etc/ssl +TRUSTSTORE_PASS=trinoRocks15 +KEYSTORE_PASS=trinoRocks15 +SSL_DIR=/etc/starburst/ssl -PRESTO_JAVA_OPTS="-Djavax.net.ssl.trustStore=${TRUSTSTORE_PATH} \n" -PRESTO_JAVA_OPTS="${PRESTO_JAVA_OPTS}-Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASS} \n" -PRESTO_JAVA_OPTS="${PRESTO_JAVA_OPTS}-Djavax.net.debug=ssl:handshake:verbose \n" +TRINO_JAVA_OPTS="-Djavax.net.ssl.trustStore=${TRUSTSTORE_PATH} \n" +TRINO_JAVA_OPTS="${TRINO_JAVA_OPTS}-Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASS} \n" +TRINO_JAVA_OPTS="${TRINO_JAVA_OPTS}-Djavax.net.debug=ssl:handshake:verbose \n" echo "Removing pre-existing SSL resources..." # These should be removed prior to provisioning to ensure they do not conflate @@ -30,13 +30,13 @@ rm -f "${SSL_DIR}"/* echo "Getting LDAP certificate..." sudo yum install -y openssl -LDAP_URI=$(cat /usr/lib/presto/etc/password-authenticator.properties | grep "ldaps" | sed -r "s/^.*ldaps:\/\/(.+:[0-9]+).*$/\1/") +LDAP_URI=$(cat /etc/starburst/password-authenticator.properties | grep "ldaps" | sed -r "s/^.*ldaps:\/\/(.+:[0-9]+).*$/\1/") LDAP_HOST=$(echo "${LDAP_URI}" | cut -d ':' -f 1) LDAP_PORT=$(echo "${LDAP_URI}" | cut -d ':' -f 2) LDAP_IP=$(ping -c 1 "${LDAP_HOST}" | grep "PING ${LDAP_HOST}" | sed -r "s/^.+\(([0-9]+(\.[0-9]+)+)\).+$/\1/") if [[ "${LDAP_URI}" != "" ]]; then - LDAP_CERT_FILE="${PRESTO_CERTS}"/ldapserver.crt + LDAP_CERT_FILE="${TRINO_CERTS}"/ldapserver.crt echo "LDAP IP resolver from [${LDAP_URI}] -> [${LDAP_IP}]" set +e && echo "Q" | openssl s_client -showcerts -connect "${LDAP_IP}:${LDAP_PORT}" > "${LDAP_CERT_FILE}" && set -e echo "LDAP SSL certificate downloaded from [${LDAP_IP}:${LDAP_PORT}] and saved in [${LDAP_CERT_FILE}]" @@ -44,13 +44,13 @@ fi echo "Generating keystore file..." keytool -genkeypair \ - -alias presto \ + -alias trino \ -keyalg RSA \ -keystore "${SSL_DIR}"/keystore.jks \ -keypass "${KEYSTORE_PASS}" \ -storepass "${KEYSTORE_PASS}" \ -dname "CN=*.starburstdata.com" \ - -ext san=dns:presto.minipresto.starburstdata.com,dns:presto,dns:localhost + -ext san=dns:trino.minitrino.starburstdata.com,dns:trino,dns:localhost echo "Change truststore password..." keytool -storepasswd \ @@ -59,7 +59,7 @@ keytool -storepasswd \ -keystore "${TRUSTSTORE_PATH}" echo "Importing external certificates..." -ls "${PRESTO_CERTS}"/* | while read -r CERT_FILE; +ls "${TRINO_CERTS}"/* | while read -r CERT_FILE; do sudo keytool -import \ -keystore "${TRUSTSTORE_PATH}" \ @@ -71,37 +71,37 @@ do done; sudo yum install -y openldap-clients -ls /usr/lib/presto/etc/ldap-users/*.ldif | while read -r LDIF_FILE; +ls /etc/starburst/ldap-users/*.ldif | while read -r LDIF_FILE; do echo "LDAP Importing user from file [${LDIF_FILE}]" - ldapmodify -x -D "cn=admin,dc=example,dc=com" -w prestoRocks15 -H ldaps://"${LDAP_IP}":"${LDAP_PORT}" -f "${LDIF_FILE}" + ldapmodify -x -D "cn=admin,dc=example,dc=com" -w trinoRocks15 -H ldaps://"${LDAP_IP}":"${LDAP_PORT}" -f "${LDIF_FILE}" done; echo "Adding JVM configs..." -echo -e "${PRESTO_JAVA_OPTS}" >> /usr/lib/presto/etc/jvm.config +echo -e "${TRINO_JAVA_OPTS}" >> /etc/starburst/jvm.config -echo "Adding Presto configs..." -cat <> /usr/lib/presto/etc/config.properties +echo "Adding Trino configs..." +cat <> /etc/starburst/config.properties http-server.authentication.type=PASSWORD http-server.https.enabled=true http-server.https.port=8443 -http-server.https.keystore.path=/usr/lib/presto/etc/ssl/keystore.jks -http-server.https.keystore.key=prestoRocks15 +http-server.https.keystore.path=/etc/starburst/ssl/keystore.jks +http-server.https.keystore.key=trinoRocks15 EOT echo "Adding keystore and truststore in ${SSL_DIR}..." keytool -export \ - -alias presto \ + -alias trino \ -keystore "${SSL_DIR}"/keystore.jks \ -rfc \ - -file "${SSL_DIR}"/presto_certificate.cer \ + -file "${SSL_DIR}"/trino_certificate.cer \ -storepass "${KEYSTORE_PASS}" \ -noprompt keytool -import -v \ -trustcacerts \ - -alias presto_trust \ - -file "${SSL_DIR}"/presto_certificate.cer \ + -alias trino_trust \ + -file "${SSL_DIR}"/trino_certificate.cer \ -keystore "${SSL_DIR}"/truststore.jks \ -storepass "${TRUSTSTORE_PASS}" \ -noprompt diff --git a/lib/modules/security/ldap/resources/ldap/ldap-users/alice.ldif b/lib/modules/security/ldap/resources/ldap/ldap-users/alice.ldif index 573fe832..8c3139b5 100644 --- a/lib/modules/security/ldap/resources/ldap/ldap-users/alice.ldif +++ b/lib/modules/security/ldap/resources/ldap/ldap-users/alice.ldif @@ -9,4 +9,4 @@ objectClass: top cn: alice sn: alice mail: alice@example.com -userPassword: prestoRocks15 \ No newline at end of file +userPassword: trinoRocks15 \ No newline at end of file diff --git a/lib/modules/security/ldap/resources/ldap/ldap-users/bob.ldif b/lib/modules/security/ldap/resources/ldap/ldap-users/bob.ldif index e49b31c9..f24189c9 100644 --- a/lib/modules/security/ldap/resources/ldap/ldap-users/bob.ldif +++ b/lib/modules/security/ldap/resources/ldap/ldap-users/bob.ldif @@ -9,4 +9,4 @@ objectClass: top cn: bob sn: bob mail: bob@example.com -userPassword: prestoRocks15 \ No newline at end of file +userPassword: trinoRocks15 \ No newline at end of file diff --git a/lib/modules/security/ldap/resources/presto/password-authenticator.properties b/lib/modules/security/ldap/resources/trino/password-authenticator.properties similarity index 59% rename from lib/modules/security/ldap/resources/presto/password-authenticator.properties rename to lib/modules/security/ldap/resources/trino/password-authenticator.properties index 737f01cc..71b9aaf7 100644 --- a/lib/modules/security/ldap/resources/presto/password-authenticator.properties +++ b/lib/modules/security/ldap/resources/trino/password-authenticator.properties @@ -1,3 +1,3 @@ password-authenticator.name=ldap -ldap.url=ldaps://ldap.minipresto.starburstdata.com:636 +ldap.url=ldaps://ldap.minitrino.starburstdata.com:636 ldap.user-bind-pattern=uid=${USER},dc=example,dc=com diff --git a/lib/modules/security/password-file/metadata.json b/lib/modules/security/password-file/metadata.json index a3f5009c..0956671f 100644 --- a/lib/modules/security/password-file/metadata.json +++ b/lib/modules/security/password-file/metadata.json @@ -1,4 +1,5 @@ { - "description": "Secures Presto with file-based password authentication.", - "incompatible_modules": ["ldap"] + "description": "Secures Trino with file-based password authentication.", + "incompatibleModules": ["ldap"], + "enterprise": false } \ No newline at end of file diff --git a/lib/modules/security/password-file/password-file.yml b/lib/modules/security/password-file/password-file.yml index 53fea980..1564f5ca 100755 --- a/lib/modules/security/password-file/password-file.yml +++ b/lib/modules/security/password-file/password-file.yml @@ -1,13 +1,13 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/security/password-file/resources/presto/password-authenticator.properties:/usr/lib/presto/etc/password-authenticator.properties" - - "~/.minipresto/ssl:/usr/lib/presto/etc/ssl" + - "./modules/security/password-file/resources/trino/password-authenticator.properties:/etc/starburst/password-authenticator.properties" + - "~/.minitrino/ssl:/etc/starburst/ssl" labels: - "com.starburst.tests.module.password-file=security-password-file" environment: - MINIPRESTO_BOOTSTRAP: "bootstrap-presto.sh" + MINITRINO_BOOTSTRAP: "bootstrap-trino.sh" ports: - "8443:8443" diff --git a/lib/modules/security/password-file/readme.md b/lib/modules/security/password-file/readme.md index c3558429..486f4517 100644 --- a/lib/modules/security/password-file/readme.md +++ b/lib/modules/security/password-file/readme.md @@ -1,6 +1,6 @@ # Password File Authentication Module -This module configures Presto to authenticate users with a password file. It -also enables SSL / TLS between Presto and clients. It is compatible with other +This module configures Trino to authenticate users with a password file. It +also enables SSL / TLS between Trino and clients. It is compatible with other security modules like **system-ranger** and **event-logger**, but is mutually-exclusive of the **ldap** module. @@ -11,42 +11,42 @@ mutually-exclusive of the **ldap** module. To provision this module, run: ```shell -minipresto provision --module ldap +minitrino provision --module ldap ``` ## Default Usernames and Passwords -- alice / prestoRocks15 -- bob / prestoRocks15 +- alice / trinoRocks15 +- bob / trinoRocks15 ## Client Keystore and Truststore The Java keystore and truststore needed for clients and drivers to securely -connect to Presto are located in a volume mount `~/.minipresto/ssl`. These two -files are transient and will be automatically replaced whenever Minipresto is +connect to Trino are located in a volume mount `~/.minitrino/ssl`. These two +files are transient and will be automatically replaced whenever Minitrino is provisioned with a security module that enables SSL. -## Accessing Presto with the CLI +## Accessing Trino with the CLI Via Docker: ``` -docker exec -it presto presto-cli --server https://presto:8443 \ - --truststore-path /usr/lib/presto/etc/ssl/truststore.jks --truststore-password prestoRocks15 \ - --keystore-path /usr/lib/presto/etc/ssl/keystore.jks --keystore-password prestoRocks15 \ +docker exec -it trino trino-cli --server https://trino:8443 \ + --truststore-path /etc/starburst/ssl/truststore.jks --truststore-password trinoRocks15 \ + --keystore-path /etc/starburst/ssl/keystore.jks --keystore-password trinoRocks15 \ --user bob --password ``` Via Host Machine: ``` -presto-cli-xxx-executable.jar --server https://localhost:8443 \ - --truststore-path ~/.minipresto/ssl/truststore.jks --truststore-password prestoRocks15 \ - --keystore-path ~/.minipresto/ssl/keystore.jks --keystore-password prestoRocks15 \ +trino-cli-xxx-executable.jar --server https://localhost:8443 \ + --truststore-path ~/.minitrino/ssl/truststore.jks --truststore-password trinoRocks15 \ + --keystore-path ~/.minitrino/ssl/keystore.jks --keystore-password trinoRocks15 \ --user bob --password ``` Note that the CLI will prompt you for the password. -## Accessing the Presto Web UI +## Accessing the Trino Web UI Open a web browser and go to https://localhost:8443 and log in with a valid username and password. @@ -62,8 +62,8 @@ this website**. ## Adding a New User to the Password File -Example with username `jeff` and password `prestoRocks15` +Example with username `jeff` and password `trinoRocks15` ``` -docker exec presto htpasswd -bB -C 10 /usr/lib/presto/etc/password.db jeff prestoRocks15 +docker exec trino htpasswd -bB -C 10 /etc/starburst/password.db jeff trinoRocks15 ``` diff --git a/lib/modules/security/password-file/resources/bootstrap/bootstrap-presto.sh b/lib/modules/security/password-file/resources/bootstrap/bootstrap-trino.sh similarity index 55% rename from lib/modules/security/password-file/resources/bootstrap/bootstrap-presto.sh rename to lib/modules/security/password-file/resources/bootstrap/bootstrap-trino.sh index a69bb27a..58f13d71 100755 --- a/lib/modules/security/password-file/resources/bootstrap/bootstrap-presto.sh +++ b/lib/modules/security/password-file/resources/bootstrap/bootstrap-trino.sh @@ -5,13 +5,13 @@ set -euxo pipefail echo "Setting variables..." TRUSTSTORE_PATH=/etc/pki/java/cacerts TRUSTSTORE_DEFAULT_PASS=changeit -TRUSTSTORE_PASS=prestoRocks15 -KEYSTORE_PASS=prestoRocks15 -SSL_DIR=/usr/lib/presto/etc/ssl +TRUSTSTORE_PASS=trinoRocks15 +KEYSTORE_PASS=trinoRocks15 +SSL_DIR=/etc/starburst/ssl -PRESTO_JAVA_OPTS="-Djavax.net.ssl.trustStore=${TRUSTSTORE_PATH} \n" -PRESTO_JAVA_OPTS="${PRESTO_JAVA_OPTS}-Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASS} \n" -PRESTO_JAVA_OPTS="${PRESTO_JAVA_OPTS}-Djavax.net.debug=ssl:handshake:verbose \n" +TRINO_JAVA_OPTS="-Djavax.net.ssl.trustStore=${TRUSTSTORE_PATH} \n" +TRINO_JAVA_OPTS="${TRINO_JAVA_OPTS}-Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASS} \n" +TRINO_JAVA_OPTS="${TRINO_JAVA_OPTS}-Djavax.net.debug=ssl:handshake:verbose \n" echo "Removing pre-existing SSL resources..." # These should be removed prior to provisioning to ensure they do not conflate @@ -19,13 +19,13 @@ echo "Removing pre-existing SSL resources..." rm -f "${SSL_DIR}"/* echo "Generating keystore file..." keytool -genkeypair \ - -alias presto \ + -alias trino \ -keyalg RSA \ -keystore "${SSL_DIR}"/keystore.jks \ -keypass "${KEYSTORE_PASS}" \ -storepass "${KEYSTORE_PASS}" \ -dname "CN=*.starburstdata.com" \ - -ext san=dns:presto.minipresto.starburstdata.com,dns:presto,dns:localhost + -ext san=dns:trino.minitrino.starburstdata.com,dns:trino,dns:localhost echo "Change truststore password..." keytool -storepasswd \ @@ -34,35 +34,35 @@ keytool -storepasswd \ -keystore "${TRUSTSTORE_PATH}" echo "Adding JVM configs..." -echo -e "${PRESTO_JAVA_OPTS}" >> /usr/lib/presto/etc/jvm.config +echo -e "${TRINO_JAVA_OPTS}" >> /etc/starburst/jvm.config -echo "Adding Presto configs..." -cat <> /usr/lib/presto/etc/config.properties +echo "Adding Trino configs..." +cat <> /etc/starburst/config.properties http-server.authentication.type=PASSWORD http-server.https.enabled=true http-server.https.port=8443 -http-server.https.keystore.path=/usr/lib/presto/etc/ssl/keystore.jks -http-server.https.keystore.key=prestoRocks15 +http-server.https.keystore.path=/etc/starburst/ssl/keystore.jks +http-server.https.keystore.key=trinoRocks15 EOT echo "Adding keystore and truststore in ${SSL_DIR}..." keytool -export \ - -alias presto \ + -alias trino \ -keystore "${SSL_DIR}"/keystore.jks \ -rfc \ - -file "${SSL_DIR}"/presto_certificate.cer \ + -file "${SSL_DIR}"/trino_certificate.cer \ -storepass "${KEYSTORE_PASS}" \ -noprompt keytool -import -v \ -trustcacerts \ - -alias presto_trust \ - -file "${SSL_DIR}"/presto_certificate.cer \ + -alias trino_trust \ + -file "${SSL_DIR}"/trino_certificate.cer \ -keystore "${SSL_DIR}"/truststore.jks \ -storepass "${TRUSTSTORE_PASS}" \ -noprompt echo "Setting up password file..." sudo yum install httpd-tools -y -htpasswd -cbB -C 10 /usr/lib/presto/etc/password.db alice prestoRocks15 -htpasswd -bB -C 10 /usr/lib/presto/etc/password.db bob prestoRocks15 +htpasswd -cbB -C 10 /etc/starburst/password.db alice trinoRocks15 +htpasswd -bB -C 10 /etc/starburst/password.db bob trinoRocks15 diff --git a/lib/modules/security/password-file/resources/presto/password-authenticator.properties b/lib/modules/security/password-file/resources/presto/password-authenticator.properties deleted file mode 100644 index 7c86d11e..00000000 --- a/lib/modules/security/password-file/resources/presto/password-authenticator.properties +++ /dev/null @@ -1,2 +0,0 @@ -password-authenticator.name=file -file.password-file=/usr/lib/presto/etc/password.db diff --git a/lib/modules/security/password-file/resources/trino/password-authenticator.properties b/lib/modules/security/password-file/resources/trino/password-authenticator.properties new file mode 100644 index 00000000..ad386170 --- /dev/null +++ b/lib/modules/security/password-file/resources/trino/password-authenticator.properties @@ -0,0 +1,2 @@ +password-authenticator.name=file +file.password-file=/etc/starburst/password.db diff --git a/lib/modules/security/system-ranger/metadata.json b/lib/modules/security/system-ranger/metadata.json index 99a6c775..2562383a 100644 --- a/lib/modules/security/system-ranger/metadata.json +++ b/lib/modules/security/system-ranger/metadata.json @@ -1,4 +1,5 @@ { - "description": "Secures Presto with Ranger role-based access control.", - "incompatible_modules": [] + "description": "Secures Trino with Ranger system access control.", + "incompatibleModules": ["file-access-control"], + "enterprise": true } \ No newline at end of file diff --git a/lib/modules/security/system-ranger/readme.md b/lib/modules/security/system-ranger/readme.md index 337369e6..d3a7eb86 100644 --- a/lib/modules/security/system-ranger/readme.md +++ b/lib/modules/security/system-ranger/readme.md @@ -9,14 +9,13 @@ This module provisions a Ranger server and Postgres database for Ranger storage. To provision this module, run: ```shell -minipresto provision --module system-ranger +minitrino provision --module system-ranger ``` ## Policies - Bob: admin access to TPCH `sf100` schema - Alice: admin access to TPCH `sf10` schema -- System: all users have select/ownership access to `system` catalog ## Accessing Ranger - Go to `localhost:6080` -- Sign in with user: admin and pass: prestoRocks15 +- Sign in with user: admin and pass: trinoRocks15 diff --git a/lib/modules/security/system-ranger/resources/bootstrap/bootstrap-ranger.sh b/lib/modules/security/system-ranger/resources/bootstrap/bootstrap-ranger.sh index 2df46246..fef57aac 100755 --- a/lib/modules/security/system-ranger/resources/bootstrap/bootstrap-ranger.sh +++ b/lib/modules/security/system-ranger/resources/bootstrap/bootstrap-ranger.sh @@ -2,401 +2,21 @@ set -euxo pipefail -echo "Waiting for Ranger Admin to come up..." -/opt/minipresto/wait-for-it.sh ranger-admin:6080 --strict --timeout=150 -- echo "Ranger Admin service is up." - -function create_sep_service() { - curl -i -v -X POST -u admin:prestoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' - { - "name":"minipresto", - "description":"Minipresto SEP service", - "isEnabled":true, - "tagService":"", - "configs":{ - "username":"presto_admin", - "password":"prestoRocks15", - "jdbc.driverClassName":"io.prestosql.jdbc.PrestoDriver", - "jdbc.url":"jdbc:presto://localhost:8080", - "resource-lookup":"true" - }, - "type":"starburst-enterprise-presto" - } - ' \ - 'http://localhost:6080/service/plugins/services'; -} - -COUNTER=0 && set +e -while [[ "${COUNTER}" -le 36 ]]; do - if create_sep_service | grep -q "HTTP/1.1 200 OK"; then - break - else - sleep 5 - ((COUNTER++)) - fi -done -set -e - -if [[ "${COUNTER}" == 36 ]]; then - echo "Timeout waiting for Starburst Enterprise Presto service to become available in Ranger Admin. Exiting." - exit 1 -fi +echo "Downloading Ranger CLI..." +STARBURST_RANGER_CLI_PATH=/usr/local/bin/starburst-ranger-cli +DIST=${STARBURST_VER:0:3} +CLI_URL=https://s3.us-east-2.amazonaws.com/software.starburstdata.net/"${DIST}"e/"${STARBURST_VER}"/starburst-ranger-cli-"${STARBURST_VER}"-executable.jar -# Create users -curl -i -v -X POST -u admin:prestoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' -{ - "name":"bob", - "password":"prestoRocks15", - "firstName":"Bob", - "lastName":"", - "emailAddress":"bob@starburstdata.com", - "userRoleList":[ - "ROLE_USER" - ], - "groupIdList":[ - "1" - ], - "status":1 -} -' \ -'http://localhost:6080/service/xusers/secure/users'; +curl -fsSL "${CLI_URL}" > "${STARBURST_RANGER_CLI_PATH}" +chmod -v +x "${STARBURST_RANGER_CLI_PATH}" +ln -vs "${STARBURST_RANGER_CLI_PATH}" starburst-ranger-cli -curl -i -v -X POST -u admin:prestoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' -{ - "name":"alice", - "password":"prestoRocks15", - "firstName":"Alice", - "lastName":"", - "emailAddress":"alice@starburstdata.com", - "userRoleList":[ - "ROLE_USER" - ], - "groupIdList":[ - "1" - ], - "status":1 -} -' \ -'http://localhost:6080/service/xusers/secure/users'; - -# Create User Policies -curl -i -v -X POST -u admin:prestoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' -{ - "policyType":"0", - "name":"Bob", - "isEnabled":true, - "policyPriority":0, - "policyLabels":[ - - ], - "description":"", - "isAuditEnabled":true, - "resources":{ - "catalog":{ - "values":[ - "tpch" - ], - "isRecursive":false, - "isExcludes":false - }, - "schema":{ - "values":[ - "sf100" - ], - "isRecursive":false, - "isExcludes":false - }, - "table":{ - "values":[ - "*" - ], - "isRecursive":false, - "isExcludes":false - }, - "column":{ - "values":[ - "*" - ], - "isRecursive":false, - "isExcludes":false - } - }, - "isDenyAllElse":false, - "policyItems":[ - { - "users":[ - "bob" - ], - "delegateAdmin":true, - "accesses":[ - { - "type":"select", - "isAllowed":true - }, - { - "type":"insert", - "isAllowed":true - }, - { - "type":"delete", - "isAllowed":true - }, - { - "type":"update", - "isAllowed":true - }, - { - "type":"ownership", - "isAllowed":true - } - ] - } - ], - "allowExceptions":[ - - ], - "denyPolicyItems":[ - - ], - "denyExceptions":[ - - ], - "service":"minipresto" -} -' \ -'http://localhost:6080/service/plugins/policies'; - -curl -i -v -X POST -u admin:prestoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' -{ - "policyType":"0", - "name":"Alice", - "isEnabled":true, - "policyPriority":0, - "policyLabels":[ - - ], - "description":"", - "isAuditEnabled":true, - "resources":{ - "catalog":{ - "values":[ - "tpch" - ], - "isRecursive":false, - "isExcludes":false - }, - "schema":{ - "values":[ - "sf1" - ], - "isRecursive":false, - "isExcludes":false - }, - "table":{ - "values":[ - "*" - ], - "isRecursive":false, - "isExcludes":false - }, - "column":{ - "values":[ - "*" - ], - "isRecursive":false, - "isExcludes":false - } - }, - "isDenyAllElse":false, - "policyItems":[ - { - "users":[ - "alice" - ], - "delegateAdmin":true, - "accesses":[ - { - "type":"select", - "isAllowed":true - }, - { - "type":"insert", - "isAllowed":true - }, - { - "type":"delete", - "isAllowed":true - }, - { - "type":"update", - "isAllowed":true - }, - { - "type":"ownership", - "isAllowed":true - } - ] - } - ], - "allowExceptions":[ - - ], - "denyPolicyItems":[ - - ], - "denyExceptions":[ - - ], - "service":"minipresto" -} -' \ -'http://localhost:6080/service/plugins/policies'; +echo "Waiting for Ranger Admin to come up..." +/opt/minitrino/wait-for-it.sh ranger-admin:6080 --strict --timeout=150 -- echo "Ranger Admin service is up." -curl -i -v -X POST -u admin:prestoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' -{ - "policyType":"0", - "name":"System", - "isEnabled":true, - "policyPriority":0, - "policyLabels":[ - - ], - "description":"", - "isAuditEnabled":true, - "resources":{ - "catalog":{ - "values":[ - "system" - ], - "isRecursive":false, - "isExcludes":false - }, - "schema":{ - "values":[ - "*" - ], - "isRecursive":false, - "isExcludes":false - }, - "table":{ - "values":[ - "*" - ], - "isRecursive":false, - "isExcludes":false - }, - "column":{ - "values":[ - "*" - ], - "isRecursive":false, - "isExcludes":false - } - }, - "isDenyAllElse":false, - "policyItems":[ - { - "users":[ - "{USER}" - ], - "accesses":[ - { - "type":"select", - "isAllowed":true - }, - { - "type":"ownership", - "isAllowed":true - } - ] - } - ], - "allowExceptions":[ - - ], - "denyPolicyItems":[ - - ], - "denyExceptions":[ - - ], - "service":"minipresto" -} -' \ -'http://localhost:6080/service/plugins/policies'; +echo "Initializing Ranger Admin with service, users, and policies..." +chmod -R +x /tmp/ranger -# Add users to query policy -curl -i -v -X PUT -u admin:prestoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' -{ - "id":2, - "isEnabled":true, - "service":"minipresto", - "name":"all - query", - "policyType":0, - "policyPriority":0, - "description":"Policy for all - query", - "isAuditEnabled":true, - "resources":{ - "query":{ - "values":[ - "*" - ], - "isRecursive":false, - "isExcludes":false - } - }, - "policyItems":[ - { - "users":[ - "presto_admin", - "{USER}" - ], - "delegateAdmin":true, - "accesses":[ - { - "type":"select", - "isAllowed":true - }, - { - "type":"execute", - "isAllowed":true - }, - { - "type":"kill", - "isAllowed":true - } - ] - } - ], - "denyPolicyItems":[ - - ], - "allowExceptions":[ - - ], - "denyExceptions":[ - - ], - "dataMaskPolicyItems":[ - - ], - "rowFilterPolicyItems":[ - - ], - "serviceType":"203", - "options":{ - - }, - "validitySchedules":[ - - ], - "policyLabels":[ - - ], - "zoneName":"", - "isDenyAllElse":false, - "sameLevel20catalog":null, - "sameLevel30schema":null, - "column":null -} -' \ -'http://localhost:6080/service/plugins/policies/2'; +/tmp/ranger/create-service.sh +/tmp/ranger/create-users.sh +/tmp/ranger/add-user-policies.sh diff --git a/lib/modules/security/system-ranger/resources/ranger-admin/ranger-admin.env b/lib/modules/security/system-ranger/resources/ranger-admin/ranger-admin.env index a67c4c08..35314e00 100644 --- a/lib/modules/security/system-ranger/resources/ranger-admin/ranger-admin.env +++ b/lib/modules/security/system-ranger/resources/ranger-admin/ranger-admin.env @@ -1,12 +1,12 @@ -RANGER__db_root_password=prestoRocks15 -RANGER__db_password=prestoRocks15 -RANGER__rangerUsersync_password=prestoRocks15 -RANGER__rangerTagsync_password=prestoRocks15 -RANGER__keyadmin_password=prestoRocks15 -RANGER__rangerAdmin_password=prestoRocks15 +RANGER__db_root_password=trinoRocks15 +RANGER__db_password=trinoRocks15 +RANGER__rangerUsersync_password=trinoRocks15 +RANGER__rangerTagsync_password=trinoRocks15 +RANGER__keyadmin_password=trinoRocks15 +RANGER__rangerAdmin_password=trinoRocks15 RANGER__db_host=ranger-postgres RANGER__db_name=ranger RANGER__db_root_user=admin RANGER__db_user=ranger -prestoServiceUser=presto_admin -prestoServicePassword=prestoRocks15 \ No newline at end of file +starburstServiceUser=starburst_service +starburstServicePassword=trinoRocks15 \ No newline at end of file diff --git a/lib/modules/security/system-ranger/resources/ranger-admin/ranger-postgres.env b/lib/modules/security/system-ranger/resources/ranger-admin/ranger-postgres.env index 269dc293..13471b0c 100644 --- a/lib/modules/security/system-ranger/resources/ranger-admin/ranger-postgres.env +++ b/lib/modules/security/system-ranger/resources/ranger-admin/ranger-postgres.env @@ -1,3 +1,3 @@ POSTGRES_USER=admin -POSTGRES_PASSWORD=prestoRocks15 +POSTGRES_PASSWORD=trinoRocks15 POSTGRES_DB=ranger \ No newline at end of file diff --git a/lib/modules/security/system-ranger/resources/presto/access-control.properties b/lib/modules/security/system-ranger/resources/trino/access-control.properties similarity index 54% rename from lib/modules/security/system-ranger/resources/presto/access-control.properties rename to lib/modules/security/system-ranger/resources/trino/access-control.properties index f0580185..be2e956c 100644 --- a/lib/modules/security/system-ranger/resources/presto/access-control.properties +++ b/lib/modules/security/system-ranger/resources/trino/access-control.properties @@ -1,7 +1,7 @@ access-control.name=ranger ranger.authentication-type=BASIC ranger.policy-rest-url=http://ranger-admin:6080 -ranger.service-name=minipresto -ranger.presto-plugin-username=presto_admin -ranger.presto-plugin-password=prestoRocks15 +ranger.service-name=starburst +ranger.username=starburst_service +ranger.password=trinoRocks15 ranger.policy-refresh-interval=10s diff --git a/lib/modules/security/system-ranger/resources/trino/ranger/add-user-policies.sh b/lib/modules/security/system-ranger/resources/trino/ranger/add-user-policies.sh new file mode 100755 index 00000000..5df2b495 --- /dev/null +++ b/lib/modules/security/system-ranger/resources/trino/ranger/add-user-policies.sh @@ -0,0 +1,652 @@ +#!/usr/bin/env bash + +set -ex + +# all - catalog, session-property +curl -i -v -X PUT -u admin:trinoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' +{ + "id":1, + "isEnabled":true, + "createdBy":"Admin", + "updatedBy":"Admin", + "version":1, + "service":"starburst", + "name":"all - catalog, session-property", + "policyType":0, + "policyPriority":0, + "description":"Policy for all - catalog, session-property", + "isAuditEnabled":true, + "resources":{ + "catalog":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + }, + "session-property":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + } + }, + "policyItems":[ + { + "users":[ + "starburst_service", + "{USER}" + ], + "delegateAdmin":true, + "accesses":[ + { + "type":"update", + "isAllowed":true + } + ] + } + ], + "denyPolicyItems":[ + + ], + "allowExceptions":[ + + ], + "denyExceptions":[ + + ], + "dataMaskPolicyItems":[ + + ], + "rowFilterPolicyItems":[ + + ], + "serviceType":"203", + "options":{ + + }, + "validitySchedules":[ + + ], + "policyLabels":[ + + ], + "zoneName":"", + "isDenyAllElse":false, + "sameLevel30schema":null, + "column":null +}' \ +"http://ranger-admin:6080/service/plugins/policies/1"; + +# all - query +curl -i -v -X PUT -u admin:trinoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' +{ + "id":2, + "guid":"25e258a8-82f6-48d5-9db0-1bfd9641a454", + "isEnabled":true, + "createdBy":"Admin", + "updatedBy":"Admin", + "version":1, + "service":"starburst", + "name":"all - query", + "policyType":0, + "policyPriority":0, + "description":"Policy for all - query", + "isAuditEnabled":true, + "resources":{ + "query":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + } + }, + "policyItems":[ + { + "users":[ + "starburst_service", + "{USER}" + ], + "delegateAdmin":true, + "accesses":[ + { + "type":"select", + "isAllowed":true + }, + { + "type":"execute", + "isAllowed":true + }, + { + "type":"kill", + "isAllowed":true + } + ] + } + ], + "denyPolicyItems":[ + + ], + "allowExceptions":[ + + ], + "denyExceptions":[ + + ], + "dataMaskPolicyItems":[ + + ], + "rowFilterPolicyItems":[ + + ], + "serviceType":"203", + "options":{ + + }, + "validitySchedules":[ + + ], + "policyLabels":[ + + ], + "zoneName":"", + "isDenyAllElse":false, + "sameLevel20catalog":null, + "sameLevel30schema":null, + "column":null +}' \ +"http://ranger-admin:6080/service/plugins/policies/2"; + +# all - system-session-property +curl -i -v -X PUT -u admin:trinoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' +{ + "id":3, + "isEnabled":true, + "createdBy":"Admin", + "updatedBy":"Admin", + "version":1, + "service":"starburst", + "name":"all - system-session-property", + "policyType":0, + "policyPriority":0, + "description":"Policy for all - system-session-property", + "isAuditEnabled":true, + "resources":{ + "system-session-property":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + } + }, + "policyItems":[ + { + "users":[ + "starburst_service", + "{USER}" + ], + "delegateAdmin":true, + "accesses":[ + { + "type":"update", + "isAllowed":true + } + ] + } + ], + "denyPolicyItems":[ + + ], + "allowExceptions":[ + + ], + "denyExceptions":[ + + ], + "dataMaskPolicyItems":[ + + ], + "rowFilterPolicyItems":[ + + ], + "serviceType":"203", + "options":{ + + }, + "validitySchedules":[ + + ], + "policyLabels":[ + + ], + "zoneName":"", + "isDenyAllElse":false, + "sameLevel20catalog":null, + "sameLevel30schema":null, + "column":null +}' \ +"http://ranger-admin:6080/service/plugins/policies/3"; + +# all - function +curl -i -v -X PUT -u admin:trinoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' +{ + "id":4, + "isEnabled":true, + "createdBy":"Admin", + "updatedBy":"Admin", + "version":1, + "service":"starburst", + "name":"all - function", + "policyType":0, + "policyPriority":0, + "description":"Policy for all - function", + "isAuditEnabled":true, + "resources":{ + "function":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + } + }, + "policyItems":[ + { + "users":[ + "starburst_service", + "{USER}" + ], + "delegateAdmin":true, + "accesses":[ + { + "type":"execute", + "isAllowed":true + } + ] + } + ], + "denyPolicyItems":[ + + ], + "allowExceptions":[ + + ], + "denyExceptions":[ + + ], + "dataMaskPolicyItems":[ + + ], + "rowFilterPolicyItems":[ + + ], + "serviceType":"203", + "options":{ + + }, + "validitySchedules":[ + + ], + "policyLabels":[ + + ], + "zoneName":"", + "isDenyAllElse":false, + "sameLevel20catalog":null, + "sameLevel30schema":null, + "column":null +}' \ +"http://ranger-admin:6080/service/plugins/policies/4"; + +# all - user +curl -i -v -X PUT -u admin:trinoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' +{ + "id":5, + "isEnabled":true, + "createdBy":"Admin", + "updatedBy":"Admin", + "version":1, + "service":"starburst", + "name":"all - user", + "policyType":0, + "policyPriority":0, + "description":"Policy for all - user", + "isAuditEnabled":true, + "resources":{ + "user":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + } + }, + "policyItems":[ + { + "users":[ + "starburst_service", + "{USER}" + ], + "delegateAdmin":true, + "accesses":[ + { + "type":"impersonate", + "isAllowed":true + } + ] + } + ], + "denyPolicyItems":[ + + ], + "allowExceptions":[ + + ], + "denyExceptions":[ + + ], + "dataMaskPolicyItems":[ + + ], + "rowFilterPolicyItems":[ + + ], + "serviceType":"203", + "options":{ + + }, + "validitySchedules":[ + + ], + "policyLabels":[ + + ], + "zoneName":"", + "isDenyAllElse":false, + "sameLevel20catalog":null, + "sameLevel30schema":null, + "column":null +}' \ +"http://ranger-admin:6080/service/plugins/policies/5"; + +# all - catalog, schema, procedure +curl -i -v -X PUT -u admin:trinoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' +{ + "id":6, + "isEnabled":true, + "createdBy":"Admin", + "updatedBy":"Admin", + "version":1, + "service":"starburst", + "name":"all - catalog, schema, procedure", + "policyType":0, + "policyPriority":0, + "description":"Policy for all - catalog, schema, procedure", + "isAuditEnabled":true, + "resources":{ + "catalog":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + }, + "schema":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + }, + "procedure":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + } + }, + "policyItems":[ + { + "users":[ + "starburst_service", + "{USER}" + ], + "delegateAdmin":true, + "accesses":[ + { + "type":"execute", + "isAllowed":true + } + ] + } + ], + "denyPolicyItems":[ + + ], + "allowExceptions":[ + + ], + "denyExceptions":[ + + ], + "dataMaskPolicyItems":[ + + ], + "rowFilterPolicyItems":[ + + ], + "serviceType":"203", + "options":{ + + }, + "validitySchedules":[ + + ], + "policyLabels":[ + + ], + "zoneName":"", + "isDenyAllElse":false, + "column":null +}' \ +"http://ranger-admin:6080/service/plugins/policies/6"; + +# Bob policy +curl -i -v -X POST -u admin:trinoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' +{ + "policyType":"0", + "name":"Bob", + "isEnabled":true, + "policyPriority":0, + "policyLabels":[ + + ], + "description":"", + "isAuditEnabled":true, + "resources":{ + "catalog":{ + "values":[ + "tpch" + ], + "isRecursive":false, + "isExcludes":false + }, + "schema":{ + "values":[ + "sf100" + ], + "isRecursive":false, + "isExcludes":false + }, + "table":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + }, + "column":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + } + }, + "isDenyAllElse":false, + "policyItems":[ + { + "users":[ + "bob", + "starburst_service" + ], + "accesses":[ + { + "type":"select", + "isAllowed":true + }, + { + "type":"insert", + "isAllowed":true + }, + { + "type":"delete", + "isAllowed":true + }, + { + "type":"update", + "isAllowed":true + }, + { + "type":"ownership", + "isAllowed":true + }, + { + "type":"create", + "isAllowed":true + }, + { + "type":"drop", + "isAllowed":true + }, + { + "type":"alter", + "isAllowed":true + } + ] + } + ], + "allowExceptions":[ + + ], + "denyPolicyItems":[ + + ], + "denyExceptions":[ + + ], + "service":"starburst" +}' \ +"http://ranger-admin:6080/service/plugins/policies"; + +# Alice policy +curl -i -v -X POST -u admin:trinoRocks15 --header 'Content-Type: application/json' --header 'Accept: application/json' -d ' +{ + "policyType":"0", + "name":"Alice", + "isEnabled":true, + "policyPriority":0, + "policyLabels":[ + + ], + "description":"", + "isAuditEnabled":true, + "resources":{ + "catalog":{ + "values":[ + "tpch" + ], + "isRecursive":false, + "isExcludes":false + }, + "schema":{ + "values":[ + "sf10" + ], + "isRecursive":false, + "isExcludes":false + }, + "table":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + }, + "column":{ + "values":[ + "*" + ], + "isRecursive":false, + "isExcludes":false + } + }, + "isDenyAllElse":false, + "policyItems":[ + { + "users":[ + "alice", + "starburst_service" + ], + "accesses":[ + { + "type":"select", + "isAllowed":true + }, + { + "type":"insert", + "isAllowed":true + }, + { + "type":"delete", + "isAllowed":true + }, + { + "type":"update", + "isAllowed":true + }, + { + "type":"ownership", + "isAllowed":true + }, + { + "type":"create", + "isAllowed":true + }, + { + "type":"drop", + "isAllowed":true + }, + { + "type":"alter", + "isAllowed":true + } + ] + } + ], + "allowExceptions":[ + + ], + "denyPolicyItems":[ + + ], + "denyExceptions":[ + + ], + "service":"starburst" +}' \ +"http://ranger-admin:6080/service/plugins/policies"; diff --git a/lib/modules/security/system-ranger/resources/trino/ranger/bootstrap-ranger.sh b/lib/modules/security/system-ranger/resources/trino/ranger/bootstrap-ranger.sh new file mode 100755 index 00000000..fef57aac --- /dev/null +++ b/lib/modules/security/system-ranger/resources/trino/ranger/bootstrap-ranger.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +echo "Downloading Ranger CLI..." +STARBURST_RANGER_CLI_PATH=/usr/local/bin/starburst-ranger-cli +DIST=${STARBURST_VER:0:3} +CLI_URL=https://s3.us-east-2.amazonaws.com/software.starburstdata.net/"${DIST}"e/"${STARBURST_VER}"/starburst-ranger-cli-"${STARBURST_VER}"-executable.jar + +curl -fsSL "${CLI_URL}" > "${STARBURST_RANGER_CLI_PATH}" +chmod -v +x "${STARBURST_RANGER_CLI_PATH}" +ln -vs "${STARBURST_RANGER_CLI_PATH}" starburst-ranger-cli + +echo "Waiting for Ranger Admin to come up..." +/opt/minitrino/wait-for-it.sh ranger-admin:6080 --strict --timeout=150 -- echo "Ranger Admin service is up." + +echo "Initializing Ranger Admin with service, users, and policies..." +chmod -R +x /tmp/ranger + +/tmp/ranger/create-service.sh +/tmp/ranger/create-users.sh +/tmp/ranger/add-user-policies.sh diff --git a/lib/modules/security/system-ranger/resources/trino/ranger/create-service.sh b/lib/modules/security/system-ranger/resources/trino/ranger/create-service.sh new file mode 100755 index 00000000..4f947617 --- /dev/null +++ b/lib/modules/security/system-ranger/resources/trino/ranger/create-service.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -ex + +function create_sep_service() { + echo "Creating Starburst Ranger service..." + curl -i -v POST -u admin:trinoRocks15 --header "Content-Type: application/json" --header "Accept: application/json" -d ' + { + "name":"starburst", + "description":"Starburst Ranger service", + "isEnabled":true, + "tagService":"", + "configs":{ + "username":"starburst_service", + "password":"", + "jdbc.driverClassName":"io.prestosql.jdbc.PrestoDriver", + "jdbc.url":"jdbc:presto://trino:8080", + "resource-lookup":"true", + "commonNameForCertificate":"" + }, + "type":"starburst-enterprise" + } + ' \ + "http://ranger-admin:6080/service/plugins/services"; +} + +COUNTER=0 && set +e +while [[ "${COUNTER}" -le 61 ]]; do + if create_sep_service | grep -q "HTTP/1.1 200 OK"; then + break + elif [[ "${COUNTER}" == 61 ]]; then + echo "Timed out waiting for Ranger Admin" + exit 1 + else + sleep 1 + ((COUNTER++)) + fi +done +set -e diff --git a/lib/modules/security/system-ranger/resources/trino/ranger/create-users.sh b/lib/modules/security/system-ranger/resources/trino/ranger/create-users.sh new file mode 100755 index 00000000..9a739f42 --- /dev/null +++ b/lib/modules/security/system-ranger/resources/trino/ranger/create-users.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -ex + +starburst-ranger-cli user create \ + --from-file=/tmp/ranger/usr/alice.json \ + --properties=/etc/starburst/access-control.properties + +starburst-ranger-cli user create \ + --from-file=/tmp/ranger/usr/bob.json \ + --properties=/etc/starburst/access-control.properties diff --git a/lib/modules/security/system-ranger/resources/trino/ranger/usr/alice.json b/lib/modules/security/system-ranger/resources/trino/ranger/usr/alice.json new file mode 100755 index 00000000..dfc301ea --- /dev/null +++ b/lib/modules/security/system-ranger/resources/trino/ranger/usr/alice.json @@ -0,0 +1,14 @@ +{ + "name": "alice", + "firstName": "Alice", + "lastName": "", + "emailAddress": "alice@starburstdata.com", + "password": "trinoRocks15", + "description": "", + "groupIdList": ["1"], + "groupNameList": [], + "status": 1, + "isVisible": 1, + "userSource": 0, + "userRoleList": ["ROLE_USER"] +} diff --git a/lib/modules/security/system-ranger/resources/trino/ranger/usr/bob.json b/lib/modules/security/system-ranger/resources/trino/ranger/usr/bob.json new file mode 100755 index 00000000..a059945a --- /dev/null +++ b/lib/modules/security/system-ranger/resources/trino/ranger/usr/bob.json @@ -0,0 +1,14 @@ +{ + "name": "bob", + "firstName": "Bob", + "lastName": "", + "emailAddress": "bob@starburstdata.com", + "password": "trinoRocks15", + "description": "", + "groupIdList": ["1"], + "groupNameList": [], + "status": 1, + "isVisible": 1, + "userSource": 0, + "userRoleList": ["ROLE_USER"] +} diff --git a/lib/modules/security/system-ranger/system-ranger.yml b/lib/modules/security/system-ranger/system-ranger.yml index bb66816c..9c84e203 100644 --- a/lib/modules/security/system-ranger/system-ranger.yml +++ b/lib/modules/security/system-ranger/system-ranger.yml @@ -1,24 +1,24 @@ -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/security/system-ranger/resources/presto/access-control.properties:/usr/lib/presto/etc/access-control.properties" + - "./modules/security/system-ranger/resources/trino/ranger/:/tmp/ranger/" + - "./modules/security/system-ranger/resources/trino/access-control.properties:/etc/starburst/access-control.properties" + environment: + MINITRINO_BOOTSTRAP: "bootstrap-ranger.sh" ranger-admin: image: harbor.starburstdata.net/starburstdata/starburst-ranger-admin:${SEP_RANGER_VER} container_name: ranger-admin environment: WAIT_FOR: ranger-postgres:5432 - MINIPRESTO_BOOTSTRAP: "bootstrap-ranger.sh" env_file: - "./modules/security/system-ranger/resources/ranger-admin/ranger-admin.env" - volumes: - - "./modules/resources/wait-for-it.sh:/opt/minipresto/wait-for-it.sh" depends_on: - ranger-postgres labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.system-ranger=security-system-ranger" ports: - 6080:6080 @@ -31,11 +31,11 @@ services: volumes: - ranger-system-postgres-data:/var/lib/postgresql/data labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.system-ranger=security-system-ranger" volumes: ranger-system-postgres-data: labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.system-ranger=security-system-ranger" diff --git a/lib/version b/lib/version index 6d7de6e6..227cea21 100644 --- a/lib/version +++ b/lib/version @@ -1 +1 @@ -1.0.2 +2.0.0 diff --git a/readme.md b/readme.md index 5441a712..26f4b278 100644 --- a/readme.md +++ b/readme.md @@ -1,16 +1,14 @@ -# Minipresto -A command line tool that makes it easy to run modular Presto environments -locally. Compatible with Starburst releases up to 350-e. After 350-e, the -refactored version of this project, Minitrino, will need to be used. +# Minitrino +A command line tool that makes it easy to run modular Trino environments +locally. Compatible with Staburst version 354-e and later. -[![PyPI version](https://badge.fury.io/py/minipresto.svg)](https://badge.fury.io/py/minipresto) -[![Build Status](https://travis-ci.org/jefflester/minitrino.svg?branch=master)](https://travis-ci.org/jefflester/minipresto) -[![Presto Slack](https://img.shields.io/static/v1?logo=slack&logoColor=959DA5&label=Slack&labelColor=333a41&message=join%20conversation&color=3AC358)](https://prestosql.io/slack.html) -[![Presto: The Definitive Guide book download](https://img.shields.io/badge/Presto%3A%20The%20Definitive%20Guide-download-brightgreen)](https://www.starburstdata.com/oreilly-presto-guide-download/) +[![PyPI version](https://badge.fury.io/py/minitrino.svg)](https://badge.fury.io/py/minitrino) +[![Build Status](https://travis-ci.org/jefflester/minitrino.svg?branch=master)](https://travis-ci.org/jefflester/minitrino) +[![Trino Slack](https://img.shields.io/static/v1?logo=slack&logoColor=959DA5&label=Slack&labelColor=333a41&message=join%20conversation&color=3AC358)](https://trinodb.io/slack.html) ----- -**Latest Stable Release**: 1.0.2 +**Latest Stable Release**: 2.0.0 ----- @@ -27,8 +25,8 @@ refactored version of this project, Minitrino, will need to be used. - [Install the Library](#install-the-library) - [Display Module Metadata](#display-module-metadata) - [Display CLI Version](#display-cli-version) - - [Pointing the CLI to the Minipresto Library](#pointing-the-cli-to-the-minipresto-library) -- [Minipresto Configuration File](#minipresto-configuration-file) + - [Pointing the CLI to the Minitrino Library](#pointing-the-cli-to-the-minitrino-library) +- [Minitrino Configuration File](#minitrino-configuration-file) - [Project Structure](#project-structure) - [Adding New Modules (Tutorial)](#adding-new-modules-tutorial) - [Troubleshooting](#troubleshooting) @@ -48,21 +46,21 @@ refactored version of this project, Minitrino, will need to be used. ## Installation ### End Users -Minipresto is available on PyPI and the library is available for public download -on GitHub. To install the Minipresto CLI, run `pip install minipresto`. To -install the library, run `minipresto lib_install`. +Minitrino is available on PyPI and the library is available for public download +on GitHub. To install the Minitrino CLI, run `pip install minitrino`. To +install the library, run `minitrino lib_install`. ### Developers -In the project's root, run `./install.sh` to install the Minipresto CLI. If you +In the project's root, run `./install.sh` to install the Minitrino CLI. If you encounter errors during installation, try running `sudo -H ./install.sh -v`. ----- ## CLI -Minipresto is built with [Click](https://click.palletsprojects.com/en/7.x/), a +Minitrino is built with [Click](https://click.palletsprojects.com/en/7.x/), a popular, open-source toolkit used to build Python-based CLIs. -All Minipresto commands/options are documented below. Note that many command +All Minitrino commands/options are documented below. Note that many command options can be specified with a shorthand alternative, which is the first letter of each option, i.e. `--module` can be `-m`. @@ -71,15 +69,15 @@ You can get help, enable verbose output, and change the runtime library directory for any command. ``` -Usage: minipresto [OPTIONS] COMMAND [ARGS]... +Usage: minitrino [OPTIONS] COMMAND [ARGS]... Options: -v, --verbose Enable verbose output. -e, --env TEXT Add or override environment variables. - Environment variables are sourced from the Minipresto - library's root 'minipresto.env' file as well as the user - config file in '~/.minipresto/minipresto.cfg'. Variables + Environment variables are sourced from the Minitrino + library's root 'minitrino.env' file as well as the user + config file in '~/.minitrino/minitrino.cfg'. Variables supplied by this option will override values from either of those sources. The variables will also be passed to the environment of the shell executing commands during the @@ -92,7 +90,7 @@ Options: You can provision an environment via the `provision` command. ``` -Usage: minipresto provision [OPTIONS] +Usage: minitrino provision [OPTIONS] Provision an environment based on specified modules. All options are optional and can be left empty. @@ -107,10 +105,10 @@ Options: `docker-compose up --help` to see all available options. - Example: minipresto provision --docker-native + Example: minitrino provision --docker-native --build - Example: minipresto provision --docker-native '-- + Example: minitrino provision --docker-native '-- remove-orphans --force-recreate' --help Show this message and exit. @@ -118,7 +116,7 @@ Options: Notes: -- If no options are passed in, the CLI will provision a standalone Presto +- If no options are passed in, the CLI will provision a standalone Trino container. - The command cannot currently be used to append additional modules to an active environment. To modify an environment, first shut it down, then re-provision @@ -127,15 +125,15 @@ Notes: Sample `provision` commands: ```bash -minipresto provision \ +minitrino provision \ --module hive-s3 \ --module elasticsearch \ --module ldap \ --docker-native '--build --force-recreate' -minipresto provision -m hive-s3 -m elasticsearch -m ldap +minitrino provision -m hive-s3 -m elasticsearch -m ldap -minipresto --env STARBURST_VER=332-e.6 provision +minitrino --env STARBURST_VER=332-e.6 provision ``` The `provision` command constructs a Docker Compose command and executes it in @@ -150,14 +148,14 @@ docker-compose -f docker-compose.yml \ up -d ``` -Using the structure of the Minipresto library, it is able to merge multiple +Using the structure of the Minitrino library, it is able to merge multiple Docker Compose files together. #### Environment Variables Environment variables passed to Docker containers are sourced through two -locations. The first is from the `minipresto.env` file in the library root. These +locations. The first is from the `minitrino.env` file in the library root. These variables define the versions of the provisioned Docker services. The second is -from from variables set in the `[MODULES]` section of the `minipresto.cfg` file. +from from variables set in the `[MODULES]` section of the `minitrino.cfg` file. These variables can contain sensitive information like access credentials, so their values are intentionally left out of library files. @@ -166,7 +164,7 @@ option, and any unset variable can be set with it. #### Using Licensed Starburst Features If you are using licensed features, you will need to provide a path to a valid -Starburst license. This can be set via `minipresto config` or provided via the +Starburst license. This can be set via `minitrino config` or provided via the `--env` option at command runtime. The variable for this is `STARBURST_LIC_PATH`. @@ -177,24 +175,24 @@ Additionally, you need to uncomment the volume mount in the library's root # Uncomment this to enable the volume mount. The variable should point to a # valid SEP license. volumes: - - "${STARBURST_LIC_PATH}:/usr/lib/presto/etc/starburstdata.license:ro" + - "${STARBURST_LIC_PATH}:/etc/starburst/starburstdata.license:ro" ``` ### Removing Resources You can remove resources with the `remove` command. ``` -Usage: minipresto remove [OPTIONS] +Usage: minitrino remove [OPTIONS] - Remove Minipresto resources. + Remove Minitrino resources. Options: - -i, --images Remove Minipresto images. - -v, --volumes Remove Minipresto container volumes. + -i, --images Remove Minitrino images. + -v, --volumes Remove Minitrino container volumes. -l, --label TEXT Target specific labels for removal (format: key-value pair(s)). - -f, --force Force the removal of Minipresto resources. Normal Docker + -f, --force Force the removal of Minitrino resources. Normal Docker removal restrictions apply. --help Show this message and exit. @@ -203,15 +201,17 @@ Options: Notes: - Named volumes tied to any *existing* container cannot be forcibly removed, - neither by Minipresto nor by the Docker CLI/SDK. + neither by Minitrino nor by the Docker CLI/SDK. - Images tied to stopped containers can be forcibly removed, but any image tied - to a running container cannot be forcibly removed, neither by Minipresto nor + to a running container cannot be forcibly removed, neither by Minitrino nor by the Docker CLI. +- You can find a module's label key by looking at the module's + `docker-compose.yml` file in the Minitrino library. Sample `remove` command: ```bash -minipresto -v remove \ +minitrino -v remove \ --volumes \ --label com.starburst.tests.module.postgres=catalog-postgres \ --force @@ -223,9 +223,9 @@ This will only remove volumes associated to the Postgres catalog module. You can shut down an active environment with the `down` command. ``` -Usage: minipresto down [OPTIONS] +Usage: minitrino down [OPTIONS] - Bring down running Minipresto containers. This command follows the + Bring down running Minitrino containers. This command follows the behavior of `docker-compose down` where containers are both stopped and removed. @@ -233,14 +233,14 @@ Options: -k, --keep Does not remove containers; instead, containers will only be stopped. - --sig-kill Stop Minipresto containers without a grace period. + --sig-kill Stop Minitrino containers without a grace period. --help Show this message and exit. ``` Sample `down` command: ```bash -minipresto -v down +minitrino -v down ``` ### Taking Environment Snapshots @@ -248,10 +248,10 @@ You can capture snapshots for both active and inactive environments with the `snapshot` command. ``` -Usage: minipresto snapshot [OPTIONS] +Usage: minitrino snapshot [OPTIONS] - Create a snapshot of a Minipresto environment. A tarball is placed in the - Minipresto `lib/snapshots/` directory. + Create a snapshot of a Minitrino environment. A tarball is placed in the + Minitrino `lib/snapshots/` directory. To take a snapshot of an active environment, leave the `--module` and option out of the command. @@ -266,7 +266,7 @@ Options: underscores. [required] -d, --directory PATH Directory to save the resulting snapshot file in. - Defaults to the snapshots directory in the Minipresto + Defaults to the snapshots directory in the Minitrino library. -f, --force Overwrite the file if it already exists. @@ -282,7 +282,7 @@ Options: Notes: -- Minipresto records the original `provision` command and places it in the +- Minitrino records the original `provision` command and places it in the snapshot file as `provision-snapshot.sh`; this can be directly executed. This makes it easier for others to reuse the environment and provision it identically. @@ -292,22 +292,22 @@ Sample `snapshot` commands: ```bash # Take a snapshot of an active environment (this will create a tarball # called `snapshot-t2533.tar.gz` in the library's `snapshots/` directory): -minipresto snapshot --name t-2533 +minitrino snapshot --name t-2533 # Take a snapshot of specific modules: -minipresto snapshot -n super-cool-env -m hive-s3 -m elasticsearch -m ldap +minitrino snapshot -n super-cool-env -m hive-s3 -m elasticsearch -m ldap ``` ### Manage User Configuration -You can manage Minipresto configuration with the `config` command. +You can manage Minitrino configuration with the `config` command. ``` -Usage: minipresto config [OPTIONS] +Usage: minitrino config [OPTIONS] - Edit the Minipresto user configuration file. + Edit the Minitrino user configuration file. Options: - -r, --reset Reset the Minipresto user configuration file and create a new + -r, --reset Reset the Minitrino user configuration file and create a new config file from a template. WARNING: This will remove your configuration file (if it @@ -317,12 +317,14 @@ Options: ``` ### Install the Library -You can install the Minipresto library with the `lib_install` command. +You can install the Minitrino library with the `lib_install` command. Note that +it is best practice to have the library version match the CLI version. You can +check these versions with `minitrino version`. ``` -Usage: minipresto lib_install [OPTIONS] +Usage: minitrino lib_install [OPTIONS] - Install the Minipresto library. + Install the Minitrino library. Options: -v, --version TEXT The version of the library to install. @@ -330,10 +332,10 @@ Options: ``` ### Display Module Metadata -You can see Minipresto module metadata with the `modules` command. +You can see Minitrino module metadata with the `modules` command. ``` -Usage: minipresto modules [OPTIONS] +Usage: minitrino modules [OPTIONS] Display module metadata. @@ -346,68 +348,69 @@ Options: --help Show this message and exit. ``` -### Display CLI Version -You can display the Minipresto CLI version with the `version` command. +### Display Minitrino Versions +You can display the Minitrino CLI and library versions with the `version` +command. ``` -Usage: minipresto version [OPTIONS] +Usage: minitrino version [OPTIONS] - Display the Minipresto version. + Display Minitrino CLI and library versions. Options: --help Show this message and exit. ``` -### Pointing the CLI to the Minipresto Library -The Minipresto CLI should always point to a compatible library with the expected +### Pointing the CLI to the Minitrino Library +The Minitrino CLI should always point to a compatible library with the expected structure. The library directory can be set one of four ways, listed below in the order of precedence: 1. Passing the `LIB_PATH` variable to the CLI's `--env` option sets the library directory for the current command. -2. The `minipresto.cfg` file's `LIB_PATH` variable sets the library directory if +2. The `minitrino.cfg` file's `LIB_PATH` variable sets the library directory if present. -3. The path `~/.minipresto/lib/` is used as the default lib path if the +3. The path `~/.minitrino/lib/` is used as the default lib path if the `LIB_PATH` var is not found. -4. As a last resort, Minipresto will check to see if the library exists in +4. As a last resort, Minitrino will check to see if the library exists in relation to the positioning of the `components.py` file and assumes the project is being run out of a cloned repository. If you not running out of a cloned repository, it is advisable to provide a -pointer to the library in Minipresto's configuration via the `LIB_PATH` config. +pointer to the library in Minitrino's configuration via the `LIB_PATH` config. ----- -## Minipresto Configuration File -Sticky configuration is set in `~/.minipresto/minipresto.cfg`. The sections in +## Minitrino Configuration File +Sticky configuration is set in `~/.minitrino/minitrino.cfg`. The sections in this file each serve a separate purpose. ### [CLI] Section -These configs allow the user to customize the behavior of Minipresto. +These configs allow the user to customize the behavior of Minitrino. -- LIB_PATH: The filesystem path of the Minipresto library (specifically to the +- LIB_PATH: The filesystem path of the Minitrino library (specifically to the `lib/` directory). - TEXT_EDITOR: The text editor to use with the `config` command, e.g. "vi", "nano", etc. Defaults to the shell's default editor. ### [DOCKER] Section -These configs allow the user to customize how Minipresto uses Docker. +These configs allow the user to customize how Minitrino uses Docker. - DOCKER_HOST: A URL pointing to an accessible Docker host. This is automatically detected by Docker otherwise. -### [PRESTO] Section -These configs allow the user to propagate config to the Presto container. Since -many modules can append to Presto's core files, the supported way to make -propagate changes to these Presto files is with these configs. +### [TRINO] Section +These configs allow the user to propagate config to the Trino container. Since +many modules can append to Trino's core files, the supported way to make +propagate changes to these Trino files is with these configs. -- CONFIG: Configuration for Presto's `config.properties` file. -- JVM_CONFIG: Configuration for Presto's `jvm.config` file. +- CONFIG: Configuration for Trino's `config.properties` file. +- JVM_CONFIG: Configuration for Trino's `jvm.config` file. A multiline example of this section (note the indentation): ``` -[PRESTO] +[TRINO] CONFIG= query.max-memory-per-node=500MB query.max-total-memory-per-node=500MB @@ -417,12 +420,12 @@ JVM_CONFIG= ### [MODULES] Section This section sets environment variables passed to containers provisioned by -Minipresto. Environment variables are only passed to a container if the variable +Minitrino. Environment variables are only passed to a container if the variable is specified in the module's `docker-compose.yml` file. -Variables propagated to the Presto container are supported by Presto secrets. +Variables propagated to the Trino container are supported by Trino secrets. -- STARBURST_LIC_PATH: Required if using licensed Starburst Enterprise Presto +- STARBURST_LIC_PATH: Required if using licensed Starburst Enterprise Trino features. It can point to any valid license on your filesystem. - S3_ENDPOINT - S3_ACCESS_KEY @@ -447,9 +450,9 @@ Variables propagated to the Presto container are supported by Presto secrets. The library is built around Docker Compose files and utilizes Docker's ability to [extend Compose files](https://docs.docker.com/compose/extends/#multiple-compose-files). -The Starburst Presto service is defined in a Compose file at the library root, +The Starburst Trino service is defined in a Compose file at the library root, and all other services look up in the directory tree to reference the parent -Presto service. In Compose files, the fully-qualified path––relative to the +Trino service. In Compose files, the fully-qualified path––relative to the library's root `docker-compose.yml` file––must be provided for Docker to locate resources. @@ -459,7 +462,7 @@ A simplified library structure: lib ├── Dockerfile ├── docker-compose.yml -├── minipresto.env +├── minitrino.env ├── modules │ ├── catalog │ │ └── postgres @@ -469,7 +472,7 @@ lib │ │ └── resources │ │ ├── postgres │ │ │ └── postgres.env -│ │ └── presto +│ │ └── trino │ │ └── postgres.properties │ ├── resources │ │ └── wait-for-it.sh @@ -481,7 +484,7 @@ lib │ └── resources │ ├── event-logger │ │ └── postgres.env -│ └── presto +│ └── trino │ ├── event-listener.properties │ └── postgres_event_logger.properties ├── snapshots @@ -491,18 +494,18 @@ lib And the contents of a `docker-compose.yml` file (`postgres.yml`): ```yaml -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/catalog/postgres/resources/presto/postgres.properties:/usr/lib/presto/etc/catalog/postgres.properties" + - "./modules/catalog/postgres/resources/trino/postgres.properties:/etc/starburst/catalog/postgres.properties" postgres: image: "postgres:${POSTGRES_VER}" container_name: "postgres" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.postgres=catalog-postgres" env_file: - "./modules/catalog/postgres/resources/postgres/postgres.env" @@ -512,20 +515,20 @@ Notice that the volume mount is not relative to the `lib/modules/catalog/postgres/` directory––it is relative to the parent directory which houses the top-level `docker-compose.yml` file. Also, notice the labels––these labels will be used to identify Docker resources tied to -Minipresto modules so that the CLI commands actually work. +Minitrino modules so that the CLI commands actually work. -### Presto Dockerfile -Minipresto modifies the Starburst Presto Docker image by adding the Presto CLI -to the image as well as by providing `sudo` to the `presto` user. This is +### Trino Dockerfile +Minitrino modifies the Starburst Trino Docker image by adding the Trino CLI +to the image as well as by providing `sudo` to the `trino` user. This is required for certain bootstrap scripts (i.e. using `yum` to install packages in -a Presto container for a module). This image is compatible with Starburst Presto -images back to Starburst Presto version `332-e.0`. +a Trino container for a module). This image is compatible with Starburst Trino +images back to Starburst Trino version `332-e.0`. ----- ## Adding New Modules (Tutorial) Adding new modules is relatively simple, but there are a few important -guidelines to follow to ensure compatibility with the Minipresto CLI. The design +guidelines to follow to ensure compatibility with the Minitrino CLI. The design rules are the same for both catalogs and security modules. The example below demonstrates the process of creating a new catalog module for a Postgres service. @@ -538,30 +541,30 @@ mkdir lib/modules/catalog/postgres/ cd lib/modules/catalog/postgres/ ``` -### Add Presto Resources +### Add Trino Resources All resources for a module go inside of a `resources/` directory within the -module. Inside this directory, place Presto-specific resources into a `presto/` -directory, then mount the resources to the Presto service defined in the root +module. Inside this directory, place Trino-specific resources into a `trino/` +directory, then mount the resources to the Trino service defined in the root `docker-compose.yml` file. ```sh -mkdir -p resources/presto/ +mkdir -p resources/trino/ ``` -In the newly-created `resources/presto/` directory, add a properties file. +In the newly-created `resources/trino/` directory, add a properties file. ```sh bash -c "cat << EOF > postgres.properties connector.name=postgresql -connection-url=jdbc:postgresql://postgres:5432/minipresto +connection-url=jdbc:postgresql://postgres:5432/minitrino connection-user=admin -connection-password=prestoRocks15 +connection-password=trinoRocks15 EOF" ``` ----- -**Note**: Passwords should always be `prestoRocks15` for consistency throughout +**Note**: Passwords should always be `trinoRocks15` for consistency throughout modules. ----- @@ -575,9 +578,9 @@ touch postgres.yml Notice the naming convention: `postgres.yml`. Giving the same root name of "postgres" to both the parent directory `postgres/` and to the Docker Compose -file `postgres.yml` will allow Minipresto to find our new catalog module. +file `postgres.yml` will allow Minitrino to find our new catalog module. -Next, add an environment file for the Postgres service. Non-Presto resources +Next, add an environment file for the Postgres service. Non-Trino resources should go into their own directory, so create one for postgres: ```sh @@ -590,17 +593,17 @@ variables in the Postgres container when it is provisioned: ```sh bash -c "cat << EOF > postgres.env POSTGRES_USER=admin -POSTGRES_PASSWORD=prestoRocks15 -POSTGRES_DB=minipresto +POSTGRES_PASSWORD=trinoRocks15 +POSTGRES_DB=minitrino EOF" ``` -This file will initialize Postgres with a database `minipresto`, a user -`presto`, and a password `prestoRocks15`. +This file will initialize Postgres with a database `minitrino`, a user +`trino`, and a password `trinoRocks15`. ### Add a Metadata File -This step is not required for personal development, but it is required to commit -a module to the Minipresto repository. +The `metadata.json` file allows Minitrino to obtain key information for the +module. It is required for a module to work with the CLI. In `lib/modules/catalog/postgres/`, add the `metadata.json` file: @@ -608,19 +611,19 @@ In `lib/modules/catalog/postgres/`, add the `metadata.json` file: bash -c 'cat << EOF > metadata.json { "description": "Creates a Postgres catalog using the standard Postgres connector.", - "incompatible_modules": [] + "incompatibleModules": [] } EOF' ``` The metadata file is presentable to the user via the `modules` command, and the -`incompatible_modules` key restricts certain modules from being provisioned +`incompatibleModules` key restricts certain modules from being provisioned alongside the given module. The `*` wildcard is a supported convention if the module is incompatible with all other modules. ### Add a Readme File This step is not required for personal development, but it is required to commit -a module to the Minipresto repository. +a module to the Minitrino repository. In `lib/modules/catalog/postgres/`, add the `readme.md` file: @@ -642,7 +645,7 @@ postgres └── resources ├── postgres │ └── postgres.env - └── presto + └── trino └── postgres.properties ``` @@ -651,19 +654,19 @@ We will now define the `postgres.yml` Docker Compose file. Set it up as follows, and **read the important notes after**: ```yaml -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - # Always place Presto files in `/usr/lib/presto/etc/` as symbolic links can change between versions - - "./modules/catalog/postgres/resources/presto/postgres.properties:/usr/lib/presto/etc/catalog/postgres.properties" + # Always place Trino files in `/etc/starburst/` as symbolic links can change between versions + - "./modules/catalog/postgres/resources/trino/postgres.properties:/etc/starburst/catalog/postgres.properties" postgres: image: "postgres:${POSTGRES_VER}" container_name: "postgres" labels: - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.postgres=catalog-postgres" env_file: - "./modules/catalog/postgres/resources/postgres/postgres.env" @@ -682,18 +685,18 @@ Compose files need to be relative to the positioning of the parent Compose file. The base Compose file is determined when you execute a Docker Compose command––the first Compose file referenced in the command becomes the base file, and that happens to be the `docker-compose.yml` file in the library root. This -is how Minipresto constructs these commands. +is how Minitrino constructs these commands. If this is confusing, you can read more about extending Compose files on the [Docker docs](https://docs.docker.com/compose/extends/#multiple-compose-files). -#### Minipresto Docker Labels +#### Minitrino Docker Labels Secondly, notice how we applied sets of labels to the Postgres service. These labels tell the CLI which resources to target when executing commands. -In general, there is no need to apply labels to the Presto service since they +In general, there is no need to apply labels to the Trino service since they are already applied in the parent Compose file **unless** the module is an -extension of the Presto service itself (i.e. the Snowflake modules). Labels +extension of the Trino service itself (i.e. the Snowflake modules). Labels should always be applied to: - Docker services (AKA the resulting container) @@ -702,7 +705,7 @@ should always be applied to: Labels should be defined in pairs of two. The convention is: -- The standard Minipresto resource label: `com.starburst.tests=minipresto` +- The standard Minitrino resource label: `com.starburst.tests=minitrino` - A module-specific resource label: `com.starburst.tests.module.=-` - For this label, the `module-type` should be either `catalog` or `security` @@ -721,18 +724,18 @@ Compose file we just created with a named volume. ----- ```yaml -version: "3.7" +version: "3.8" services: - presto: + trino: volumes: - - "./modules/catalog/postgres/resources/presto/postgres.properties:/usr/lib/presto/etc/catalog/postgres.properties" + - "./modules/catalog/postgres/resources/trino/postgres.properties:/etc/starburst/catalog/postgres.properties" postgres: image: "postgres:${POSTGRES_VER}" container_name: "postgres" labels: # These labels are applied to the service/container - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.postgres=catalog-postgres" env_file: - "./modules/catalog/postgres/resources/postgres/postgres.env" @@ -740,33 +743,33 @@ services: volumes: postgres-data: labels: # These labels are applied to the named volume - - "com.starburst.tests=minipresto" + - "com.starburst.tests=minitrino" - "com.starburst.tests.module.postgres=catalog-postgres" ``` ----- -**Note**: Certain modules will only extend the parent Presto service and do not +**Note**: Certain modules will only extend the parent Trino service and do not actually define any new services/containers. See the Snowflake catalog modules for an example of this. For these modules, the only label requirement is to add -the module-specific label to the Presto service in the relevant +the module-specific label to the Trino service in the relevant `docker-compose.yml` file ----- ### Test the New Catalog -We are all finished up. We can test our new catalog through the Minipresto CLI: +We are all finished up. We can test our new catalog through the Minitrino CLI: ```sh -minipresto provision -m postgres +minitrino provision -m postgres ``` -We can now shell into the `presto` container and run some tests: +We can now shell into the `trino` container and run some tests: ``` -docker exec -it presto bash -presto-cli -presto> show catalogs; +docker exec -it trino bash +trino-cli +trino> show catalogs; ``` ### Customizing Images @@ -777,56 +780,56 @@ and the image build context will follow the same convention as volume mount paths described earlier. ### Bootstrap Scripts -Minipresto supports container bootstrap scripts. These scripts **do not +Minitrino supports container bootstrap scripts. These scripts **do not replace** the entrypoint (or default command) for a given container. The script -is copied from the Minipresto library to the container, executed, and then +is copied from the Minitrino library to the container, executed, and then removed from the container. Containers are restarted after each bootstrap script execution, **so the bootstrap scripts themselves should not restart the container's service**. If a bootstrap script has already executed in a container *and* the volume -associated with the container still exists, Minipresto will not re-execute the +associated with the container still exists, Minitrino will not re-execute the bootstrap script *unless the contents of the script have changed*. The is useful -after running `minipresto down --keep` (which does not remove unnamed container +after running `minitrino down --keep` (which does not remove unnamed container volumes), so that the subsequent `provision` command will not re-execute the same bootstrap script(s). If a bootstrap script is updated, it is recommended to destroy the associated -container(s) via `minipresto down` and then to re-provision. +container(s) via `minitrino down` and then to re-provision. To add a bootstrap script, add a `resources/bootstrap/` directory in any given module, create a shell script, and then reference the script name in the Compose YAML file: ```yaml -version: "3.7" +version: "3.8" services: - presto: + trino: environment: - MINIPRESTO_BOOTSTRAP: "bootstrap.sh" + MINITRINO_BOOTSTRAP: "bootstrap.sh" ``` The `elasticsearch` module is a good example of this. -### Managing Presto's `config.properties` File -Many modules can change the Presto `config.properties` and `jvm.config` files. +### Managing Trino's `config.properties` File +Many modules can change the Trino `config.properties` and `jvm.config` files. Because of this, there are two supported ways to modify these files with -Minipresto. +Minitrino. -The first way is by setting the `CONFIG` variable in your `minipresto.cfg` file. -This will propagate the config to the Presto container when it is provisioned. +The first way is by setting the `CONFIG` variable in your `minitrino.cfg` file. +This will propagate the config to the Trino container when it is provisioned. Generally speaking, this can be used for any type of configuration (i.e. memory configuration) that is unlikely to be modified by any module. This also applies to the `jvm.config` file, which has identical support via the `JVM_CONFIG` -variable. If there are duplicate configs in either file, Minipresto will warn +variable. If there are duplicate configs in either file, Minitrino will warn the user. To set these configs, your configuration file should look like: ``` -[PRESTO] +[TRINO] CONFIG= query.max-memory-per-node=500MB query.max-total-memory-per-node=500MB @@ -834,17 +837,17 @@ JVM_CONFIG= -Dsun.security.krb5.debug=true ``` -The second way to modify core Presto configuration is via module bootstrap +The second way to modify core Trino configuration is via module bootstrap scripts. This method is utilized by modules that need to make module-specific -changes to Presto files. An example bootstrap snippet can be found below: +changes to Trino files. An example bootstrap snippet can be found below: ```bash #!/usr/bin/env bash set -euxo pipefail -echo "Adding Presto configs..." -cat <> /usr/lib/presto/etc/config.properties +echo "Adding Trino configs..." +cat <> /etc/starburst/config.properties query.max-stage-count=105 query.max-execution-time=1h EOT @@ -854,16 +857,25 @@ EOT ## Troubleshooting -- If you experience issues executing a Minipresto command, re-run it with the +- If you experience issues executing a Minitrino command, re-run it with the `-v` option for verbose output. This will often reveal the issue - If you experience an issue with a particular Docker container, consider running these commands: - `docker logs `: Print the logs for a given container to the terminal - `docker ps`: Show all running Docker containers and associated statistics + - `docker inspect ` to see various details about a container - If you experience issues with a library module, check that that module is structured correctly according to the [module - tutorial](#adding-new-modules-tutorial) + tutorial](#adding-new-modules-tutorial), and ensure the library and the CLI + versions match +- Sometimes, a lingering persistent volume can cause problem (i.e. a stale Hive + metastore database volume from a previous module), so you can run: + - `minitrino down` + - `minitrino -v remove --volumes` to remove **all** existing Minitrino + volumes. Alternatively, run `minitrino -v remove --volumes --label ` to specifiy a specific module for which to remove volumes. See the + [removing resources](#removing-resources) section for more information. If none of these troubleshooting tips help to resolve your issue, [please file a GitHub issue](#reporting-bugs-and-contributing) and provide as much information @@ -872,8 +884,8 @@ as possible. ----- ## Reporting Bugs and Contributing -To report bugs, please file a GitHub issue on the [Minipresto -repository](https://github.com/jefflester/minipresto). Bug reports should: +To report bugs, please file a GitHub issue on the [Minitrino +repository](https://github.com/jefflester/minitrino). Bug reports should: - Contain any relevant log messages (if the bug is tied to a command, running with the `-v` flag will make debugging easier) diff --git a/release-notes/2.0.0.md b/release-notes/2.0.0.md new file mode 100644 index 00000000..73cb2031 --- /dev/null +++ b/release-notes/2.0.0.md @@ -0,0 +1,32 @@ +# Minitrino Release Notes: 2.0.0 + +## Release Overview +This release marks the deprecation of Minipresto. The original package is still +available on PyPi and can therefore be installed with `pip install minipresto`. + +Version 2.0.0 requires an updated Docker installation. Check the readme for +details. + +- [CLI Changes and Additions](#cli-changes-and-additions) +- [Library Changes and Additions](#library-changes-and-additions) +- [Other](#other) + +## CLI Changes and Additions +- Refactored from Presto to Trino +- Update Python packages (Docker) +- Fix Minipresto library version file not being captured in snapshot +- Fix text wrapping issue in terminal +- Display library version with `minitrino version` command +- Handle enterprise modules if there is not SEP license provided + +## Library Changes and Additions +- Refactored from Presto to Trino +- Update Docker Compose reference from 3.7 -> 3.8 +- Change `hive-s3` module catalog from `hive_hms` to `hive_s3` +- Change `hive-minio` module catalog from `hive_hms_minio` to `hive_minio` +- Add `delta-lake` module +- Add `file-access-control` module +- Update `system-ranger` module to be compatible with Presto -> Trino refactor + +## Other +- None diff --git a/release-notes/_template.md b/release-notes/_template.md index de8ce918..a77ee28f 100644 --- a/release-notes/_template.md +++ b/release-notes/_template.md @@ -1,4 +1,4 @@ -# Minipresto Release Notes: ${_RELEASE_VER} +# Minitrino Release Notes: ${_RELEASE_VER} ## Release Overview - [CLI Changes and Additions](#cli-changes-and-additions)