From c3d5266e77a730041d7d7ef7a10090109093b62f Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Sun, 9 Sep 2018 14:03:50 -0400 Subject: [PATCH 01/12] test: Move env var tests down to their own section --- tests/test_main.py | 56 +++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 0c192af2..7a98b90f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -356,32 +356,6 @@ def test_arbitrary_docker_args(self): assert_str_equalish(out, data) - def test_env_var_keyval(self): - '''Verify -e KEY=VAL works''' - with open('.scuba.yml', 'w') as f: - f.write('image: {0}\n'.format(DOCKER_IMAGE)) - args = [ - '-e', 'KEY=VAL', - '/bin/sh', '-c', 'echo $KEY', - ] - out, _ = self.run_scuba(args) - assert_str_equalish(out, 'VAL') - - def test_env_var_key_only(self): - '''Verify -e KEY works''' - with open('.scuba.yml', 'w') as f: - f.write('image: {0}\n'.format(DOCKER_IMAGE)) - args = [ - '-e', 'KEY', - '/bin/sh', '-c', 'echo $KEY', - ] - def mocked_getenv(key): - self.assertEqual(key, 'KEY') - return 'mockedvalue' - with mock.patch('os.getenv', side_effect=mocked_getenv): - out, _ = self.run_scuba(args) - assert_str_equalish(out, 'mockedvalue') - def test_image_entrypoint(self): '''Verify scuba doesn't interfere with the configured image ENTRYPOINT''' @@ -493,6 +467,36 @@ def test_root_hook(self): self._test_one_hook('root', 0, 0) + ############################################################################ + # Environment + + def test_env_var_keyval(self): + '''Verify -e KEY=VAL works''' + with open('.scuba.yml', 'w') as f: + f.write('image: {0}\n'.format(DOCKER_IMAGE)) + args = [ + '-e', 'KEY=VAL', + '/bin/sh', '-c', 'echo $KEY', + ] + out, _ = self.run_scuba(args) + assert_str_equalish(out, 'VAL') + + def test_env_var_key_only(self): + '''Verify -e KEY works''' + with open('.scuba.yml', 'w') as f: + f.write('image: {0}\n'.format(DOCKER_IMAGE)) + args = [ + '-e', 'KEY', + '/bin/sh', '-c', 'echo $KEY', + ] + def mocked_getenv(key): + self.assertEqual(key, 'KEY') + return 'mockedvalue' + with mock.patch('os.getenv', side_effect=mocked_getenv): + out, _ = self.run_scuba(args) + assert_str_equalish(out, 'mockedvalue') + + ############################################################################ # Misc def test_list_aliases(self): From 801ae9af2a56dc6619de57cecd6ef5b81842f111 Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Sun, 9 Sep 2018 14:22:36 -0400 Subject: [PATCH 02/12] test: Refactor out mocked_os_env() This simplifies mocking the OS environment --- tests/test_main.py | 5 +---- tests/utils.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 7a98b90f..75fd0957 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -489,10 +489,7 @@ def test_env_var_key_only(self): '-e', 'KEY', '/bin/sh', '-c', 'echo $KEY', ] - def mocked_getenv(key): - self.assertEqual(key, 'KEY') - return 'mockedvalue' - with mock.patch('os.getenv', side_effect=mocked_getenv): + with mocked_os_env(KEY='mockedvalue'): out, _ = self.run_scuba(args) assert_str_equalish(out, 'mockedvalue') diff --git a/tests/utils.py b/tests/utils.py index 1444b3d4..d7ebea26 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -6,6 +6,11 @@ import shutil import unittest import logging +try: + from unittest import mock +except ImportError: + import mock + def assert_set_equal(a, b): assert_equal(set(a), set(b)) @@ -49,6 +54,11 @@ def make_executable(path): mode |= (mode & 0o444) >> 2 # copy R bits to X os.chmod(path, mode) +def mocked_os_env(**env): + def mocked_getenv(key): + return env.get(key) + return mock.patch('os.getenv', side_effect=mocked_getenv) + # http://stackoverflow.com/a/8389373/119527 class PseudoTTY(object): From 21c48fd4b8dfedde21f7bb183bf4bbcca3e571de Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Sat, 8 Sep 2018 14:38:39 -0400 Subject: [PATCH 03/12] scuba: Add environment to config --- scuba/config.py | 34 ++++++++++++++++++++++- tests/test_config.py | 65 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/scuba/config.py b/scuba/config.py index b8e0417f..111a9b35 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -129,6 +129,30 @@ def _process_script_node(node, name): raise ConfigError("{0}: must be string or dict".format(name)) +def _process_environment(node, name): + # Environment can be either a list of strings ("KEY=VALUE") or a mapping + # Environment keys and values are always strings + result = {} + + if not node: + pass + elif isinstance(node, dict): + for k, v in node.items(): + if v is None: + v = os.getenv(k) + result[k] = str(v) + elif isinstance(node, list): + for e in node: + k, v = parse_env_var(e) + result[k] = v + else: + raise ConfigError("'{0}' must be list or mapping, not {1}".format( + name, type(node))) + + return result + + + class ScubaAlias(object): def __init__(self, name, script, image): self.name = name @@ -147,7 +171,7 @@ class ScubaContext(object): class ScubaConfig(object): def __init__(self, **data): required_nodes = ('image',) - optional_nodes = ('aliases','hooks',) + optional_nodes = ('aliases','hooks','environment') # Check for missing required nodes missing = [n for n in required_nodes if not n in data] @@ -165,6 +189,7 @@ def __init__(self, **data): self._load_aliases(data) self._load_hooks(data) + self._environment = self._load_environment(data) @@ -187,6 +212,9 @@ def _load_hooks(self, data): hook = _process_script_node(node, name) self._hooks[name] = hook + def _load_environment(self, data): + return _process_environment(data.get('environment'), 'environment') + @property def image(self): @@ -200,6 +228,10 @@ def aliases(self): def hooks(self): return self._hooks + @property + def environment(self): + return self._environment + def process_command(self, command): '''Processes a user command using aliases diff --git a/tests/test_config.py b/tests/test_config.py index 118379e2..c7bfe81f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -372,3 +372,68 @@ def test_hooks_missing_script(self): ''') assert_raises(scuba.config.ConfigError, scuba.config.load_config, '.scuba.yml') + + + ############################################################################ + # Env + + def test_env_top_dict(self): + '''Top-level environment can be loaded (dict)''' + with open('.scuba.yml', 'w') as f: + f.write(r''' + image: na + environment: + FOO: This is foo + FOO_WITH_QUOTES: "\"Quoted foo\"" # Quotes included in value + BAR: "This is bar" + MAGIC: 42 + SWITCH_1: true # YAML boolean + SWITCH_2: "true" # YAML string + EMPTY: "" + EXTERNAL: # Comes from os env + ''') + + with mocked_os_env(EXTERNAL='Outside world'): + config = scuba.config.load_config('.scuba.yml') + + expect = dict( + FOO = "This is foo", + FOO_WITH_QUOTES = "\"Quoted foo\"", + BAR = "This is bar", + MAGIC = "42", # N.B. string + SWITCH_1 = "True", # Unfortunately this is due to str(bool(1)) + SWITCH_2 = "true", + EMPTY = "", + EXTERNAL = "Outside world", + ) + self.assertEqual(expect, config.environment) + + + def test_env_top_list(self): + '''Top-level environment can be loaded (list)''' + with open('.scuba.yml', 'w') as f: + f.write(r''' + image: na + environment: + - FOO=This is foo # No quotes + - FOO_WITH_QUOTES="Quoted foo" # Quotes included in value + - BAR=This is bar + - MAGIC=42 + - SWITCH_2=true + - EMPTY= + - EXTERNAL # Comes from os env + ''') + + with mocked_os_env(EXTERNAL='Outside world'): + config = scuba.config.load_config('.scuba.yml') + + expect = dict( + FOO = "This is foo", + FOO_WITH_QUOTES = "\"Quoted foo\"", + BAR = "This is bar", + MAGIC = "42", # N.B. string + SWITCH_2 = "true", + EMPTY = "", + EXTERNAL = "Outside world", + ) + self.assertEqual(expect, config.environment) From caf247e0599f8a762fe58ad7bfbe49756cc89f9c Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Sun, 9 Sep 2018 13:24:06 -0400 Subject: [PATCH 04/12] scuba: Allow per-alias environment which overrides the top-level --- scuba/config.py | 20 +++++++++++++++++--- tests/test_config.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/scuba/config.py b/scuba/config.py index 111a9b35..d7b4b735 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -154,16 +154,25 @@ def _process_environment(node, name): class ScubaAlias(object): - def __init__(self, name, script, image): + def __init__(self, name, script, image, environment): self.name = name self.script = script self.image = image + self.environment = environment @classmethod def from_dict(cls, name, node): script = _process_script_node(node, name) - image = node.get('image') if isinstance(node, dict) else None - return cls(name, script, image) + image = None + environment = None + + if isinstance(node, dict): # Rich alias + image = node.get('image') + environment = _process_environment( + node.get('environment'), + '{0}.{1}'.format(name, 'environment')) + + return cls(name, script, image, environment) class ScubaContext(object): pass @@ -246,6 +255,7 @@ def process_command(self, command): result = ScubaContext() result.script = None result.image = self.image + result.environment = self.environment.copy() if command: alias = self.aliases.get(command[0]) @@ -258,6 +268,10 @@ def process_command(self, command): if alias.image: result.image = alias.image + # Merge/override the environment + if alias.environment: + result.environment.update(alias.environment) + if len(alias.script) > 1: # Alias is a multiline script; no additional # arguments are allowed in the scuba invocation. diff --git a/tests/test_config.py b/tests/test_config.py index c7bfe81f..9818a9c4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -437,3 +437,41 @@ def test_env_top_list(self): EXTERNAL = "Outside world", ) self.assertEqual(expect, config.environment) + + + def test_env_alias(self): + '''Alias can have environment which overrides top-level''' + with open('.scuba.yml', 'w') as f: + f.write(r''' + image: na + environment: + FOO: Top-level + BAR: 42 + aliases: + al: + script: Don't care + environment: + FOO: Overridden + MORE: Hello world + ''') + + config = scuba.config.load_config('.scuba.yml') + + self.assertEqual(config.environment, dict( + FOO = "Top-level", + BAR = "42", + )) + + self.assertEqual(config.aliases['al'].environment, dict( + FOO = "Overridden", + MORE = "Hello world", + )) + + # Does the environment get overridden / merged? + ctx = config.process_command(['al']) + + self.assertEqual(ctx.environment, dict( + FOO = "Overridden", + BAR = "42", + MORE = "Hello world", + )) From e432e7141471d5b842001b835d794b3a7343bc31 Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Sun, 9 Sep 2018 14:04:33 -0400 Subject: [PATCH 05/12] scuba: Use env vars from yaml --- scuba/__main__.py | 3 +++ tests/test_main.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/scuba/__main__.py b/scuba/__main__.py index 72c89d6d..c3eb3e6f 100644 --- a/scuba/__main__.py +++ b/scuba/__main__.py @@ -132,6 +132,9 @@ def prepare(self): # Docker is running natively self.__setup_native_run() + # Apply environment vars from .scuba.yml + self.env_vars.update(self.context.environment) + def __str__(self): s = StringIO() writeln(s, 'ScubaDive') diff --git a/tests/test_main.py b/tests/test_main.py index 75fd0957..9e4641ad 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -13,6 +13,7 @@ import sys from tempfile import TemporaryFile, NamedTemporaryFile import subprocess +import shlex import scuba.__main__ as main import scuba.constants @@ -494,6 +495,59 @@ def test_env_var_key_only(self): assert_str_equalish(out, 'mockedvalue') + def test_env_var_sources(self): + '''Verify scuba handles all possible environment variable sources''' + with open('.scuba.yml', 'w') as f: + f.write(r''' + image: {image} + environment: + FOO: Top-level + BAR: 42 + EXTERNAL_2: + aliases: + al: + script: + - echo "FOO=\"$FOO\"" + - echo "BAR=\"$BAR\"" + - echo "MORE=\"$MORE\"" + - echo "EXTERNAL_1=\"$EXTERNAL_1\"" + - echo "EXTERNAL_2=\"$EXTERNAL_2\"" + - echo "EXTERNAL_3=\"$EXTERNAL_3\"" + - echo "BAZ=\"$BAZ\"" + environment: + FOO: Overridden + MORE: Hello world + EXTERNAL_3: + '''.format(image=DOCKER_IMAGE)) + + args = [ + '-e', 'EXTERNAL_1', + '-e', 'BAZ=From the command line', + 'al', + ] + + m = mocked_os_env( + EXTERNAL_1 = "External value 1", + EXTERNAL_2 = "External value 2", + EXTERNAL_3 = "External value 3", + ) + with m: + out, _ = self.run_scuba(args) + + # Convert key/pair output to dictionary + result = dict( pair.split('=', 1) for pair in shlex.split(out) ) + + self.assertEqual(result, dict( + FOO = "Overridden", + BAR = "42", + MORE = "Hello world", + EXTERNAL_1 = "External value 1", + EXTERNAL_2 = "External value 2", + EXTERNAL_3 = "External value 3", + BAZ = "From the command line", + )) + + ############################################################################ # Misc def test_list_aliases(self): From a6a90379c5524928671c76922bd397004d1cfc0d Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Sun, 9 Sep 2018 16:11:53 -0400 Subject: [PATCH 06/12] example: Add example of using environment variables --- example/env_vars/.scuba.yml | 21 +++++++++++++++++++++ example/env_vars/run_example.sh | 7 +++++++ 2 files changed, 28 insertions(+) create mode 100644 example/env_vars/.scuba.yml create mode 100755 example/env_vars/run_example.sh diff --git a/example/env_vars/.scuba.yml b/example/env_vars/.scuba.yml new file mode 100644 index 00000000..b28ca8a2 --- /dev/null +++ b/example/env_vars/.scuba.yml @@ -0,0 +1,21 @@ +image: !from_yaml ../common.yml image + +environment: + FOO: Top-level + BAR: 42 + EMPTY: "" + EXTERNAL_1: + +aliases: + example: + environment: + FOO: Overridden by alias + EXTERNAL_2: + script: + - echo "FOO=\"$FOO\"" + - echo "BAR=\"$BAR\"" + - echo "EMPTY=\"$EMPTY\"" + - echo "EXTERNAL_1=\"$EXTERNAL_1\"" + - echo "EXTERNAL_2=\"$EXTERNAL_2\"" + - echo "CMDLINE=\"$CMDLINE\"" + diff --git a/example/env_vars/run_example.sh b/example/env_vars/run_example.sh new file mode 100755 index 00000000..bc742d11 --- /dev/null +++ b/example/env_vars/run_example.sh @@ -0,0 +1,7 @@ +#!/bin/bash +cd $(dirname $0) + +export EXTERNAL_1="Value 1 taken from external environment" +export EXTERNAL_2="Value 2 taken from external environment" + +scuba -e CMDLINE="This comes from the cmdline" example From 456144d4e5d424fa129ba846ba260153d8e1d82f Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Sun, 9 Sep 2018 16:21:31 -0400 Subject: [PATCH 07/12] doc: Add environment variables to the YAML reference --- doc/yaml-reference.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/doc/yaml-reference.md b/doc/yaml-reference.md index 95826e8c..996f5549 100644 --- a/doc/yaml-reference.md +++ b/doc/yaml-reference.md @@ -18,6 +18,25 @@ Example: image: debian:8.2 ``` +### `environment` + +The optional `environment` node allows environment variables to be specified. +This can be either a mapping (dictionary), or a list of `KEY=VALUE` pairs. +If a value is not specified, the value is taken from the external environment. + +Examples: +```yaml +environment: + FOO: "This is foo" + SECRET: +``` +```yaml +environment: + - FOO=This is foo + - SECRET +``` + + ### `aliases` The optional `aliases` node is a mapping (dictionary) of bash-like aliases, @@ -53,6 +72,21 @@ aliases: - cat /etc/os-release ``` +Aliases can add to the top-level `environment` and override its values using +the same syntax: +```yaml +environment: + FOO: "Top-level" +aliases: + example: + environment: + FOO: "Override" + BAR: "New" + script: + - echo $FOO $BAR +``` + + ### `hooks` The optional `hooks` node is a mapping (dictionary) of "hook" scripts that run From 85d99049a4648b41493e89720d19c8e7c0a682e5 Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Sun, 9 Sep 2018 16:49:31 -0400 Subject: [PATCH 08/12] scuba: Show simple type name in alias environment error messages --- scuba/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scuba/config.py b/scuba/config.py index d7b4b735..6ae7e819 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -147,7 +147,7 @@ def _process_environment(node, name): result[k] = v else: raise ConfigError("'{0}' must be list or mapping, not {1}".format( - name, type(node))) + name, type(node).__name__)) return result From 732ef6e3344a84dc002b4b556c44360de139fa4c Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Mon, 10 Sep 2018 17:22:59 -0400 Subject: [PATCH 09/12] test: Update mocked_os_env() to handle default argument --- tests/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index d7ebea26..56db18cb 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -55,8 +55,8 @@ def make_executable(path): os.chmod(path, mode) def mocked_os_env(**env): - def mocked_getenv(key): - return env.get(key) + def mocked_getenv(key, default=None): + return env.get(key, default) return mock.patch('os.getenv', side_effect=mocked_getenv) From ea9259ba88aebdbf171c6ea881d555f6186efe40 Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Mon, 10 Sep 2018 17:23:30 -0400 Subject: [PATCH 10/12] scuba: Return empty string if external environment variable is not set ...rather than 'None' --- scuba/config.py | 2 +- scuba/utils.py | 2 +- tests/test_config.py | 4 ++++ tests/test_utils.py | 12 +++++++----- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/scuba/config.py b/scuba/config.py index 6ae7e819..9b79ffdc 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -139,7 +139,7 @@ def _process_environment(node, name): elif isinstance(node, dict): for k, v in node.items(): if v is None: - v = os.getenv(k) + v = os.getenv(k, '') result[k] = str(v) elif isinstance(node, list): for e in node: diff --git a/scuba/utils.py b/scuba/utils.py index f1555fcc..1af6f455 100644 --- a/scuba/utils.py +++ b/scuba/utils.py @@ -74,4 +74,4 @@ def parse_env_var(s): return (k, v) k = parts[0] - return (k, os.getenv(k)) + return (k, os.getenv(k, '')) diff --git a/tests/test_config.py b/tests/test_config.py index 9818a9c4..5d8a5b4f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -391,6 +391,7 @@ def test_env_top_dict(self): SWITCH_2: "true" # YAML string EMPTY: "" EXTERNAL: # Comes from os env + EXTERNAL_NOTSET: # Missing in os env ''') with mocked_os_env(EXTERNAL='Outside world'): @@ -405,6 +406,7 @@ def test_env_top_dict(self): SWITCH_2 = "true", EMPTY = "", EXTERNAL = "Outside world", + EXTERNAL_NOTSET = "", ) self.assertEqual(expect, config.environment) @@ -422,6 +424,7 @@ def test_env_top_list(self): - SWITCH_2=true - EMPTY= - EXTERNAL # Comes from os env + - EXTERNAL_NOTSET # Missing in os env ''') with mocked_os_env(EXTERNAL='Outside world'): @@ -435,6 +438,7 @@ def test_env_top_list(self): SWITCH_2 = "true", EMPTY = "", EXTERNAL = "Outside world", + EXTERNAL_NOTSET = "", ) self.assertEqual(expect, config.environment) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0e6c1307..02c3aaf7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -89,10 +89,12 @@ def test_parse_env_var_more_equals(self): def test_parse_env_var_no_equals(self): '''parse_env_var handles no equals and gets value from environment''' - def mocked_getenv(key): - self.assertEqual(key, 'KEY') - return 'mockedvalue' - - with mock.patch('os.getenv', side_effect=mocked_getenv): + with mocked_os_env(KEY='mockedvalue'): result = scuba.utils.parse_env_var('KEY') self.assertEqual(result, ('KEY', 'mockedvalue')) + + def test_parse_env_var_not_set(self): + '''parse_env_var returns an empty string if not set''' + with mocked_os_env(): + result = scuba.utils.parse_env_var('NOTSET') + self.assertEqual(result, ('NOTSET', '')) From ec9094bf16699173a3dafc36ae8155180bca674b Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Mon, 10 Sep 2018 17:30:55 -0400 Subject: [PATCH 11/12] test: Simplify mocked_os_env by using env.get directly as side_effect --- tests/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 56db18cb..7ca20248 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -55,9 +55,7 @@ def make_executable(path): os.chmod(path, mode) def mocked_os_env(**env): - def mocked_getenv(key, default=None): - return env.get(key, default) - return mock.patch('os.getenv', side_effect=mocked_getenv) + return mock.patch('os.getenv', side_effect=env.get) # http://stackoverflow.com/a/8389373/119527 From a2f3332de61f9a4d6868fe203db3599d9641cc30 Mon Sep 17 00:00:00 2001 From: Jonathon Reinhart Date: Mon, 10 Sep 2018 17:48:31 -0400 Subject: [PATCH 12/12] Update change log with .scuba.yml environment [ci skip] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eefb350..3b9ff891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Added - Add -e/--env command-line option (#111) +- Add support for setting environment in .scuba.yml (#120) ### Changed - Implemented auto-versioning using Git and Travis (#112)