diff --git a/docs/configuration.rst b/docs/configuration.rst index 7b522fda..5f35c0c4 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -166,6 +166,19 @@ of volume options: hostpath: /host/foo options: ro,cached +The keys used in volume mappings can contain environment variables **that are +expanded in the host environment**. For example, this configuration would map +the user's ``/home/username/.config/application1`` directory into the container +at the same path. + +.. code-block:: yaml + + volumes: + $TEST_HOME/.config/application1: $TEST_HOME/.config/application1 + +Note that because variable expansion is now applied to all volume keys, if one +desires to have a key with an explicit ``$`` character, it must be written as +``$$``. .. _conf_aliases: diff --git a/scuba/config.py b/scuba/config.py index 45039d66..603a2617 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -261,9 +261,22 @@ def _get_volumes(data): vols = {} for cpath, v in voldata.items(): + cpath = _expand_path(cpath) vols[cpath] = ScubaVolume.from_dict(cpath, v) return vols +def _expand_path(in_str): + try: + output = expand_env_vars(in_str) + except KeyError as ke: + # pylint: disable=raise-missing-from + raise ConfigError("Unset environment variable '{}' used in '{}'".format(ke.args[0], in_str)) + except ValueError as ve: + raise ConfigError("Unable to expand string '{}' due to parsing " + "errors".format(in_str)) from ve + + return output + class ScubaVolume: def __init__(self, container_path, host_path=None, options=None): self.container_path = container_path @@ -282,7 +295,7 @@ def from_dict(cls, cpath, node): if isinstance(node, str): return cls( container_path = cpath, - host_path = node, + host_path = _expand_path(node), ) # Complex form @@ -296,7 +309,7 @@ def from_dict(cls, cpath, node): raise ConfigError("Volume {} must have a 'hostpath' subkey".format(cpath)) return cls( container_path = cpath, - host_path = hpath, + host_path = _expand_path(hpath), options = _get_delimited_str_list(node, 'options', ','), ) diff --git a/scuba/utils.py b/scuba/utils.py index 8a629781..8159aedf 100644 --- a/scuba/utils.py +++ b/scuba/utils.py @@ -1,6 +1,7 @@ import errno import os from shlex import quote as shell_quote +import string def shell_quote_cmd(cmdlist): @@ -79,3 +80,10 @@ def get_umask(): def writeln(f, line): f.write(line + '\n') + +def expand_env_vars(in_str): + """Expand environment variables in a string + + Can raise `KeyError` if a variable is referenced but not defined, similar to + bash's nounset (set -u) option""" + return string.Template(in_str).substitute(os.environ) diff --git a/tests/test_config.py b/tests/test_config.py index 568f041e..c711f308 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -883,3 +883,76 @@ def test_alias_volumes_set(self): assert v.container_path == '/bar' assert v.host_path == '/host/bar' assert v.options == ['z', 'ro'] + + def test_volumes_with_env_vars_simple(self, monkeypatch): + '''volume definitions can contain environment variables''' + monkeypatch.setenv("TEST_VOL_PATH", "/bar/baz") + monkeypatch.setenv("TEST_VOL_PATH2", "/moo/doo") + with open('.scuba.yml', 'w') as f: + f.write(r''' + image: na + volumes: + $TEST_VOL_PATH/foo: ${TEST_VOL_PATH2}/foo + ''') + + config = scuba.config.load_config('.scuba.yml') + vols = config.volumes + assert len(vols) == 1 + + v = list(vols.values())[0] + assert isinstance(v, scuba.config.ScubaVolume) + assert v.container_path == '/bar/baz/foo' + assert v.host_path == '/moo/doo/foo' + assert v.options == [] + + def test_volumes_with_env_vars_complex(self, monkeypatch): + '''complex volume definitions can contain environment variables''' + monkeypatch.setenv("TEST_HOME", "/home/testuser") + monkeypatch.setenv("TEST_TMP", "/tmp") + monkeypatch.setenv("TEST_MAIL", "/var/spool/mail/testuser") + + with open('.scuba.yml', 'w') as f: + f.write(r''' + image: na + volumes: + $TEST_HOME/.config: ${TEST_HOME}/.config + $TEST_TMP/: + hostpath: $TEST_HOME/scuba/myproject/tmp + /var/spool/mail/container: + hostpath: $TEST_MAIL + options: z,ro + ''') + + config = scuba.config.load_config('.scuba.yml') + vols = config.volumes + assert len(vols) == 3 + + v = vols['/home/testuser/.config'] + assert isinstance(v, scuba.config.ScubaVolume) + assert v.container_path == '/home/testuser/.config' + assert v.host_path == '/home/testuser/.config' + assert v.options == [] + + v = vols['/tmp/'] + assert isinstance(v, scuba.config.ScubaVolume) + assert v.container_path == '/tmp/' + assert v.host_path == '/home/testuser/scuba/myproject/tmp' + assert v.options == [] + + v = vols['/var/spool/mail/container'] + assert isinstance(v, scuba.config.ScubaVolume) + assert v.container_path == '/var/spool/mail/container' + assert v.host_path == "/var/spool/mail/testuser" + assert v.options == ['z', 'ro'] + + def test_volumes_with_invalid_env_vars(self, monkeypatch): + '''Volume definitions cannot include unset env vars''' + # Ensure that the entry does not exist in the environment + monkeypatch.delenv("TEST_VAR1", raising=False) + with open('.scuba.yml', 'w') as f: + f.write(r''' + image: na + volumes: + $TEST_VAR1/foo: /host/foo + ''') + self._invalid_config('TEST_VAR1') diff --git a/tests/test_main.py b/tests/test_main.py index 78bb8487..3d9865eb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,7 @@ from .utils import * from unittest import mock import pytest +import warnings import logging import os diff --git a/tests/test_utils.py b/tests/test_utils.py index 243f3a5d..d8075e5b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -128,3 +128,26 @@ def test_writeln(): scuba.utils.writeln(s, 'hello') scuba.utils.writeln(s, 'goodbye') assert s.getvalue() == 'hello\ngoodbye\n' + +def test_expand_env_vars(monkeypatch): + monkeypatch.setenv("MY_VAR", "my favorite variable") + assert scuba.utils.expand_env_vars("This is $MY_VAR") == \ + "This is my favorite variable" + assert scuba.utils.expand_env_vars("What is ${MY_VAR}?") == \ + "What is my favorite variable?" + +def test_expand_missing_env_vars(monkeypatch): + monkeypatch.delenv("MY_VAR", raising=False) + # Verify that a KeyError is raised for unset env variables + with pytest.raises(KeyError) as kerr: + scuba.utils.expand_env_vars("Where is ${MY_VAR}?") + assert kerr.value.args[0] == "MY_VAR" + + +def test_expand_env_vars_dollars(): + # Verify that a ValueError is raised for bare, unescaped '$' characters + with pytest.raises(ValueError): + scuba.utils.expand_env_vars("Just a lonely $") + + # Verify that it is possible to get '$' characters in an expanded string + assert scuba.utils.expand_env_vars(r"Just a lonely $$") == "Just a lonely $"