diff --git a/README.md b/README.md index 1bb519a..0fef9ec 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,323 @@ YaML-based secret manager for secret load/set/transfer. +## Usage + +### Commands + +#### Run + +Run all transfers in the given YaML file. + +```bash +secret-transfer run -f +``` + +Options: + +- `-f, --file ` - Path to the YaML file with secrets. [required] +- `-n, --name ` - Name of the transfer to run. [optional] + +#### Clean + +Clean all secrets in all transfer destinations in the given YaML file. + +```bash +secret-transfer clean -f +``` + +Options: + +- `-f, --file ` - Path to the YaML file with secrets. [required] +- `-n, --name ` - Name of the transfer to clean. [optional] + +### YaML schema + +The YaML file should contain the following optional sections: + +Custom class definitions for sources, destinations, collections, and transfers. + +```yaml +source_classes: + : # Name to be registered + module: # Module to import from + class_name: # Class to be registered +destination_classes: ... +collection_classes: ... +transfer_classes: ... +``` + +Sources, destinations, collections, and transfers. + +```yaml +sources: + : + class_name: # Class to use, either from source_classes or built-in + init_args: # Arguments to pass to the class constructor, can be omitted if no arguments are needed + : + ... +destinations: + ... +collections: + ... +transfers: + ... +``` + +### Lasy loading + +All items are lazily loaded, so they are not created until they are used. This allows to define objects in no particular order and reference them from one another. The only exception is built-in classes, which are registered at the start. + +Cross-referencing is allowed, so you can reference sources, destinations, collections, transfers and their values from init_args. Circular references are not allowed. + +### Cross-referencing + +#### Instance references + +To reference instances of sources, destinations, collections, and transfers, use the following syntax: + +```yaml +$[] +``` + +Where: + +- `` - Type of the instance (source, destination, collection, or transfer). +- `` - Name of the instance. + +#### Value references + +To reference values from sources or collections, use the following syntax: + +```yaml +$[][] +``` + +Where: + +- `` - Type of the gettable instance (usually, but not necessary source or collection). +- `` - Name of the instance. +- `` - Key of the value. + +Where: + +### Built-in classes + +#### Sources + +##### DotEnvSource + +Load secrets from a .env file using `dotenv` package. +Arguments: + +- `file_path` - Path to the .env file. [required] + +Example: + +```yaml +sources: + dotenv: + class_name: DotEnvSource + init_args: + file_path: .env +``` + +##### EnvSource + +Load secrets from environment variables. +Arguments: none. + +Already registered with name `env`, no need to define in the YaML file. + +Example: + +```yaml +collections: + default: + init_args: + TEST_KEY: + source: $sources[env] +``` + +##### PresetSource + +Load secrets from a preset dictionary. +Arguments: key-value pairs of variables. + +Example: + +```yaml +sources: + preset: + class_name: PresetSource + init_args: + TEST_KEY: test_value +``` + +##### UserInputSource + +Asks the user for input. +Arguments: none. + +Already registered with name `user_input`, no need to define in the YaML file. + +Example: + +```yaml +collections: + default: + init_args: + TEST_KEY: + source: $sources[user_input] +``` + +##### VaultCLIKVSource + +Load secrets from HashiCorp Vault KV using `vault kv get` CLI command. +Arguments: + +- `address` - Address of the Vault server. [required] +- `mount` - Mount point of the KV engine. [required] +- `secret_name` - Name(path) of the secret. [required] + +Pre-requisites: + +- `vault` CLI installed and authenticated. + +Example: + +```yaml +sources: + vault: + class_name: VaultCLIKVSource + init_args: + address: https://vault.example.com + mount: secrets + secret_name: TEST_SECRET +``` + +##### YCCLILockboxSource + +Load secrets from Yandex.Cloud Lockbox using `yc lockbox payload get` CLI command. +Arguments: + +- `profile` - Name of the Yandex.Cloud CLI profile. [required] +- `folder` - Folder name. [required] +- `lockbox` - Lockbox name. [required] + +Pre-requisites: + +- `yc` CLI installed and authenticated. + +Example: + +```yaml +sources: + yc_lockbox: + class_name: YCCLILockboxSource + init_args: + profile: my-profile + folder: my-folder + lockbox: my-lockbox +``` + +#### Destinations + +##### BashExportDestination + +Print secrets as `export` commands to the console. Useful for setting environment variables. Never let stdout to be captured by a process, as it will expose the secrets. +Arguments: none. + +Already registered with name `bash_export`, no need to define in the YaML file. + +Example: + +```yaml +transfers: + default: + init_args: + source: ... + destination: $destinations[bash_export] +``` + +##### EnvDestination + +Set secrets as environment variables. +Arguments: none. + +Already registered with name `env`, no need to define in the YaML file. + +Example: + +```yaml +transfers: + default: + init_args: + source: ... + destination: $destinations[env] +``` + +##### GithubCliSecretsDestination + +Set secrets as GitHub repository secrets using `gh secret set` CLI command. +Arguments: + +- `repo_name` - Name of the repository. [required] +- `owner_name` - Name of the repository owner. [required] +- `base_url` - Base URL of the GitHub API. [optional] (default: https://github.com) + +Pre-requisites: + +- `gh` CLI installed and authenticated. + +Example: + +```yaml +destinations: + github: + class_name: GithubCliSecretsDestination + init_args: + repo_name: my-repo + owner_name: my-org +``` + +#### Collections + +##### DefaultCollection + +Default collection to combine secrets from sources. Default collection class, so class_name can be omitted. + +Example: + +```yaml +collections: + default: + init_args: + COLLECTION_KEY: + source: $sources[env] + key: SOURCE_KEY +``` + +#### Transfers + +##### DefaultTransfer + +Default transfer to transfer secrets from collection to destination. Default transfer class, so class_name can be omitted. + +Example: + +```yaml +transfers: + default: + init_args: + collection: $collections[default] + destination: $destinations[env] +``` + +### Usage Examples + +Check [examples](examples/README.md) for usage examples. + ## Development ### Taskfile commands diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..42284ec --- /dev/null +++ b/examples/README.md @@ -0,0 +1,11 @@ +# Examples + +## Basic + +- [Transfer secrets from Lockbox to GitHub](basic/lockbox_to_GitHub/README.md) +- [Use secrets from Lockbox inside Python app](basic/lockbox_to_python_app/README.md) +- [Load secrets from Vault into terminal](basic/vault_to_terminal/README.md) + +## Advanced + +- [Import local files to use custom defined classes](advanced/import_local_files/README.md) diff --git a/examples/advanced/import_local_files/README.md b/examples/advanced/import_local_files/README.md new file mode 100644 index 0000000..711da76 --- /dev/null +++ b/examples/advanced/import_local_files/README.md @@ -0,0 +1,3 @@ +# Import local files to use custom defined classes + +This example demonstrates how to import local files to use custom defined classes. diff --git a/examples/extentions/import_local_files/Taskfile.yaml b/examples/advanced/import_local_files/Taskfile.yaml similarity index 100% rename from examples/extentions/import_local_files/Taskfile.yaml rename to examples/advanced/import_local_files/Taskfile.yaml diff --git a/examples/extentions/import_local_files/custom_file.py b/examples/advanced/import_local_files/custom_file.py similarity index 100% rename from examples/extentions/import_local_files/custom_file.py rename to examples/advanced/import_local_files/custom_file.py diff --git a/examples/extentions/import_local_files/secrets.yaml b/examples/advanced/import_local_files/secrets.yaml similarity index 68% rename from examples/extentions/import_local_files/secrets.yaml rename to examples/advanced/import_local_files/secrets.yaml index d0f4848..d8bf0f1 100644 --- a/examples/extentions/import_local_files/secrets.yaml +++ b/examples/advanced/import_local_files/secrets.yaml @@ -9,21 +9,21 @@ destination_classes: class_name: CustomDestination sources: - default: # use the custom source class + custom: # use the custom source class class_name: CustomSource destinations: - default: # use the custom destination class + custom: # use the custom destination class class_name: CustomDestination collections: default: init_args: test_key: - source: $sources[default] + source: $sources[custom] transfers: default: init_args: - collection: $collections[default] - destination: $destinations[default] + collection: $collections[custom] + destination: $destinations[custom] diff --git a/examples/basic/lockbox_to_github/README.md b/examples/basic/lockbox_to_github/README.md new file mode 100644 index 0000000..fb4cb80 --- /dev/null +++ b/examples/basic/lockbox_to_github/README.md @@ -0,0 +1,3 @@ +# Transfer secrets from lockbox to GitHub + +This example demonstrates how to transfer secrets from Yandex Cloud Lockbox to GitHub. diff --git a/examples/basic/lockbox_to_github/secrets.yaml b/examples/basic/lockbox_to_github/secrets.yaml index c2385c9..01d9c40 100644 --- a/examples/basic/lockbox_to_github/secrets.yaml +++ b/examples/basic/lockbox_to_github/secrets.yaml @@ -2,9 +2,9 @@ sources: lockbox: class_name: YCCLILockboxSource init_args: - profile: secret-transfer # name of the profile in ~/.config/yandex-cloud/config.yaml - folder: default # name of the folder in Yandex.Cloud Lockbox - lockbox: TEST_SECRET # name of the Lockbox secret + profile: secret-transfer + folder: default + lockbox: TEST_SECRET destinations: github: diff --git a/examples/basic/lockbox_to_python_app/README.md b/examples/basic/lockbox_to_python_app/README.md new file mode 100644 index 0000000..1a8cc1e --- /dev/null +++ b/examples/basic/lockbox_to_python_app/README.md @@ -0,0 +1,3 @@ +# Use secrets from Lockbox inside Python app + +This example demonstrates how to use secrets from Lockbox inside a Python app. diff --git a/examples/basic/lockbox_to_python_app/secrets.yaml b/examples/basic/lockbox_to_python_app/secrets.yaml index f00bc06..c6a2f06 100644 --- a/examples/basic/lockbox_to_python_app/secrets.yaml +++ b/examples/basic/lockbox_to_python_app/secrets.yaml @@ -2,9 +2,9 @@ sources: lockbox: class_name: YCCLILockboxSource init_args: - profile: secret-transfer # name of the profile in ~/.config/yandex-cloud/config.yaml - folder: default # name of the folder in Yandex.Cloud Lockbox - lockbox: TEST_SECRET # name of the Lockbox secret + profile: secret-transfer + folder: default + lockbox: TEST_SECRET collections: default: diff --git a/examples/basic/vault_to_terminal/README.md b/examples/basic/vault_to_terminal/README.md new file mode 100644 index 0000000..27c726f --- /dev/null +++ b/examples/basic/vault_to_terminal/README.md @@ -0,0 +1,3 @@ +# Load secrets from Vault into terminal + +This example demonstrates how to load secrets from Vault into terminal. diff --git a/secret_transfer/utils/cli/vault.py b/secret_transfer/utils/cli/vault.py index bf2d4ed..398a157 100644 --- a/secret_transfer/utils/cli/vault.py +++ b/secret_transfer/utils/cli/vault.py @@ -21,7 +21,7 @@ def get(cls, address: str, mount: str, field: str, value: str) -> str: :raises KeyNotFoundError: if the key is not found """ try: - return cls._run(f"get" f" -address={address}" f" -mount={mount}" f" -field={field}" f" {value}") + return cls._run(f"get -address={address} -mount={mount} -field={field} {value}") except base.RunError as exc: if "Code: 403" in exc.stderr: raise cls.MountForbiddenError(f"Access to mount({mount}) is forbidden") from exc diff --git a/tests/e2e/cli/test_clean.py b/tests/e2e/cli/test_clean.py index de7c223..6c512e1 100644 --- a/tests/e2e/cli/test_clean.py +++ b/tests/e2e/cli/test_clean.py @@ -126,15 +126,15 @@ def test_with_name(settings_file_context: common.SettingsFileContext): " init_args:\n" " test_key: test_value\n" " test_key2: test_value2\n" + " preset2:\n" + " class_name: PresetSource\n" + " init_args:\n" + " test_key: $sources[preset][test_key]\n" "collections:\n" " test_collection:\n" " init_args:\n" " test_key:\n" - " source: $sources[preset]\n" - " test_collection2:\n" - " init_args:\n" - " test_key2:\n" - " source: $sources[preset]\n" + " source: $sources[preset2]\n" "transfers:\n" " test_transfer:\n" " init_args:\n" diff --git a/tests/e2e/cli/test_run.py b/tests/e2e/cli/test_run.py index f85a3e1..4355cf9 100644 --- a/tests/e2e/cli/test_run.py +++ b/tests/e2e/cli/test_run.py @@ -126,11 +126,15 @@ def test_with_name(settings_file_context: common.SettingsFileContext): " init_args:\n" " test_key: test_value\n" " test_key2: test_value2\n" + " preset2:\n" + " class_name: PresetSource\n" + " init_args:\n" + " test_key: $sources[preset][test_key]\n" "collections:\n" " test_collection:\n" " init_args:\n" " test_key:\n" - " source: $sources[preset]\n" + " source: $sources[preset2]\n" " test_collection2:\n" " init_args:\n" " test_key2:\n"