diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7992e621..a9af94b7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,8 +8,6 @@ on: paths: - "!**/README.md" pull_request: - paths: - - "!**/README.md" permissions: id-token: write @@ -37,6 +35,9 @@ jobs: - uses: Swatinem/rust-cache@v2.7.5 if: ${{ matrix.lang == 'rust'}} + with: + # The build script creates a `release` build so use separate cache + key: "release" - uses: actions/setup-go@v5 if: ${{ matrix.lang == 'go'}} @@ -218,9 +219,6 @@ jobs: - name: Delete secret with Rust run: bin/rust/vault -d "secret-${{github.sha}}.zip" - - name: Verify that key has been deleted with Rust - run: bin/rust/vault exists secret-${{github.sha}}.zip | grep -q "does not exist" - - name: Verify that keys have been deleted using Rust run: | bin/rust/vault exists secret-python | grep -q "key 'secret-python' does not exist" @@ -229,6 +227,33 @@ jobs: bin/rust/vault exists secret-rust | grep -q "key 'secret-rust' does not exist" bin/rust/vault exists secret-nodejs | grep -q "key 'secret-nodejs' does not exist" + - name: Check Python vault package + run: python -m pip show nitor-vault + + - name: Store secret using Python library + run: | + python -c "from n_vault import Vault; Vault().store('secret-python-library', 'sha-${{github.sha}}')" + + - name: Verify secret using Python library + run: | + python -c "from n_vault import Vault; print('true') if Vault().exists('secret-python-library') else print('false')" | grep -q "true" + + - name: Validate storing worked with Rust + run: diff <(bin/rust/vault -l secret-python-library) <(echo -n sha-${{github.sha}}) + + - name: Lookup with Python library + run: | + diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library').decode('utf-8'), end='', flush=True)") <(echo -n sha-${{github.sha}}) + + - name: List with Python library + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" + + - name: Delete with Python library + run: python -c "from n_vault import Vault; Vault().delete('secret-python-library')" + + - name: Verify that key has been deleted with Rust + run: bin/rust/vault exists secret-python-library | grep -q "key 'secret-python-library' does not exist" + - name: Install Python PyO3 vault run: python -m pip install --force-reinstall . working-directory: python-pyo3 @@ -310,3 +335,36 @@ jobs: - name: Verify that key has been deleted with Python-pyo3 run: vault exists secret-${{github.sha}}.zip | grep -q "does not exist" + + - name: Check Python vault package + run: python -m pip show nitor-vault + + - name: Delete all keys with Python library + run: python -c "from n_vault import Vault; Vault().delete_many(Vault().list_all())" + + - name: List with Python library + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | grep -ve '^\s*$' | wc -l | grep -q "0" + + - name: Store secret using Python library + run: | + python -c "from n_vault import Vault; Vault().store('secret-python-library', 'sha-${{github.sha}}')" + + - name: Verify secret using Python library + run: | + python -c "from n_vault import Vault; print('true') if Vault().exists('secret-python-library') else print('false')" | grep -q "true" + + - name: Validate storing worked with Rust + run: diff <(bin/rust/vault -l secret-python-library) <(echo -n sha-${{github.sha}}) + + - name: Lookup with Python library + run: | + diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end='', flush=True)") <(echo -n sha-${{github.sha}}) + + - name: List with Python library + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1" + + - name: Delete with Python library + run: python -c "from n_vault import Vault; Vault().delete('secret-python-library')" + + - name: Verify that key has been deleted with Rust + run: bin/rust/vault exists secret-python-library | grep -q "key 'secret-python-library' does not exist" diff --git a/.github/workflows/python-wheel.yml b/.github/workflows/python-wheel.yml index 67c557ee..da0dd418 100644 --- a/.github/workflows/python-wheel.yml +++ b/.github/workflows/python-wheel.yml @@ -28,6 +28,11 @@ on: permissions: contents: read +# Cancel previous runs for PRs but not pushes to main +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + jobs: linux: runs-on: ${{ matrix.platform.runner }} diff --git a/.gitignore b/.gitignore index ddb1c942..18b61beb 100644 --- a/.gitignore +++ b/.gitignore @@ -79,7 +79,8 @@ celerybeat-schedule .env # virtualenv -venv/ +venv*/ +.venv*/ ENV/ # Spyder project settings @@ -112,4 +113,3 @@ dependency-reduced-pom.xml # Go binary /go/vault /go/vault -.venv diff --git a/Cargo.lock b/Cargo.lock index 70b429f7..f2977db6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -627,9 +627,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.37" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" +checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8" dependencies = [ "shlex", ] @@ -652,9 +652,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", "clap_derive", @@ -662,9 +662,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" dependencies = [ "anstream", "anstyle", @@ -674,9 +674,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.37" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11611dca53440593f38e6b25ec629de50b14cdfa63adc0fb856115a2c6d97595" +checksum = "d9647a559c112175f17cf724dc72d3645680a883c58481332779192b0d8e7a01" dependencies = [ "clap", ] @@ -695,9 +695,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" [[package]] name = "colorchoice" @@ -739,9 +739,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" dependencies = [ "libc", ] @@ -1478,7 +1478,7 @@ dependencies = [ [[package]] name = "nitor-vault" -version = "2.1.2" +version = "2.2.0" dependencies = [ "aes-gcm", "anyhow", @@ -1501,7 +1501,7 @@ dependencies = [ [[package]] name = "nitor-vault-pyo3" -version = "2.1.2" +version = "2.2.0" dependencies = [ "anyhow", "nitor-vault", @@ -1965,18 +1965,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index a9102d5e..7d1ad6bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "2.1.2" +version = "2.2.0" [profile.release] lto = "thin" diff --git a/python-pyo3/README.md b/python-pyo3/README.md index 00423c8f..378d8d02 100644 --- a/python-pyo3/README.md +++ b/python-pyo3/README.md @@ -4,7 +4,7 @@ Python vault implementation using the Rust vault library. See the [repo](https://github.com/NitorCreations/vault) root readme for more general information. -## Usage +## Vault CLI ```console Encrypted AWS key-value storage utility @@ -40,12 +40,13 @@ Options: -V, --version Print version ``` -## Install +### Install -### From PyPI +#### From PyPI Use [pipx](https://github.com/pypa/pipx) or [uv](https://github.com/astral-sh/uv) -to install the Python vault package from [PyPI](https://pypi.org/project/nitor-vault/) globally in an isolated environment. +to install the Python vault package from [PyPI](https://pypi.org/project/nitor-vault/) +globally in an isolated environment. ```shell pipx install nitor-vault @@ -55,7 +56,7 @@ uv tool install nitor-vault The command `vault` should now be available in path. -### From source +#### From source Build and install locally from source code using pip. This requires a [Rust toolchain](https://rustup.rs/) to be able to build the Rust library. @@ -77,6 +78,33 @@ and will not be available in path globally. which -a vault ``` +## Vault library + +This Python package can also be used as a Python library to interact with the Vault directly from Python code. + +Add the `nitor-vault` package to your project dependencies, +or install directly with pip. + +Example usage: + +```python +from n_vault import Vault + +if not Vault().exists("key"): + Vault().store("key", "value") + +keys = Vault().list_all() + +value = Vault().lookup("key") + +if Vault().exists("key"): + Vault().delete("key") + +# specify vault parameters +vault = Vault(vault_stack="stack-name", profile="aws-credentials-name") +value = vault.lookup("key") +``` + ## Development Uses: @@ -112,9 +140,9 @@ Run Python CLI: ```shell # uv -uv run python/p_vault/vault.py -h +uv run python/n_vault/cli.py -h # venv -python3 python/p_vault/vault.py -h +python3 python/n_vault/cli.py -h ``` Install and run vault inside virtual env: diff --git a/python-pyo3/pyproject.toml b/python-pyo3/pyproject.toml index 091cfded..e0ff9397 100644 --- a/python-pyo3/pyproject.toml +++ b/python-pyo3/pyproject.toml @@ -22,7 +22,7 @@ authors = [ { name = "Pasi Niemi", email = "pasi@nitor.com" }, { name = "Akseli Lukkarila", email = "akseli.lukkarila@nitor.com" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", @@ -30,14 +30,15 @@ classifiers = [ dependencies = [] [project.optional-dependencies] -build = ["maturin", "twine", "wheel"] -dev = ["ruff"] +build = ["maturin", "wheel"] +dev = ["maturin", "ruff"] [project.urls] Repository = "https://github.com/NitorCreations/vault" +Homepage = "https://github.com/NitorCreations/vault" [project.scripts] -vault = "p_vault.vault:main" +vault = "n_vault.cli:main" [build-system] requires = ["maturin>=1.7,<2.0"] @@ -46,9 +47,9 @@ build-backend = "maturin" [tool.maturin] bindings = "pyo3" features = ["pyo3/extension-module"] -module-name = "p_vault.nitor_vault_rs" +module-name = "n_vault.nitor_vault_rs" profile = "release" -python-packages = ["p_vault"] +python-packages = ["n_vault"] python-source = "python" strip = true @@ -59,7 +60,7 @@ venv = ".venv" [tool.ruff] # https://docs.astral.sh/ruff/configuration/ include = ["*.py", "*.pyi", "**/pyproject.toml"] -target-version = "py311" +target-version = "py39" line-length = 120 [tool.ruff.lint] diff --git a/python-pyo3/python/n_vault/__init__.py b/python-pyo3/python/n_vault/__init__.py new file mode 100644 index 00000000..7a5086a5 --- /dev/null +++ b/python-pyo3/python/n_vault/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2016-2024 Nitor Creations Oy +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Vault module for securely storing secrets in s3 with local encryption with data keys from AWS KMS. +""" + +# Simplify importing for library users, enabling: +# `from n_vault import Vault` + +from n_vault.vault import Vault as Vault # noqa: E402 diff --git a/python-pyo3/python/n_vault/cli.py b/python-pyo3/python/n_vault/cli.py new file mode 100644 index 00000000..0f1b8fd6 --- /dev/null +++ b/python-pyo3/python/n_vault/cli.py @@ -0,0 +1,33 @@ +# Copyright 2016-2024 Nitor Creations Oy +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from n_vault import nitor_vault_rs + + +def main(): + try: + # Override the script name in the arguments list so the Rust CLI works correctly + args = ["vault"] + sys.argv[1:] + nitor_vault_rs.run(args) + except KeyboardInterrupt: + print("\naborted") + exit(1) + except Exception as e: + print(f"Error: {e}") + + +if __name__ == "__main__": + main() diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py new file mode 100644 index 00000000..51e5dad4 --- /dev/null +++ b/python-pyo3/python/n_vault/vault.py @@ -0,0 +1,272 @@ +# Copyright 2016-2024 Nitor Creations Oy +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from collections.abc import Collection +from dataclasses import dataclass +from typing import Optional, Union + +from n_vault import nitor_vault_rs + + +@dataclass +class CloudFormationStackData: + """Vault stack data from AWS CloudFormation describe stack.""" + + result: str + bucket: Optional[str] + key: Optional[str] + status: Optional[str] + status_reason: Optional[str] + version: Optional[int] + + +@dataclass +class StackCreated: + """Result data for vault init.""" + + result: str + stack_name: Optional[str] + stack_id: Optional[str] + region: Optional[str] + + +@dataclass +class StackUpdated: + """Result data for vault update.""" + + result: str + stack_id: Optional[str] + previous_version: Optional[int] + new_version: Optional[int] + + +class Vault: + """ + Nitor Vault Python wrapper class around the Rust vault library. + + Note that initializing this class only saves the optional parameters, + but does *not* construct an actual vault instance. + Each method in this class creates its own Vault instance internally in the Rust library. + """ + + def __init__( + self, + vault_stack: str = None, + vault_key: str = None, + vault_bucket: str = None, + vault_prefix: str = None, + vault_region: str = None, + profile: str = None, + ): + self.vault_stack = vault_stack + self.vault_key = vault_key + self.vault_bucket = vault_bucket + self.vault_prefix = vault_prefix + self.vault_region = vault_region + self.profile = profile + + def all(self) -> str: + """ + Return a string with all keys separated by os.linesep. + """ + return os.linesep.join(item for item in self.list_all()) + + def delete(self, name: str) -> None: + """ + Delete data in S3 for given key name. + """ + return nitor_vault_rs.delete( + name, + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + + def delete_many(self, names: Collection[str]) -> None: + """ + Delete data for multiple keys. + + Takes in a collection of key name strings, such as a `list`, `tuple`, or `set`. + """ + return nitor_vault_rs.delete_many( + sorted(names), + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + + def direct_decrypt(self, encrypted_data: bytes) -> bytes: + """ + Decrypt data with KMS. + """ + return nitor_vault_rs.direct_decrypt( + encrypted_data, + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + + def direct_encrypt(self, data: Union[bytes, str]) -> bytes: + """ + Encrypt data with KMS. + """ + if isinstance(data, str): + data = data.encode("utf-8") + + return nitor_vault_rs.direct_encrypt( + data, + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + + def exists(self, name: str) -> bool: + """ + Check if the given key name already exists in the S3 bucket. + + Returns True if the key exists, False otherwise. + """ + return nitor_vault_rs.exists( + name, + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + + def init(self) -> Union[StackCreated, CloudFormationStackData]: + """ + Initialize new Vault stack. + + This will create all required resources in AWS, + after which the Vault can be used to store and lookup values. + + Returns a `StackCreated` if a new vault stack was initialized, + or `CloudFormationStackData` if it already exists. + """ + result = nitor_vault_rs.init( + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + profile=self.profile, + ) + result_status = result.get("result") + if result_status == "CREATED": + return StackCreated(**result) + elif result_status == "EXISTS" or result_status == "EXISTS_WITH_FAILED_STATE": + return CloudFormationStackData(**result) + + raise RuntimeError(f"Unexpected result data: {result}") + + def list_all(self) -> list[str]: + """ + Get all available secrets. + + Returns a list of key names. + """ + return nitor_vault_rs.list_all( + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + + def lookup(self, name: str) -> str: + """ + Lookup value for given key name. + + Always returns a string, with binary data encoded in base64. + """ + return nitor_vault_rs.lookup( + name, + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + + def stack_status(self) -> CloudFormationStackData: + """ + Get vault Cloudformation stack status. + """ + data = nitor_vault_rs.stack_status( + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + return CloudFormationStackData(**data) + + def store(self, name: str, data: Union[bytes, str]) -> None: + """ + Store encrypted value with given key name in S3. + """ + if isinstance(data, str): + data = data.encode("utf-8") + + return nitor_vault_rs.store( + name, + data, + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + + def update(self) -> Union[StackUpdated, CloudFormationStackData]: + """ + Update the vault Cloudformation stack with the current template. + + Returns `StackUpdated` if the vault stack was updated to a new version, + or `CloudFormationStackData` if it is already up to date. + """ + result = nitor_vault_rs.update( + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + result_status = result.get("result") + if result_status == "UPDATED": + return StackUpdated(**result) + elif result_status == "UP_TO_DATE": + return CloudFormationStackData(**result) + + raise RuntimeError(f"Unexpected result data: {result}") diff --git a/python-pyo3/python/p_vault/__init__.py b/python-pyo3/python/p_vault/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python-pyo3/python/p_vault/vault.py b/python-pyo3/python/p_vault/vault.py deleted file mode 100644 index 0af080fc..00000000 --- a/python-pyo3/python/p_vault/vault.py +++ /dev/null @@ -1,19 +0,0 @@ -import sys - -from p_vault import nitor_vault_rs - - -def main(): - try: - # Override the script name in the arguments list so the Rust CLI works correctly - args = ["vault"] + sys.argv[1:] - nitor_vault_rs.run(args) - except KeyboardInterrupt: - print("\naborted") - exit(1) - except Exception as e: - print(f"Error: {e}") - - -if __name__ == "__main__": - main() diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index 107df22e..82c34262 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -1,7 +1,239 @@ +use std::borrow::Cow; + use pyo3::prelude::*; +use pyo3::types::{IntoPyDict, PyDict}; use tokio::runtime::Runtime; +use nitor_vault::cloudformation::CloudFormationStackData; +use nitor_vault::errors::VaultError; +use nitor_vault::{CreateStackResult, UpdateStackResult, Value, Vault}; + +/// Convert `VaultError` to `anyhow::Error` +fn vault_error_to_anyhow(err: VaultError) -> anyhow::Error { + err.into() +} + +/// Convert `CloudFormationStackData` to a Python dictionary. +// Lifetime annotations are required due to `&str` usage, +// could be left out if passing a `String` for the result message. +fn stack_data_to_pydict<'a>( + py: Python<'a>, + data: CloudFormationStackData, + result: &'a str, +) -> Bound<'a, PyDict> { + let key_vals: Vec<(&str, PyObject)> = vec![ + ("result", result.to_string().to_object(py)), + ("bucket", data.bucket_name.to_object(py)), + ("key", data.key_arn.to_object(py)), + ( + "status", + data.status.map(|status| status.to_string()).to_object(py), + ), + ("status_reason", data.status_reason.to_object(py)), + ("version", data.version.to_object(py)), + ]; + key_vals.into_py_dict_bound(py) +} + +#[pyfunction(signature = (name, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn delete( + name: &str, + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult<()> { + Runtime::new()?.block_on(async { + Ok( + Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .delete(name) + .await + .map_err(vault_error_to_anyhow)?, + ) + }) +} + +#[pyfunction(signature = (names, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +#[allow(clippy::needless_pass_by_value)] +fn delete_many( + names: Vec, + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult<()> { + Runtime::new()?.block_on(async { + Ok( + Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .delete_many(&names) + .await + .map_err(vault_error_to_anyhow)?, + ) + }) +} + +#[pyfunction(signature = (data, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn direct_decrypt( + data: &[u8], + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult> { + // Returns Cow<[u8]> instead of Vec since that will get mapped to bytes for the Python side + // https://pyo3.rs/main/conversions/tables#returning-rust-values-to-python + Runtime::new()?.block_on(async { + let result = Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .direct_decrypt(data) + .await + .map_err(vault_error_to_anyhow)?; + + Ok(result.into()) + }) +} + +#[pyfunction(signature = (data, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn direct_encrypt( + data: &[u8], + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult> { + Runtime::new()?.block_on(async { + let result = Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .direct_encrypt(data) + .await + .map_err(vault_error_to_anyhow)?; + + Ok(result.into()) + }) +} + +#[pyfunction(signature = (name, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn exists( + name: &str, + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult { + Runtime::new()?.block_on(async { + let result: bool = Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .exists(name) + .await + .map_err(vault_error_to_anyhow)?; + + Ok(result) + }) +} + +#[pyfunction(signature = (vault_stack=None, region=None, bucket=None, profile=None))] +fn init( + vault_stack: Option, + region: Option, + bucket: Option, + profile: Option, +) -> PyResult { + let result = Runtime::new()?.block_on(async { + Vault::init(vault_stack, region, bucket, profile) + .await + .map_err(vault_error_to_anyhow) + })?; + Python::with_gil(|py| match result { + CreateStackResult::Exists { data } => { + let dict = stack_data_to_pydict(py, data, "EXISTS"); + Ok(dict.into()) + } + CreateStackResult::ExistsWithFailedState { data } => { + let dict = stack_data_to_pydict(py, data, "EXISTS_WITH_FAILED_STATE"); + Ok(dict.into()) + } + CreateStackResult::Created { + stack_name, + stack_id, + region, + } => { + let key_vals: Vec<(&str, PyObject)> = vec![ + ("result", "CREATED".to_string().to_object(py)), + ("stack_name", stack_name.to_object(py)), + ("stack_id", stack_id.to_object(py)), + ("region", region.to_string().to_object(py)), + ]; + let dict = key_vals.into_py_dict_bound(py); + Ok(dict.into()) + } + }) +} + +#[pyfunction(signature = (vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn list_all( + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult> { + Runtime::new()?.block_on(async { + let result = Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .all() + .await + .map_err(vault_error_to_anyhow)?; + + Ok(result) + }) +} + +#[pyfunction(signature = (name, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn lookup( + name: &str, + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult { + Runtime::new()?.block_on(async { + let result: Value = Box::pin( + Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .lookup(name), + ) + .await + .map_err(vault_error_to_anyhow)?; + + // Binary data will get base64 encoded in the Display trait implementation + Ok(result.to_string()) + }) +} + #[pyfunction] +/// Run Vault CLI with given args. fn run(args: Vec) -> PyResult<()> { Runtime::new()?.block_on(async { nitor_vault::run_cli_with_args(args).await?; @@ -9,9 +241,107 @@ fn run(args: Vec) -> PyResult<()> { }) } +#[pyfunction(signature = (vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn stack_status( + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult { + let data = Runtime::new()?.block_on(async { + Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .stack_status() + .await + .map_err(vault_error_to_anyhow) + })?; + + Python::with_gil(|py| { + let dict = stack_data_to_pydict(py, data, "SUCCESS"); + Ok(dict.into()) + }) +} + +#[pyfunction(signature = (name, value, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn store( + name: &str, + value: &[u8], + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult<()> { + Runtime::new()?.block_on(async { + Ok(Box::pin( + Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .store(name, value), + ) + .await + .map_err(vault_error_to_anyhow)?) + }) +} + +#[pyfunction(signature = (vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn update( + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult { + let result = Runtime::new()?.block_on(async { + Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .update_stack() + .await + .map_err(vault_error_to_anyhow) + })?; + + Python::with_gil(|py| match result { + UpdateStackResult::UpToDate { data } => { + let dict = stack_data_to_pydict(py, data, "UP_TO_DATE"); + Ok(dict.into()) + } + UpdateStackResult::Updated { + stack_id, + previous_version, + new_version, + } => { + let key_vals: Vec<(&str, PyObject)> = vec![ + ("result", "UPDATED".to_string().to_object(py)), + ("stack_id", stack_id.to_object(py)), + ("previous_version", previous_version.to_object(py)), + ("new_version", new_version.to_object(py)), + ]; + let dict = key_vals.into_py_dict_bound(py); + Ok(dict.into()) + } + }) +} + #[pymodule] #[pyo3(name = "nitor_vault_rs")] -fn vault(m: &Bound<'_, PyModule>) -> PyResult<()> { +fn nitor_vault_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(delete, m)?)?; + m.add_function(wrap_pyfunction!(delete_many, m)?)?; + m.add_function(wrap_pyfunction!(direct_decrypt, m)?)?; + m.add_function(wrap_pyfunction!(direct_encrypt, m)?)?; + m.add_function(wrap_pyfunction!(exists, m)?)?; + m.add_function(wrap_pyfunction!(init, m)?)?; + m.add_function(wrap_pyfunction!(list_all, m)?)?; + m.add_function(wrap_pyfunction!(lookup, m)?)?; m.add_function(wrap_pyfunction!(run, m)?)?; + m.add_function(wrap_pyfunction!(stack_status, m)?)?; + m.add_function(wrap_pyfunction!(store, m)?)?; + m.add_function(wrap_pyfunction!(update, m)?)?; Ok(()) } diff --git a/python-pyo3/uv.lock b/python-pyo3/uv.lock index 184250e7..ba3a3b42 100644 --- a/python-pyo3/uv.lock +++ b/python-pyo3/uv.lock @@ -350,7 +350,7 @@ wheels = [ [[package]] name = "nitor-vault" -version = "1.0.0" +version = "2.2.0" source = { editable = "." } [package.optional-dependencies] diff --git a/python/n_vault/__init__.py b/python/n_vault/__init__.py index 5bb77c73..66faddc2 100644 --- a/python/n_vault/__init__.py +++ b/python/n_vault/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. """ -Vault module for securely storing secrets in s3 with local encryption with data keys from AWS KMS +Vault module for securely storing secrets in s3 with local encryption with data keys from AWS KMS. """ import sys diff --git a/python/n_vault/vault.py b/python/n_vault/vault.py index 1a0bd592..675ac367 100644 --- a/python/n_vault/vault.py +++ b/python/n_vault/vault.py @@ -113,6 +113,9 @@ def __init__( self._vault_bucket = self._stack + "-" + self._region + "-" + account_id def store(self, name, data): + if isinstance(data, str): + data = data.encode("utf-8") + encrypted = self._encrypt(data) s3(**self._c_args).put_object( Bucket=self._vault_bucket, @@ -204,6 +207,7 @@ def all(self): ret = "" for item in self.list_all(): ret = ret + item + os.linesep + return ret def list_all(self): @@ -213,6 +217,7 @@ def list_all(self): ret.append(next_object.key[:-17]) elif next_object.key.endswith(".encrypted") and next_object.key[:-10] not in ret: ret.append(next_object.key[:-10]) + return ret def get_key(self): diff --git a/python/pyproject.toml b/python/pyproject.toml index f15382af..73bb59ae 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -22,7 +22,7 @@ authors = [ { name = "Pasi Niemi", email = "pasi@nitor.com" }, { name = "Akseli Lukkarila", email = "akseli.lukkarila@nitor.com" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } dependencies = [ "argcomplete", "cryptography", diff --git a/rust/src/args.rs b/rust/src/args.rs index d38c9199..6b7abba6 100644 --- a/rust/src/args.rs +++ b/rust/src/args.rs @@ -262,8 +262,17 @@ enum Command { }, } -/// Run Vault CLI with the given arguments -pub async fn run_cli_with_args(args: Vec) -> Result<()> { +/// Run Vault CLI with the given arguments. +/// +/// The argument list needs to include the binary name as the first element. +pub async fn run_cli_with_args(mut args: Vec) -> Result<()> { + // If args are empty, need to manually trigger the help output. + // `parse_from` does not do it automatically unlike `parse`. + if args.is_empty() { + args = vec!["vault".to_string(), "-h".to_string()]; + } else if args.len() == 1 { + args.push("-h".to_string()); + } let args = Args::parse_from(args); let quiet = args.quiet; @@ -279,7 +288,7 @@ pub async fn run_cli_with_args(args: Vec) -> Result<()> { Ok(()) } -/// Run Vault CLI +/// Run Vault CLI. pub async fn run_cli() -> Result<()> { let args = Args::parse(); let quiet = args.quiet; diff --git a/rust/src/errors.rs b/rust/src/errors.rs index 59819a7b..d338d0c9 100644 --- a/rust/src/errors.rs +++ b/rust/src/errors.rs @@ -54,8 +54,8 @@ pub enum VaultError { S3GetObjectError(#[from] SdkError), #[error("Failed deleting object from S3")] S3DeleteObjectError(#[from] SdkError), - #[error("Key does not exist in S3")] - S3DeleteObjectKeyMissingError, + #[error("Key does not exist in S3: '{name}'")] + S3DeleteObjectKeyMissingError { name: String }, #[error("Failed getting head-object from S3")] S3HeadObjectError(#[from] HeadObjectError), #[error("Failed to decrypt S3-object body")] diff --git a/rust/src/value.rs b/rust/src/value.rs index 3117ad70..66d0169e 100644 --- a/rust/src/value.rs +++ b/rust/src/value.rs @@ -160,11 +160,12 @@ impl fmt::Display for Value { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Utf8(text) => write!(f, "{text}"), - Self::Binary(data) => { - for byte in data { - write!(f, "{byte:02x}")?; - } - Ok(()) + Self::Binary(bytes) => { + write!( + f, + "{}", + base64::engine::general_purpose::STANDARD.encode(bytes) + ) } } } diff --git a/rust/src/vault.rs b/rust/src/vault.rs index bea967e5..2c480da0 100644 --- a/rust/src/vault.rs +++ b/rust/src/vault.rs @@ -222,12 +222,14 @@ impl Vault { } } - /// Get Cloudformation stack status. + /// Get Cloudformation vault stack status. pub async fn stack_status(&self) -> Result { cloudformation::get_stack_data(&self.cf, &self.cloudformation_params.stack_name).await } /// Get all available secrets. + /// + /// Returns a list of key names. pub async fn all(&self) -> Result, VaultError> { let output = self .s3 @@ -258,7 +260,9 @@ impl Vault { self.cloudformation_params.clone() } - /// Check if key already exists in bucket. + /// Check if the given key name already exists in the S3 bucket. + /// + /// Returns `true` if the key exists, `false` otherwise. pub async fn exists(&self, name: &str) -> Result { let name = self.full_key_name(name); match self @@ -283,7 +287,7 @@ impl Vault { } } - /// Store encrypted data in S3. + /// Store encrypted data with given key name in S3 pub async fn store(&self, name: &str, data: &[u8]) -> Result<(), VaultError> { let encrypted = self.encrypt(data).await?; @@ -300,10 +304,12 @@ impl Vault { Ok(()) } - /// Delete data in S3 for given key. + /// Delete data in S3 for given key name. pub async fn delete(&self, name: &str) -> Result<(), VaultError> { if !self.exists(name).await? { - return Err(VaultError::S3DeleteObjectKeyMissingError); + return Err(VaultError::S3DeleteObjectKeyMissingError { + name: name.to_string(), + }); } let key = &self.full_key_name(name); @@ -318,6 +324,14 @@ impl Vault { Ok(()) } + /// Delete data for multiple keys. + pub async fn delete_many(&self, names: &[String]) -> Result<(), VaultError> { + for name in names { + self.delete(name).await?; + } + Ok(()) + } + /// Return value for the given key name. /// If the data is valid UTF-8, it will be returned as a string. /// Otherwise, the raw bytes will be returned.