From 0964cb6765bc499ba68b4eba03b974e77708ace1 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 16:15:26 +0200 Subject: [PATCH 01/40] cargo update --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 70b429f7..5fb51798 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -627,9 +627,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.37" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" +checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8" dependencies = [ "shlex", ] @@ -739,9 +739,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" dependencies = [ "libc", ] @@ -1965,18 +1965,18 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", From 81cf61db0d52d4e1dd9750db778391f9c2edbf50 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 16:49:04 +0200 Subject: [PATCH 02/40] rename p_vault to n_vault again and move cli to cli.py --- python-pyo3/README.md | 4 ++-- python-pyo3/pyproject.toml | 6 +++--- python-pyo3/python/{p_vault => n_vault}/__init__.py | 0 python-pyo3/python/{p_vault/vault.py => n_vault/cli.py} | 2 +- python-pyo3/python/n_vault/vault.py | 0 5 files changed, 6 insertions(+), 6 deletions(-) rename python-pyo3/python/{p_vault => n_vault}/__init__.py (100%) rename python-pyo3/python/{p_vault/vault.py => n_vault/cli.py} (91%) create mode 100644 python-pyo3/python/n_vault/vault.py diff --git a/python-pyo3/README.md b/python-pyo3/README.md index 00423c8f..467bbffd 100644 --- a/python-pyo3/README.md +++ b/python-pyo3/README.md @@ -112,9 +112,9 @@ Run Python CLI: ```shell # uv -uv run python/p_vault/vault.py -h +uv run python/n_vault/cli.py -h # venv -python3 python/p_vault/vault.py -h +python3 python/n_vault/cli.py -h ``` Install and run vault inside virtual env: diff --git a/python-pyo3/pyproject.toml b/python-pyo3/pyproject.toml index 091cfded..47263b3b 100644 --- a/python-pyo3/pyproject.toml +++ b/python-pyo3/pyproject.toml @@ -37,7 +37,7 @@ dev = ["ruff"] Repository = "https://github.com/NitorCreations/vault" [project.scripts] -vault = "p_vault.vault:main" +vault = "n_vault.cli:main" [build-system] requires = ["maturin>=1.7,<2.0"] @@ -46,9 +46,9 @@ build-backend = "maturin" [tool.maturin] bindings = "pyo3" features = ["pyo3/extension-module"] -module-name = "p_vault.nitor_vault_rs" +module-name = "n_vault.nitor_vault_rs" profile = "release" -python-packages = ["p_vault"] +python-packages = ["n_vault"] python-source = "python" strip = true diff --git a/python-pyo3/python/p_vault/__init__.py b/python-pyo3/python/n_vault/__init__.py similarity index 100% rename from python-pyo3/python/p_vault/__init__.py rename to python-pyo3/python/n_vault/__init__.py diff --git a/python-pyo3/python/p_vault/vault.py b/python-pyo3/python/n_vault/cli.py similarity index 91% rename from python-pyo3/python/p_vault/vault.py rename to python-pyo3/python/n_vault/cli.py index 0af080fc..31c18e36 100644 --- a/python-pyo3/python/p_vault/vault.py +++ b/python-pyo3/python/n_vault/cli.py @@ -1,6 +1,6 @@ import sys -from p_vault import nitor_vault_rs +from n_vault import nitor_vault_rs def main(): diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py new file mode 100644 index 00000000..e69de29b From 610c4203598d4c92665a7c6a191fac64d0c735c9 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 16:49:25 +0200 Subject: [PATCH 03/40] expose list_all and lookup from rust library --- python-pyo3/python/n_vault/vault.py | 13 ++++++++++ python-pyo3/src/lib.rs | 37 ++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index e69de29b..0c4c691c 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -0,0 +1,13 @@ +from n_vault import nitor_vault_rs + + +class Vault: + """Nitor Vault wrapper around the Rust vault library.""" + + @staticmethod + def lookup(name: str) -> str: + return nitor_vault_rs.lookup(name) + + @staticmethod + def list_all() -> list[str]: + return nitor_vault_rs.list_all() diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index 107df22e..b5b6a1b2 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -1,6 +1,13 @@ +use nitor_vault::errors::VaultError; +use nitor_vault::{Value, Vault}; use pyo3::prelude::*; use tokio::runtime::Runtime; +/// Convert `VaultError` to `anyhow::Error` +fn vault_error_to_anyhow(err: VaultError) -> anyhow::Error { + err.into() +} + #[pyfunction] fn run(args: Vec) -> PyResult<()> { Runtime::new()?.block_on(async { @@ -9,9 +16,37 @@ fn run(args: Vec) -> PyResult<()> { }) } +#[pyfunction] +fn lookup(name: &str) -> PyResult { + Runtime::new()?.block_on(async { + let result: Value = Vault::default() + .await + .map_err(vault_error_to_anyhow)? + .lookup(name) + .map_err(vault_error_to_anyhow)?; + + Ok(result.to_string()) + }) +} + +#[pyfunction] +fn list_all() -> PyResult> { + Runtime::new()?.block_on(async { + let result = Vault::default() + .await + .map_err(vault_error_to_anyhow)? + .all() + .map_err(vault_error_to_anyhow)?; + + Ok(result) + }) +} + #[pymodule] #[pyo3(name = "nitor_vault_rs")] -fn vault(m: &Bound<'_, PyModule>) -> PyResult<()> { +fn module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(run, m)?)?; + m.add_function(wrap_pyfunction!(lookup, m)?)?; + m.add_function(wrap_pyfunction!(list_all, m)?)?; Ok(()) } From 6135a414322c000a3bf18e67b9b7fccbf1073a96 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 16:49:39 +0200 Subject: [PATCH 04/40] output binary data in base64 --- rust/src/value.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rust/src/value.rs b/rust/src/value.rs index 3117ad70..c9e0f06e 100644 --- a/rust/src/value.rs +++ b/rust/src/value.rs @@ -160,13 +160,11 @@ impl fmt::Display for Value { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Utf8(text) => write!(f, "{text}"), - Self::Binary(data) => { - for byte in data { - write!(f, "{byte:02x}")?; - } - Ok(()) + Self::Binary(bytes) => { + write!(f, "{}", base64::engine::general_purpose::STANDARD.encode(bytes))?; } } + Ok(()) } } From 98771bb25e0e7459077358924f7742de425a01f0 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 16:53:52 +0200 Subject: [PATCH 05/40] rust fixes --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- python-pyo3/src/lib.rs | 16 ++++++++++------ rust/src/value.rs | 7 +++++-- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5fb51798..d450614e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1478,7 +1478,7 @@ dependencies = [ [[package]] name = "nitor-vault" -version = "2.1.2" +version = "2.2.0" dependencies = [ "aes-gcm", "anyhow", @@ -1501,7 +1501,7 @@ dependencies = [ [[package]] name = "nitor-vault-pyo3" -version = "2.1.2" +version = "2.2.0" dependencies = [ "anyhow", "nitor-vault", diff --git a/Cargo.toml b/Cargo.toml index a9102d5e..7d1ad6bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "2.1.2" +version = "2.2.0" [profile.release] lto = "thin" diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index b5b6a1b2..35465b89 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -19,11 +19,14 @@ fn run(args: Vec) -> PyResult<()> { #[pyfunction] fn lookup(name: &str) -> PyResult { Runtime::new()?.block_on(async { - let result: Value = Vault::default() - .await - .map_err(vault_error_to_anyhow)? - .lookup(name) - .map_err(vault_error_to_anyhow)?; + let result: Value = Box::pin( + Vault::default() + .await + .map_err(vault_error_to_anyhow)? + .lookup(name), + ) + .await + .map_err(vault_error_to_anyhow)?; Ok(result.to_string()) }) @@ -36,6 +39,7 @@ fn list_all() -> PyResult> { .await .map_err(vault_error_to_anyhow)? .all() + .await .map_err(vault_error_to_anyhow)?; Ok(result) @@ -44,7 +48,7 @@ fn list_all() -> PyResult> { #[pymodule] #[pyo3(name = "nitor_vault_rs")] -fn module(m: &Bound<'_, PyModule>) -> PyResult<()> { +fn nitor_vault_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(run, m)?)?; m.add_function(wrap_pyfunction!(lookup, m)?)?; m.add_function(wrap_pyfunction!(list_all, m)?)?; diff --git a/rust/src/value.rs b/rust/src/value.rs index c9e0f06e..66d0169e 100644 --- a/rust/src/value.rs +++ b/rust/src/value.rs @@ -161,10 +161,13 @@ impl fmt::Display for Value { match self { Self::Utf8(text) => write!(f, "{text}"), Self::Binary(bytes) => { - write!(f, "{}", base64::engine::general_purpose::STANDARD.encode(bytes))?; + write!( + f, + "{}", + base64::engine::general_purpose::STANDARD.encode(bytes) + ) } } - Ok(()) } } From ed71953ff2762f63821fabe11efc3e51609c9b4f Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 20:00:43 +0200 Subject: [PATCH 06/40] add license texts --- python-pyo3/python/n_vault/__init__.py | 23 +++++++++++++++++++++++ python-pyo3/python/n_vault/cli.py | 14 ++++++++++++++ python-pyo3/python/n_vault/vault.py | 15 +++++++++++++++ python/n_vault/__init__.py | 2 +- 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/python-pyo3/python/n_vault/__init__.py b/python-pyo3/python/n_vault/__init__.py index e69de29b..1cca808d 100644 --- a/python-pyo3/python/n_vault/__init__.py +++ b/python-pyo3/python/n_vault/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2016-2024 Nitor Creations Oy +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Vault module for securely storing secrets in s3 with local encryption with data keys from AWS KMS. +""" + +# Simplify importing for library users, +# so one can simply do: +# from n_vault import Vault + +from n_vault.vault import Vault as Vault # noqa: E402 diff --git a/python-pyo3/python/n_vault/cli.py b/python-pyo3/python/n_vault/cli.py index 31c18e36..0f1b8fd6 100644 --- a/python-pyo3/python/n_vault/cli.py +++ b/python-pyo3/python/n_vault/cli.py @@ -1,3 +1,17 @@ +# Copyright 2016-2024 Nitor Creations Oy +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys from n_vault import nitor_vault_rs diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 0c4c691c..3b6b051d 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -1,3 +1,18 @@ +# Copyright 2016-2024 Nitor Creations Oy +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + from n_vault import nitor_vault_rs diff --git a/python/n_vault/__init__.py b/python/n_vault/__init__.py index 5bb77c73..66faddc2 100644 --- a/python/n_vault/__init__.py +++ b/python/n_vault/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. """ -Vault module for securely storing secrets in s3 with local encryption with data keys from AWS KMS +Vault module for securely storing secrets in s3 with local encryption with data keys from AWS KMS. """ import sys From a8feeeed4d1ac06e731a377cc42c670b4b43d34b Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 20:03:15 +0200 Subject: [PATCH 07/40] implement store, exists, and delete --- python-pyo3/python/n_vault/vault.py | 15 +++++++ python-pyo3/src/lib.rs | 70 +++++++++++++++++++++++------ python-pyo3/uv.lock | 2 +- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 3b6b051d..42b50f2e 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -26,3 +26,18 @@ def lookup(name: str) -> str: @staticmethod def list_all() -> list[str]: return nitor_vault_rs.list_all() + + @staticmethod + def delete(name: str) -> None: + return nitor_vault_rs.delete(name) + + @staticmethod + def store(key: str, value: bytes | str) -> None: + if isinstance(value, str): + value = value.encode("utf-8") + + return nitor_vault_rs.store(key, value) + + @staticmethod + def exists(name: str) -> bool: + return nitor_vault_rs.exists(name) diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index 35465b89..f210a604 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -9,10 +9,42 @@ fn vault_error_to_anyhow(err: VaultError) -> anyhow::Error { } #[pyfunction] -fn run(args: Vec) -> PyResult<()> { +fn delete(name: &str) -> PyResult<()> { Runtime::new()?.block_on(async { - nitor_vault::run_cli_with_args(args).await?; - Ok(()) + Ok(Vault::default() + .await + .map_err(vault_error_to_anyhow)? + .delete(name) + .await + .map_err(vault_error_to_anyhow)?) + }) +} + +#[pyfunction] +fn exists(name: &str) -> PyResult { + Runtime::new()?.block_on(async { + let result: bool = Vault::default() + .await + .map_err(vault_error_to_anyhow)? + .exists(name) + .await + .map_err(vault_error_to_anyhow)?; + + Ok(result) + }) +} + +#[pyfunction] +fn list_all() -> PyResult> { + Runtime::new()?.block_on(async { + let result = Vault::default() + .await + .map_err(vault_error_to_anyhow)? + .all() + .await + .map_err(vault_error_to_anyhow)?; + + Ok(result) }) } @@ -33,24 +65,36 @@ fn lookup(name: &str) -> PyResult { } #[pyfunction] -fn list_all() -> PyResult> { +/// Run Vault CLI with given args. +fn run(args: Vec) -> PyResult<()> { Runtime::new()?.block_on(async { - let result = Vault::default() - .await - .map_err(vault_error_to_anyhow)? - .all() - .await - .map_err(vault_error_to_anyhow)?; + nitor_vault::run_cli_with_args(args).await?; + Ok(()) + }) +} - Ok(result) +#[pyfunction] +fn store(key: &str, value: &[u8]) -> PyResult<()> { + Runtime::new()?.block_on(async { + Ok(Box::pin( + Vault::default() + .await + .map_err(vault_error_to_anyhow)? + .store(key, value), + ) + .await + .map_err(vault_error_to_anyhow)?) }) } #[pymodule] #[pyo3(name = "nitor_vault_rs")] fn nitor_vault_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(run, m)?)?; - m.add_function(wrap_pyfunction!(lookup, m)?)?; + m.add_function(wrap_pyfunction!(delete, m)?)?; + m.add_function(wrap_pyfunction!(exists, m)?)?; m.add_function(wrap_pyfunction!(list_all, m)?)?; + m.add_function(wrap_pyfunction!(lookup, m)?)?; + m.add_function(wrap_pyfunction!(run, m)?)?; + m.add_function(wrap_pyfunction!(store, m)?)?; Ok(()) } diff --git a/python-pyo3/uv.lock b/python-pyo3/uv.lock index 184250e7..ba3a3b42 100644 --- a/python-pyo3/uv.lock +++ b/python-pyo3/uv.lock @@ -350,7 +350,7 @@ wheels = [ [[package]] name = "nitor-vault" -version = "1.0.0" +version = "2.2.0" source = { editable = "." } [package.optional-dependencies] From 52e6dcc73bfa97671043b047bf7225d40c896a9f Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 20:25:15 +0200 Subject: [PATCH 08/40] update pyproject information --- python-pyo3/pyproject.toml | 3 ++- python/pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python-pyo3/pyproject.toml b/python-pyo3/pyproject.toml index 47263b3b..21e0a303 100644 --- a/python-pyo3/pyproject.toml +++ b/python-pyo3/pyproject.toml @@ -22,7 +22,7 @@ authors = [ { name = "Pasi Niemi", email = "pasi@nitor.com" }, { name = "Akseli Lukkarila", email = "akseli.lukkarila@nitor.com" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", @@ -35,6 +35,7 @@ dev = ["ruff"] [project.urls] Repository = "https://github.com/NitorCreations/vault" +Homepage = "https://github.com/NitorCreations/vault" [project.scripts] vault = "n_vault.cli:main" diff --git a/python/pyproject.toml b/python/pyproject.toml index f15382af..73bb59ae 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -22,7 +22,7 @@ authors = [ { name = "Pasi Niemi", email = "pasi@nitor.com" }, { name = "Akseli Lukkarila", email = "akseli.lukkarila@nitor.com" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } dependencies = [ "argcomplete", "cryptography", From 2d289683b4f5cfe34bac706670fa33f4f52e5823 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 20:39:48 +0200 Subject: [PATCH 09/40] add integration test cases for python library --- .github/workflows/integration.yml | 57 +++++++++++++++++++++++++++-- python-pyo3/python/n_vault/vault.py | 16 ++++---- 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7992e621..e163b4ed 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -218,9 +218,6 @@ jobs: - name: Delete secret with Rust run: bin/rust/vault -d "secret-${{github.sha}}.zip" - - name: Verify that key has been deleted with Rust - run: bin/rust/vault exists secret-${{github.sha}}.zip | grep -q "does not exist" - - name: Verify that keys have been deleted using Rust run: | bin/rust/vault exists secret-python | grep -q "key 'secret-python' does not exist" @@ -229,6 +226,33 @@ jobs: bin/rust/vault exists secret-rust | grep -q "key 'secret-rust' does not exist" bin/rust/vault exists secret-nodejs | grep -q "key 'secret-nodejs' does not exist" + - name: Check Python vault package + run: python -m pip show nitor-vault + + - name: Store secret using Python library + run: | + python -c "from n_vault import Vault; Vault().store('secret-python-library', 'sha-${{github.sha}}')" + + - name: Verify secret using Python library + run: | + python -c "from n_vault import Vault; print('true') if Vault().exists('secret-python-library') else print('false')" | grep -q "true" + + - name: Validate storing worked with Rust + run: diff <(bin/rust/vault -l secret-python-library) <(echo -n sha-${{github.sha}}) + + - name: Lookup with Python library + run: | + diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end="", flush=True)") <(echo -n sha-${{github.sha}}) + + - name: List with Python library + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1" + + - name: Delete with Python library + run: python -c "from n_vault import Vault; Vault().delete('secret-python-library')" + + - name: Verify that key has been deleted with Rust + run: bin/rust/vault exists secret-python-library | grep -q "key 'secret-python-library' does not exist" + - name: Install Python PyO3 vault run: python -m pip install --force-reinstall . working-directory: python-pyo3 @@ -310,3 +334,30 @@ jobs: - name: Verify that key has been deleted with Python-pyo3 run: vault exists secret-${{github.sha}}.zip | grep -q "does not exist" + + - name: Check Python vault package + run: python -m pip show nitor-vault + + - name: Store secret using Python library + run: | + python -c "from n_vault import Vault; Vault().store('secret-python-library', 'sha-${{github.sha}}')" + + - name: Verify secret using Python library + run: | + python -c "from n_vault import Vault; print('true') if Vault().exists('secret-python-library') else print('false')" | grep -q "true" + + - name: Validate storing worked with Rust + run: diff <(bin/rust/vault -l secret-python-library) <(echo -n sha-${{github.sha}}) + + - name: Lookup with Python library + run: | + diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end="", flush=True)") <(echo -n sha-${{github.sha}}) + + - name: List with Python library + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1" + + - name: Delete with Python library + run: python -c "from n_vault import Vault; Vault().delete('secret-python-library')" + + - name: Verify that key has been deleted with Rust + run: bin/rust/vault exists secret-python-library | grep -q "key 'secret-python-library' does not exist" diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 42b50f2e..c272a7e1 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -20,16 +20,20 @@ class Vault: """Nitor Vault wrapper around the Rust vault library.""" @staticmethod - def lookup(name: str) -> str: - return nitor_vault_rs.lookup(name) + def delete(name: str) -> None: + return nitor_vault_rs.delete(name) + + @staticmethod + def exists(name: str) -> bool: + return nitor_vault_rs.exists(name) @staticmethod def list_all() -> list[str]: return nitor_vault_rs.list_all() @staticmethod - def delete(name: str) -> None: - return nitor_vault_rs.delete(name) + def lookup(name: str) -> str: + return nitor_vault_rs.lookup(name) @staticmethod def store(key: str, value: bytes | str) -> None: @@ -37,7 +41,3 @@ def store(key: str, value: bytes | str) -> None: value = value.encode("utf-8") return nitor_vault_rs.store(key, value) - - @staticmethod - def exists(name: str) -> bool: - return nitor_vault_rs.exists(name) From e21b8f0a05570cea6ff8803950979bc08f64eed8 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 20:56:07 +0200 Subject: [PATCH 10/40] tweak comment --- python-pyo3/python/n_vault/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python-pyo3/python/n_vault/__init__.py b/python-pyo3/python/n_vault/__init__.py index 1cca808d..7a5086a5 100644 --- a/python-pyo3/python/n_vault/__init__.py +++ b/python-pyo3/python/n_vault/__init__.py @@ -16,8 +16,7 @@ Vault module for securely storing secrets in s3 with local encryption with data keys from AWS KMS. """ -# Simplify importing for library users, -# so one can simply do: -# from n_vault import Vault +# Simplify importing for library users, enabling: +# `from n_vault import Vault` from n_vault.vault import Vault as Vault # noqa: E402 From db45f2a3a9e5c6030efbcf0277c76c78a071bb30 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 21:05:11 +0200 Subject: [PATCH 11/40] fix integration test trigger for PRs --- .github/workflows/integration.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e163b4ed..6aa721e4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,8 +8,6 @@ on: paths: - "!**/README.md" pull_request: - paths: - - "!**/README.md" permissions: id-token: write From 784caf3b18ec6c0cf26561eacfa54657c96754e7 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 21:07:07 +0200 Subject: [PATCH 12/40] cancel previous runs for PR wheel build --- .github/workflows/python-wheel.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/python-wheel.yml b/.github/workflows/python-wheel.yml index 67c557ee..da0dd418 100644 --- a/.github/workflows/python-wheel.yml +++ b/.github/workflows/python-wheel.yml @@ -28,6 +28,11 @@ on: permissions: contents: read +# Cancel previous runs for PRs but not pushes to main +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + jobs: linux: runs-on: ${{ matrix.platform.runner }} From 5c75a509dfbdab013b373249922d89fbae3135bf Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 21:17:58 +0200 Subject: [PATCH 13/40] fix store when input argument is a string --- python/n_vault/vault.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/n_vault/vault.py b/python/n_vault/vault.py index 1a0bd592..81fbc90f 100644 --- a/python/n_vault/vault.py +++ b/python/n_vault/vault.py @@ -113,6 +113,9 @@ def __init__( self._vault_bucket = self._stack + "-" + self._region + "-" + account_id def store(self, name, data): + if isinstance(data, str): + data = data.encode("utf-8") + encrypted = self._encrypt(data) s3(**self._c_args).put_object( Bucket=self._vault_bucket, From 2b2a55a4d3a59d64cc2aacc5203683b08924b83a Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 21:22:45 +0200 Subject: [PATCH 14/40] fix empty string for print end arg --- .github/workflows/integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6aa721e4..2d505625 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -240,7 +240,7 @@ jobs: - name: Lookup with Python library run: | - diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end="", flush=True)") <(echo -n sha-${{github.sha}}) + diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end='', flush=True)") <(echo -n sha-${{github.sha}}) - name: List with Python library run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1" @@ -349,7 +349,7 @@ jobs: - name: Lookup with Python library run: | - diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end="", flush=True)") <(echo -n sha-${{github.sha}}) + diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end='', flush=True)") <(echo -n sha-${{github.sha}}) - name: List with Python library run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1" From 979c9aaec67aeb967e2fbb6584476828dac0433b Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 21:29:12 +0200 Subject: [PATCH 15/40] add extra rust cache key for release build --- .github/workflows/integration.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2d505625..74081f39 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -35,6 +35,9 @@ jobs: - uses: Swatinem/rust-cache@v2.7.5 if: ${{ matrix.lang == 'rust'}} + with: + # The build script creates a `release` build so use separate cache + key: "release" - uses: actions/setup-go@v5 if: ${{ matrix.lang == 'go'}} From 47f28e168094322b9413941ce73724cba4147704 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 21:43:34 +0200 Subject: [PATCH 16/40] convert lookup bytes output to string --- .github/workflows/integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 74081f39..f7b52e0e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -243,7 +243,7 @@ jobs: - name: Lookup with Python library run: | - diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end='', flush=True)") <(echo -n sha-${{github.sha}}) + diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library').decode('utf-8'), end='', flush=True)") <(echo -n sha-${{github.sha}}) - name: List with Python library run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1" @@ -352,7 +352,7 @@ jobs: - name: Lookup with Python library run: | - diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end='', flush=True)") <(echo -n sha-${{github.sha}}) + diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library').decode('utf-8'), end='', flush=True)") <(echo -n sha-${{github.sha}}) - name: List with Python library run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1" From 1b44ce543c20884b056f69548598337e9cc9a8e4 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 21:50:02 +0200 Subject: [PATCH 17/40] workaround list all case since there seems to be unexpected extra keys --- .github/workflows/integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f7b52e0e..9d3270e3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -246,7 +246,7 @@ jobs: diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library').decode('utf-8'), end='', flush=True)") <(echo -n sha-${{github.sha}}) - name: List with Python library - run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1" + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" - name: Delete with Python library run: python -c "from n_vault import Vault; Vault().delete('secret-python-library')" @@ -355,7 +355,7 @@ jobs: diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library').decode('utf-8'), end='', flush=True)") <(echo -n sha-${{github.sha}}) - name: List with Python library - run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1" + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" - name: Delete with Python library run: python -c "from n_vault import Vault; Vault().delete('secret-python-library')" From cbb3378c5daf646273ba848c40e0d7a2257e4e4b Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 22:17:43 +0200 Subject: [PATCH 18/40] remove unneeded decode from python rust lib test --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9d3270e3..6c5be8c4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -352,7 +352,7 @@ jobs: - name: Lookup with Python library run: | - diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library').decode('utf-8'), end='', flush=True)") <(echo -n sha-${{github.sha}}) + diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end='', flush=True)") <(echo -n sha-${{github.sha}}) - name: List with Python library run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" From 045fc0e2302b5dc4ca7b9b0054256bc30531f175 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 22:18:11 +0200 Subject: [PATCH 19/40] add `delete_many` to library --- .github/workflows/integration.yml | 11 ++++++++++- python-pyo3/python/n_vault/vault.py | 6 ++++++ python-pyo3/src/lib.rs | 14 ++++++++++++++ rust/src/errors.rs | 4 ++-- rust/src/vault.rs | 12 +++++++++++- 5 files changed, 43 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6c5be8c4..2e8348bb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -339,6 +339,15 @@ jobs: - name: Check Python vault package run: python -m pip show nitor-vault + - name: List with Python library + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" + + - name: Delete all keys with Python library + run: python -c "from n_vault import Vault; Vault().delete_many(Vault().list_all())" + + - name: List with Python library + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "0" + - name: Store secret using Python library run: | python -c "from n_vault import Vault; Vault().store('secret-python-library', 'sha-${{github.sha}}')" @@ -355,7 +364,7 @@ jobs: diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end='', flush=True)") <(echo -n sha-${{github.sha}}) - name: List with Python library - run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1" - name: Delete with Python library run: python -c "from n_vault import Vault; Vault().delete('secret-python-library')" diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index c272a7e1..0d8a2893 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -13,6 +13,8 @@ # limitations under the License. +from collections.abc import Collection + from n_vault import nitor_vault_rs @@ -23,6 +25,10 @@ class Vault: def delete(name: str) -> None: return nitor_vault_rs.delete(name) + @staticmethod + def delete_many(names: Collection[str]) -> None: + return nitor_vault_rs.delete_many(sorted(names)) + @staticmethod def exists(name: str) -> bool: return nitor_vault_rs.exists(name) diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index f210a604..9013ad78 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -20,6 +20,19 @@ fn delete(name: &str) -> PyResult<()> { }) } +#[pyfunction] +#[allow(clippy::needless_pass_by_value)] +fn delete_many(names: Vec) -> PyResult<()> { + Runtime::new()?.block_on(async { + Ok(Vault::default() + .await + .map_err(vault_error_to_anyhow)? + .delete_many(&names) + .await + .map_err(vault_error_to_anyhow)?) + }) +} + #[pyfunction] fn exists(name: &str) -> PyResult { Runtime::new()?.block_on(async { @@ -91,6 +104,7 @@ fn store(key: &str, value: &[u8]) -> PyResult<()> { #[pyo3(name = "nitor_vault_rs")] fn nitor_vault_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(delete, m)?)?; + m.add_function(wrap_pyfunction!(delete_many, m)?)?; m.add_function(wrap_pyfunction!(exists, m)?)?; m.add_function(wrap_pyfunction!(list_all, m)?)?; m.add_function(wrap_pyfunction!(lookup, m)?)?; diff --git a/rust/src/errors.rs b/rust/src/errors.rs index 59819a7b..d338d0c9 100644 --- a/rust/src/errors.rs +++ b/rust/src/errors.rs @@ -54,8 +54,8 @@ pub enum VaultError { S3GetObjectError(#[from] SdkError), #[error("Failed deleting object from S3")] S3DeleteObjectError(#[from] SdkError), - #[error("Key does not exist in S3")] - S3DeleteObjectKeyMissingError, + #[error("Key does not exist in S3: '{name}'")] + S3DeleteObjectKeyMissingError { name: String }, #[error("Failed getting head-object from S3")] S3HeadObjectError(#[from] HeadObjectError), #[error("Failed to decrypt S3-object body")] diff --git a/rust/src/vault.rs b/rust/src/vault.rs index bea967e5..b7bc4c74 100644 --- a/rust/src/vault.rs +++ b/rust/src/vault.rs @@ -303,7 +303,9 @@ impl Vault { /// Delete data in S3 for given key. pub async fn delete(&self, name: &str) -> Result<(), VaultError> { if !self.exists(name).await? { - return Err(VaultError::S3DeleteObjectKeyMissingError); + return Err(VaultError::S3DeleteObjectKeyMissingError { + name: name.to_string(), + }); } let key = &self.full_key_name(name); @@ -318,6 +320,14 @@ impl Vault { Ok(()) } + /// Delete data for multiple keys. + pub async fn delete_many(&self, names: &[String]) -> Result<(), VaultError> { + for name in names { + self.delete(name).await?; + } + Ok(()) + } + /// Return value for the given key name. /// If the data is valid UTF-8, it will be returned as a string. /// Otherwise, the raw bytes will be returned. From cb1728f09266c1d95853c77e6ef1c6a2ce81f792 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 23:17:40 +0200 Subject: [PATCH 20/40] fix wc -l to not count empty lines --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2e8348bb..32f19335 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -346,7 +346,7 @@ jobs: run: python -c "from n_vault import Vault; Vault().delete_many(Vault().list_all())" - name: List with Python library - run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "0" + run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | grep -ve '^\s*$' | wc -l | grep -q "0" - name: Store secret using Python library run: | From 3b6b9c1b1ea9d06e59838ba4f5f2f7ccbb0303b8 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 23:27:47 +0200 Subject: [PATCH 21/40] remove extra list all --- .github/workflows/integration.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 32f19335..a9af94b7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -339,9 +339,6 @@ jobs: - name: Check Python vault package run: python -m pip show nitor-vault - - name: List with Python library - run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" - - name: Delete all keys with Python library run: python -c "from n_vault import Vault; Vault().delete_many(Vault().list_all())" From 5714c023cc40ada68f9afe53dd6abd844a8ed68a Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Tue, 12 Nov 2024 23:59:47 +0200 Subject: [PATCH 22/40] add readme section for python library usage --- python-pyo3/README.md | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/python-pyo3/README.md b/python-pyo3/README.md index 467bbffd..31782dda 100644 --- a/python-pyo3/README.md +++ b/python-pyo3/README.md @@ -4,7 +4,7 @@ Python vault implementation using the Rust vault library. See the [repo](https://github.com/NitorCreations/vault) root readme for more general information. -## Usage +## Vault CLI ```console Encrypted AWS key-value storage utility @@ -40,12 +40,13 @@ Options: -V, --version Print version ``` -## Install +### Install -### From PyPI +#### From PyPI Use [pipx](https://github.com/pypa/pipx) or [uv](https://github.com/astral-sh/uv) -to install the Python vault package from [PyPI](https://pypi.org/project/nitor-vault/) globally in an isolated environment. +to install the Python vault package from [PyPI](https://pypi.org/project/nitor-vault/) +globally in an isolated environment. ```shell pipx install nitor-vault @@ -55,7 +56,7 @@ uv tool install nitor-vault The command `vault` should now be available in path. -### From source +#### From source Build and install locally from source code using pip. This requires a [Rust toolchain](https://rustup.rs/) to be able to build the Rust library. @@ -77,6 +78,29 @@ and will not be available in path globally. which -a vault ``` +## Vault library + +This Python package can also be used as a Python library to interact with the Vault directly from Python code. + +Add the `nitor-vault` package to your project dependencies, +or install directly with pip. + +Example usage: + +```python +from n_vault import Vault + +if not Vault().exists("key"): + Vault().store("key", "value") + +keys = Vault().list_all() + +value = Vault().lookup("key") + +if Vault().exists("key"): + Vault().delete("key") +``` + ## Development Uses: From 41d3a7c8932694e5effc54852e19582f804c4c44 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 00:05:42 +0200 Subject: [PATCH 23/40] support specifying vault parameters --- python-pyo3/README.md | 4 + python-pyo3/python/n_vault/vault.py | 94 +++++++++++++++++++----- python-pyo3/src/lib.rs | 110 ++++++++++++++++++++-------- 3 files changed, 161 insertions(+), 47 deletions(-) diff --git a/python-pyo3/README.md b/python-pyo3/README.md index 31782dda..378d8d02 100644 --- a/python-pyo3/README.md +++ b/python-pyo3/README.md @@ -99,6 +99,10 @@ value = Vault().lookup("key") if Vault().exists("key"): Vault().delete("key") + +# specify vault parameters +vault = Vault(vault_stack="stack-name", profile="aws-credentials-name") +value = vault.lookup("key") ``` ## Development diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 0d8a2893..294e7f46 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -21,29 +21,87 @@ class Vault: """Nitor Vault wrapper around the Rust vault library.""" - @staticmethod - def delete(name: str) -> None: - return nitor_vault_rs.delete(name) + def __init__( + self, + vault_stack: str = None, + region: str = None, + bucket: str = None, + key: str = None, + prefix: str = None, + profile: str = None, + ): + self.vault_stack = vault_stack + self.region = region + self.bucket = bucket + self.key = key + self.prefix = prefix + self.profile = profile - @staticmethod - def delete_many(names: Collection[str]) -> None: - return nitor_vault_rs.delete_many(sorted(names)) + def delete(self, name: str) -> None: + return nitor_vault_rs.delete( + name, + vault_stack=self.vault_stack, + region=self.region, + bucket=self.bucket, + key=self.key, + prefix=self.prefix, + profile=self.profile, + ) - @staticmethod - def exists(name: str) -> bool: - return nitor_vault_rs.exists(name) + def delete_many(self, names: Collection[str]) -> None: + return nitor_vault_rs.delete_many( + sorted(names), + vault_stack=self.vault_stack, + region=self.region, + bucket=self.bucket, + key=self.key, + prefix=self.prefix, + profile=self.profile, + ) - @staticmethod - def list_all() -> list[str]: - return nitor_vault_rs.list_all() + def exists(self, name: str) -> bool: + return nitor_vault_rs.exists( + name, + vault_stack=self.vault_stack, + region=self.region, + bucket=self.bucket, + key=self.key, + prefix=self.prefix, + profile=self.profile, + ) - @staticmethod - def lookup(name: str) -> str: - return nitor_vault_rs.lookup(name) + def list_all(self) -> list[str]: + return nitor_vault_rs.list_all( + vault_stack=self.vault_stack, + region=self.region, + bucket=self.bucket, + key=self.key, + prefix=self.prefix, + profile=self.profile, + ) - @staticmethod - def store(key: str, value: bytes | str) -> None: + def lookup(self, name: str) -> str: + return nitor_vault_rs.lookup( + name, + vault_stack=self.vault_stack, + region=self.region, + bucket=self.bucket, + key=self.key, + prefix=self.prefix, + profile=self.profile, + ) + + def store(self, key: str, value: bytes | str) -> None: if isinstance(value, str): value = value.encode("utf-8") - return nitor_vault_rs.store(key, value) + return nitor_vault_rs.store( + key, + value, + vault_stack=self.vault_stack, + region=self.region, + bucket=self.bucket, + key=self.key, + prefix=self.prefix, + profile=self.profile, + ) diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index 9013ad78..2fe28fd2 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -8,35 +8,63 @@ fn vault_error_to_anyhow(err: VaultError) -> anyhow::Error { err.into() } -#[pyfunction] -fn delete(name: &str) -> PyResult<()> { +#[pyfunction(signature = (name, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn delete( + name: &str, + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult<()> { Runtime::new()?.block_on(async { - Ok(Vault::default() - .await - .map_err(vault_error_to_anyhow)? - .delete(name) - .await - .map_err(vault_error_to_anyhow)?) + Ok( + Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .delete(name) + .await + .map_err(vault_error_to_anyhow)?, + ) }) } -#[pyfunction] +#[pyfunction(signature = (names, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] #[allow(clippy::needless_pass_by_value)] -fn delete_many(names: Vec) -> PyResult<()> { +fn delete_many( + names: Vec, + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult<()> { Runtime::new()?.block_on(async { - Ok(Vault::default() - .await - .map_err(vault_error_to_anyhow)? - .delete_many(&names) - .await - .map_err(vault_error_to_anyhow)?) + Ok( + Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .delete_many(&names) + .await + .map_err(vault_error_to_anyhow)?, + ) }) } -#[pyfunction] -fn exists(name: &str) -> PyResult { +#[pyfunction(signature = (name, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn exists( + name: &str, + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult { Runtime::new()?.block_on(async { - let result: bool = Vault::default() + let result: bool = Vault::new(vault_stack, region, bucket, key, prefix, profile) .await .map_err(vault_error_to_anyhow)? .exists(name) @@ -47,10 +75,17 @@ fn exists(name: &str) -> PyResult { }) } -#[pyfunction] -fn list_all() -> PyResult> { +#[pyfunction(signature = (vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn list_all( + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult> { Runtime::new()?.block_on(async { - let result = Vault::default() + let result = Vault::new(vault_stack, region, bucket, key, prefix, profile) .await .map_err(vault_error_to_anyhow)? .all() @@ -61,11 +96,19 @@ fn list_all() -> PyResult> { }) } -#[pyfunction] -fn lookup(name: &str) -> PyResult { +#[pyfunction(signature = (name, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn lookup( + name: &str, + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult { Runtime::new()?.block_on(async { let result: Value = Box::pin( - Vault::default() + Vault::new(vault_stack, region, bucket, key, prefix, profile) .await .map_err(vault_error_to_anyhow)? .lookup(name), @@ -86,14 +129,23 @@ fn run(args: Vec) -> PyResult<()> { }) } -#[pyfunction] -fn store(key: &str, value: &[u8]) -> PyResult<()> { +#[pyfunction(signature = (name, value, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn store( + name: &str, + value: &[u8], + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult<()> { Runtime::new()?.block_on(async { Ok(Box::pin( - Vault::default() + Vault::new(vault_stack, region, bucket, key, prefix, profile) .await .map_err(vault_error_to_anyhow)? - .store(key, value), + .store(name, value), ) .await .map_err(vault_error_to_anyhow)?) From 80b646a08e61475d771cdfe57775b7a292e3880c Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 02:07:55 +0200 Subject: [PATCH 24/40] add init and update --- python-pyo3/python/n_vault/vault.py | 18 +++++ python-pyo3/src/lib.rs | 106 +++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 294e7f46..65f6a1b1 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -70,6 +70,14 @@ def exists(self, name: str) -> bool: profile=self.profile, ) + def init(self) -> dict[str]: + return nitor_vault_rs.init( + vault_stack=self.vault_stack, + region=self.region, + bucket=self.bucket, + profile=self.profile, + ) + def list_all(self) -> list[str]: return nitor_vault_rs.list_all( vault_stack=self.vault_stack, @@ -105,3 +113,13 @@ def store(self, key: str, value: bytes | str) -> None: prefix=self.prefix, profile=self.profile, ) + + def update(self) -> dict[str]: + return nitor_vault_rs.update( + vault_stack=self.vault_stack, + region=self.region, + bucket=self.bucket, + key=self.key, + prefix=self.prefix, + profile=self.profile, + ) diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index 2fe28fd2..ca4be788 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -1,5 +1,8 @@ +use std::collections::HashMap; + +use nitor_vault::cloudformation::CloudFormationStackData; use nitor_vault::errors::VaultError; -use nitor_vault::{Value, Vault}; +use nitor_vault::{CreateStackResult, UpdateStackResult, Value, Vault}; use pyo3::prelude::*; use tokio::runtime::Runtime; @@ -8,6 +11,38 @@ fn vault_error_to_anyhow(err: VaultError) -> anyhow::Error { err.into() } +fn to_hash_map(stack_data: CloudFormationStackData, result: String) -> HashMap { + let mut map = HashMap::new(); + map.insert("result".to_string(), result); + map.insert( + "bucket_name".to_string(), + stack_data.bucket_name.clone().unwrap_or_default(), + ); + map.insert( + "key_arn".to_string(), + stack_data.key_arn.clone().unwrap_or_default(), + ); + map.insert( + "version".to_string(), + stack_data + .version + .map_or_else(String::new, |v| v.to_string()), + ); + map.insert( + "status".to_string(), + stack_data + .status + .as_ref() + .map_or_else(String::new, std::string::ToString::to_string), + ); + map.insert( + "status_reason".to_string(), + stack_data.status_reason.unwrap_or_default(), + ); + + map +} + #[pyfunction(signature = (name, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] fn delete( name: &str, @@ -75,6 +110,38 @@ fn exists( }) } +#[pyfunction(signature = (vault_stack=None, region=None, bucket=None, profile=None))] +fn init( + vault_stack: Option, + region: Option, + bucket: Option, + profile: Option, +) -> PyResult> { + Runtime::new()?.block_on(async { + let result = Vault::init(vault_stack, region, bucket, profile) + .await + .map_err(vault_error_to_anyhow)?; + match result { + CreateStackResult::Exists { data } => Ok(to_hash_map(data, "exists".to_string())), + CreateStackResult::ExistsWithFailedState { data } => { + Ok(to_hash_map(data, "error".to_string())) + } + CreateStackResult::Created { + stack_name, + stack_id, + region, + } => { + let mut dict = HashMap::new(); + dict.insert("result".to_string(), "created".to_string()); + dict.insert("stack_name".to_string(), stack_name); + dict.insert("stack_id".to_string(), stack_id); + dict.insert("region".to_string(), region.to_string()); + Ok(dict) + } + } + }) +} + #[pyfunction(signature = (vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] fn list_all( vault_stack: Option, @@ -152,15 +219,52 @@ fn store( }) } +#[pyfunction(signature = (vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn update( + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult> { + Runtime::new()?.block_on(async { + let result = Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .update_stack() + .await + .map_err(vault_error_to_anyhow)?; + + match result { + UpdateStackResult::UpToDate { data } => Ok(to_hash_map(data, "up-to-date".to_string())), + UpdateStackResult::Updated { + stack_id, + previous_version, + new_version, + } => { + let mut map = HashMap::new(); + map.insert("result".to_string(), "updated".to_string()); + map.insert("stack_id".to_string(), stack_id); + map.insert("previous_version".to_string(), previous_version.to_string()); + map.insert("new_version".to_string(), new_version.to_string()); + Ok(map) + } + } + }) +} + #[pymodule] #[pyo3(name = "nitor_vault_rs")] fn nitor_vault_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(delete, m)?)?; m.add_function(wrap_pyfunction!(delete_many, m)?)?; m.add_function(wrap_pyfunction!(exists, m)?)?; + m.add_function(wrap_pyfunction!(init, m)?)?; m.add_function(wrap_pyfunction!(list_all, m)?)?; m.add_function(wrap_pyfunction!(lookup, m)?)?; m.add_function(wrap_pyfunction!(run, m)?)?; m.add_function(wrap_pyfunction!(store, m)?)?; + m.add_function(wrap_pyfunction!(update, m)?)?; Ok(()) } From 84982e553cb01b3877ad5470e69c59b2cebfb499 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 02:33:17 +0200 Subject: [PATCH 25/40] add stack status --- python-pyo3/python/n_vault/vault.py | 10 ++++++++++ python-pyo3/src/lib.rs | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 65f6a1b1..651a6da7 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -99,6 +99,16 @@ def lookup(self, name: str) -> str: profile=self.profile, ) + def stack_status(self) -> dict[str]: + return nitor_vault_rs.stack_status( + vault_stack=self.vault_stack, + region=self.region, + bucket=self.bucket, + key=self.key, + prefix=self.prefix, + profile=self.profile, + ) + def store(self, key: str, value: bytes | str) -> None: if isinstance(value, str): value = value.encode("utf-8") diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index ca4be788..a8e6d0f8 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -196,6 +196,27 @@ fn run(args: Vec) -> PyResult<()> { }) } +#[pyfunction(signature = (vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn stack_status( + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult> { + Runtime::new()?.block_on(async { + let data = Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .stack_status() + .await + .map_err(vault_error_to_anyhow)?; + + Ok(to_hash_map(data, "success".to_string())) + }) +} + #[pyfunction(signature = (name, value, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] fn store( name: &str, @@ -264,6 +285,7 @@ fn nitor_vault_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(list_all, m)?)?; m.add_function(wrap_pyfunction!(lookup, m)?)?; m.add_function(wrap_pyfunction!(run, m)?)?; + m.add_function(wrap_pyfunction!(stack_status, m)?)?; m.add_function(wrap_pyfunction!(store, m)?)?; m.add_function(wrap_pyfunction!(update, m)?)?; Ok(()) From 80698cd7a37c9a898f592534841e6f472cabe3a4 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 02:33:17 +0200 Subject: [PATCH 26/40] return python dict --- python-pyo3/python/n_vault/vault.py | 61 +++++++++-- python-pyo3/src/lib.rs | 158 +++++++++++++++------------- 2 files changed, 137 insertions(+), 82 deletions(-) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 651a6da7..b0411391 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -14,10 +14,39 @@ from collections.abc import Collection +from dataclasses import dataclass from n_vault import nitor_vault_rs +@dataclass +class CloudFormationStackData: + """Vault stack data from AWS CloudFormation describe stack.""" + + result: str + bucket: str | None + key: str | None + status: str | None + status_reason: str | None + version: int | None + + +@dataclass +class StackCreated: + result: str + stack_name: str | None + stack_id: str | None + region: str | None + + +@dataclass +class StackUpdated: + result: str + stack_id: str | None + previous_version: int | None + new_version: int | None + + class Vault: """Nitor Vault wrapper around the Rust vault library.""" @@ -70,13 +99,20 @@ def exists(self, name: str) -> bool: profile=self.profile, ) - def init(self) -> dict[str]: - return nitor_vault_rs.init( + def init(self) -> StackCreated | CloudFormationStackData: + result = nitor_vault_rs.init( vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, profile=self.profile, ) + result_status = result.get("result") + if result_status == "CREATED": + return StackCreated(**result) + elif result_status == "EXISTS" or result_status == "EXISTS_WITH_FAILED_STATE": + return CloudFormationStackData(**result) + + raise RuntimeError(f"Unexpected result data: {result}") def list_all(self) -> list[str]: return nitor_vault_rs.list_all( @@ -89,6 +125,11 @@ def list_all(self) -> list[str]: ) def lookup(self, name: str) -> str: + """ + Lookup value for given key name. + + Always returns a string, with binary data encoded in base64. + """ return nitor_vault_rs.lookup( name, vault_stack=self.vault_stack, @@ -99,8 +140,8 @@ def lookup(self, name: str) -> str: profile=self.profile, ) - def stack_status(self) -> dict[str]: - return nitor_vault_rs.stack_status( + def stack_status(self) -> CloudFormationStackData: + data = nitor_vault_rs.stack_status( vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, @@ -108,6 +149,7 @@ def stack_status(self) -> dict[str]: prefix=self.prefix, profile=self.profile, ) + return CloudFormationStackData(**data) def store(self, key: str, value: bytes | str) -> None: if isinstance(value, str): @@ -124,8 +166,8 @@ def store(self, key: str, value: bytes | str) -> None: profile=self.profile, ) - def update(self) -> dict[str]: - return nitor_vault_rs.update( + def update(self) -> StackUpdated | CloudFormationStackData: + result = nitor_vault_rs.update( vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, @@ -133,3 +175,10 @@ def update(self) -> dict[str]: prefix=self.prefix, profile=self.profile, ) + result_status = result.get("result") + if result_status == "UPDATED": + return StackUpdated(**result) + elif result_status == "UP_TO_DATE": + return CloudFormationStackData(**result) + + raise RuntimeError(f"Unexpected result data: {result}") diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index a8e6d0f8..14d9b2f7 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -1,9 +1,8 @@ -use std::collections::HashMap; - use nitor_vault::cloudformation::CloudFormationStackData; use nitor_vault::errors::VaultError; use nitor_vault::{CreateStackResult, UpdateStackResult, Value, Vault}; use pyo3::prelude::*; +use pyo3::types::{IntoPyDict, PyDict}; use tokio::runtime::Runtime; /// Convert `VaultError` to `anyhow::Error` @@ -11,36 +10,23 @@ fn vault_error_to_anyhow(err: VaultError) -> anyhow::Error { err.into() } -fn to_hash_map(stack_data: CloudFormationStackData, result: String) -> HashMap { - let mut map = HashMap::new(); - map.insert("result".to_string(), result); - map.insert( - "bucket_name".to_string(), - stack_data.bucket_name.clone().unwrap_or_default(), - ); - map.insert( - "key_arn".to_string(), - stack_data.key_arn.clone().unwrap_or_default(), - ); - map.insert( - "version".to_string(), - stack_data - .version - .map_or_else(String::new, |v| v.to_string()), - ); - map.insert( - "status".to_string(), - stack_data - .status - .as_ref() - .map_or_else(String::new, std::string::ToString::to_string), - ); - map.insert( - "status_reason".to_string(), - stack_data.status_reason.unwrap_or_default(), - ); - - map +fn stack_data_to_to_pydict<'a>( + py: Python<'a>, + data: CloudFormationStackData, + result: &'a str, +) -> Bound<'a, PyDict> { + let key_vals: Vec<(&str, PyObject)> = vec![ + ("result", result.to_string().to_object(py)), + ("bucket", data.bucket_name.to_object(py)), + ("key", data.key_arn.to_object(py)), + ( + "status", + data.status.map(|status| status.to_string()).to_object(py), + ), + ("status_reason", data.status_reason.to_object(py)), + ("version", data.version.to_object(py)), + ]; + key_vals.into_py_dict_bound(py) } #[pyfunction(signature = (name, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] @@ -116,28 +102,37 @@ fn init( region: Option, bucket: Option, profile: Option, -) -> PyResult> { - Runtime::new()?.block_on(async { - let result = Vault::init(vault_stack, region, bucket, profile) +) -> PyResult { + let result = Runtime::new()?.block_on(async { + Vault::init(vault_stack, region, bucket, profile) .await - .map_err(vault_error_to_anyhow)?; - match result { - CreateStackResult::Exists { data } => Ok(to_hash_map(data, "exists".to_string())), - CreateStackResult::ExistsWithFailedState { data } => { - Ok(to_hash_map(data, "error".to_string())) - } - CreateStackResult::Created { - stack_name, - stack_id, - region, - } => { - let mut dict = HashMap::new(); - dict.insert("result".to_string(), "created".to_string()); - dict.insert("stack_name".to_string(), stack_name); - dict.insert("stack_id".to_string(), stack_id); - dict.insert("region".to_string(), region.to_string()); - Ok(dict) - } + .map_err(vault_error_to_anyhow) + })?; + Python::with_gil(|py| match result { + CreateStackResult::Exists { data } => { + let dict = stack_data_to_to_pydict(py, data, "EXISTS"); + + Ok(dict.into()) + } + CreateStackResult::ExistsWithFailedState { data } => { + let dict = stack_data_to_to_pydict(py, data, "EXISTS_WITH_FAILED_STATE"); + + Ok(dict.into()) + } + CreateStackResult::Created { + stack_name, + stack_id, + region, + } => { + let key_vals: Vec<(&str, PyObject)> = vec![ + ("result", "CREATED".to_string().to_object(py)), + ("stack_name", stack_name.to_object(py)), + ("stack_id", stack_id.to_object(py)), + ("region", region.to_string().to_object(py)), + ]; + + let dict = key_vals.into_py_dict_bound(py); + Ok(dict.into()) } }) } @@ -204,16 +199,20 @@ fn stack_status( key: Option, prefix: Option, profile: Option, -) -> PyResult> { - Runtime::new()?.block_on(async { - let data = Vault::new(vault_stack, region, bucket, key, prefix, profile) +) -> PyResult { + let data = Runtime::new()?.block_on(async { + Vault::new(vault_stack, region, bucket, key, prefix, profile) .await .map_err(vault_error_to_anyhow)? .stack_status() .await - .map_err(vault_error_to_anyhow)?; + .map_err(vault_error_to_anyhow) + })?; + + Python::with_gil(|py| { + let dict = stack_data_to_to_pydict(py, data, "SUCCESS"); - Ok(to_hash_map(data, "success".to_string())) + Ok(dict.into()) }) } @@ -248,29 +247,36 @@ fn update( key: Option, prefix: Option, profile: Option, -) -> PyResult> { - Runtime::new()?.block_on(async { - let result = Vault::new(vault_stack, region, bucket, key, prefix, profile) +) -> PyResult { + let result = Runtime::new()?.block_on(async { + Vault::new(vault_stack, region, bucket, key, prefix, profile) .await .map_err(vault_error_to_anyhow)? .update_stack() .await - .map_err(vault_error_to_anyhow)?; + .map_err(vault_error_to_anyhow) + })?; + + Python::with_gil(|py| match result { + UpdateStackResult::UpToDate { data } => { + let dict = stack_data_to_to_pydict(py, data, "UP_TO_DATE"); + + Ok(dict.into()) + } + UpdateStackResult::Updated { + stack_id, + previous_version, + new_version, + } => { + let key_vals: Vec<(&str, PyObject)> = vec![ + ("result", "UPDATED".to_string().to_object(py)), + ("stack_id", stack_id.to_object(py)), + ("previous_version", previous_version.to_object(py)), + ("new_version", new_version.to_object(py)), + ]; - match result { - UpdateStackResult::UpToDate { data } => Ok(to_hash_map(data, "up-to-date".to_string())), - UpdateStackResult::Updated { - stack_id, - previous_version, - new_version, - } => { - let mut map = HashMap::new(); - map.insert("result".to_string(), "updated".to_string()); - map.insert("stack_id".to_string(), stack_id); - map.insert("previous_version".to_string(), previous_version.to_string()); - map.insert("new_version".to_string(), new_version.to_string()); - Ok(map) - } + let dict = key_vals.into_py_dict_bound(py); + Ok(dict.into()) } }) } From 892928ef527f746d015cfeed0deb9f5e6bcf8640 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 13:42:34 +0200 Subject: [PATCH 27/40] update docstrings for library functions --- python-pyo3/python/n_vault/vault.py | 28 ++++++++++++++++++++++++++++ rust/src/vault.rs | 8 ++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index b0411391..409f507a 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -89,6 +89,11 @@ def delete_many(self, names: Collection[str]) -> None: ) def exists(self, name: str) -> bool: + """ + Check if the given key name already exists in the S3 bucket. + + Returns True if the key exists, False otherwise. + """ return nitor_vault_rs.exists( name, vault_stack=self.vault_stack, @@ -100,6 +105,15 @@ def exists(self, name: str) -> bool: ) def init(self) -> StackCreated | CloudFormationStackData: + """ + Initialize new Vault stack. + + This will create all required resources in AWS, + after which the Vault can be used to store and lookup values. + + Returns a `StackCreated` if a new vault stack was initialized, + or `CloudFormationStackData` if it already exists. + """ result = nitor_vault_rs.init( vault_stack=self.vault_stack, region=self.region, @@ -115,6 +129,11 @@ def init(self) -> StackCreated | CloudFormationStackData: raise RuntimeError(f"Unexpected result data: {result}") def list_all(self) -> list[str]: + """ + Get all available secrets. + + Returns a list of key names. + """ return nitor_vault_rs.list_all( vault_stack=self.vault_stack, region=self.region, @@ -152,6 +171,9 @@ def stack_status(self) -> CloudFormationStackData: return CloudFormationStackData(**data) def store(self, key: str, value: bytes | str) -> None: + """ + Store encrypted value with given key name in S3. + """ if isinstance(value, str): value = value.encode("utf-8") @@ -167,6 +189,12 @@ def store(self, key: str, value: bytes | str) -> None: ) def update(self) -> StackUpdated | CloudFormationStackData: + """ + Update the vault Cloudformation stack with the current template. + + Returns `StackUpdated` if the vault stack was updated to a new version, + or `CloudFormationStackData` if it is already up to date. + """ result = nitor_vault_rs.update( vault_stack=self.vault_stack, region=self.region, diff --git a/rust/src/vault.rs b/rust/src/vault.rs index b7bc4c74..570bec64 100644 --- a/rust/src/vault.rs +++ b/rust/src/vault.rs @@ -228,6 +228,8 @@ impl Vault { } /// Get all available secrets. + /// + /// Returns a list of key names. pub async fn all(&self) -> Result, VaultError> { let output = self .s3 @@ -258,7 +260,9 @@ impl Vault { self.cloudformation_params.clone() } - /// Check if key already exists in bucket. + /// Check if the given key name already exists in the S3 bucket. + /// + /// Returns `true` if the key exists, `false` otherwise. pub async fn exists(&self, name: &str) -> Result { let name = self.full_key_name(name); match self @@ -283,7 +287,7 @@ impl Vault { } } - /// Store encrypted data in S3. + /// Store encrypted data with given key name in S3 pub async fn store(&self, name: &str, data: &[u8]) -> Result<(), VaultError> { let encrypted = self.encrypt(data).await?; From 0fe5e2ff5a12b9207d219f07e6ee84c78b99a8c6 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 15:41:24 +0200 Subject: [PATCH 28/40] docstrings for all public library methods --- python-pyo3/python/n_vault/vault.py | 19 ++++++++++++++++++- python-pyo3/src/lib.rs | 10 ++++------ rust/src/vault.rs | 4 ++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 409f507a..8c5f0f29 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -33,6 +33,8 @@ class CloudFormationStackData: @dataclass class StackCreated: + """Result data for vault init.""" + result: str stack_name: str | None stack_id: str | None @@ -41,6 +43,8 @@ class StackCreated: @dataclass class StackUpdated: + """Result data for vault update.""" + result: str stack_id: str | None previous_version: int | None @@ -48,7 +52,9 @@ class StackUpdated: class Vault: - """Nitor Vault wrapper around the Rust vault library.""" + """ + Nitor Vault wrapper around the Rust vault library. + """ def __init__( self, @@ -67,6 +73,9 @@ def __init__( self.profile = profile def delete(self, name: str) -> None: + """ + Delete data in S3 for given key name. + """ return nitor_vault_rs.delete( name, vault_stack=self.vault_stack, @@ -78,6 +87,11 @@ def delete(self, name: str) -> None: ) def delete_many(self, names: Collection[str]) -> None: + """ + Delete data for multiple keys. + + Takes in a collection of key name strings, such as a `list`, `tuple`, or `set`. + """ return nitor_vault_rs.delete_many( sorted(names), vault_stack=self.vault_stack, @@ -160,6 +174,9 @@ def lookup(self, name: str) -> str: ) def stack_status(self) -> CloudFormationStackData: + """ + Get vault Cloudformation stack status. + """ data = nitor_vault_rs.stack_status( vault_stack=self.vault_stack, region=self.region, diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index 14d9b2f7..3e6cbe72 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -10,6 +10,9 @@ fn vault_error_to_anyhow(err: VaultError) -> anyhow::Error { err.into() } +/// Convert `CloudFormationStackData` to a Python dictionary. +// Lifetime annotations are required due to `&str` usage, +// could be left out if passing a `String` for result message. fn stack_data_to_to_pydict<'a>( py: Python<'a>, data: CloudFormationStackData, @@ -111,12 +114,10 @@ fn init( Python::with_gil(|py| match result { CreateStackResult::Exists { data } => { let dict = stack_data_to_to_pydict(py, data, "EXISTS"); - Ok(dict.into()) } CreateStackResult::ExistsWithFailedState { data } => { let dict = stack_data_to_to_pydict(py, data, "EXISTS_WITH_FAILED_STATE"); - Ok(dict.into()) } CreateStackResult::Created { @@ -130,7 +131,6 @@ fn init( ("stack_id", stack_id.to_object(py)), ("region", region.to_string().to_object(py)), ]; - let dict = key_vals.into_py_dict_bound(py); Ok(dict.into()) } @@ -178,6 +178,7 @@ fn lookup( .await .map_err(vault_error_to_anyhow)?; + // Binary data will get base64 encoded in the Display trait implementation Ok(result.to_string()) }) } @@ -211,7 +212,6 @@ fn stack_status( Python::with_gil(|py| { let dict = stack_data_to_to_pydict(py, data, "SUCCESS"); - Ok(dict.into()) }) } @@ -260,7 +260,6 @@ fn update( Python::with_gil(|py| match result { UpdateStackResult::UpToDate { data } => { let dict = stack_data_to_to_pydict(py, data, "UP_TO_DATE"); - Ok(dict.into()) } UpdateStackResult::Updated { @@ -274,7 +273,6 @@ fn update( ("previous_version", previous_version.to_object(py)), ("new_version", new_version.to_object(py)), ]; - let dict = key_vals.into_py_dict_bound(py); Ok(dict.into()) } diff --git a/rust/src/vault.rs b/rust/src/vault.rs index 570bec64..2c480da0 100644 --- a/rust/src/vault.rs +++ b/rust/src/vault.rs @@ -222,7 +222,7 @@ impl Vault { } } - /// Get Cloudformation stack status. + /// Get Cloudformation vault stack status. pub async fn stack_status(&self) -> Result { cloudformation::get_stack_data(&self.cf, &self.cloudformation_params.stack_name).await } @@ -304,7 +304,7 @@ impl Vault { Ok(()) } - /// Delete data in S3 for given key. + /// Delete data in S3 for given key name. pub async fn delete(&self, name: &str) -> Result<(), VaultError> { if !self.exists(name).await? { return Err(VaultError::S3DeleteObjectKeyMissingError { From a92879c3572dae9d5bfbef4c0d88a0065f21acfe Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 15:53:29 +0200 Subject: [PATCH 29/40] match parameter names to old python library --- python-pyo3/python/n_vault/vault.py | 37 +++++++++++++++++------------ python/n_vault/vault.py | 2 ++ 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 8c5f0f29..c9382320 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os from collections.abc import Collection from dataclasses import dataclass @@ -61,17 +62,23 @@ def __init__( vault_stack: str = None, region: str = None, bucket: str = None, - key: str = None, + vault_key: str = None, prefix: str = None, profile: str = None, ): self.vault_stack = vault_stack self.region = region self.bucket = bucket - self.key = key + self.vault_key = vault_key self.prefix = prefix self.profile = profile + def all(self) -> str: + """ + Return a string with all keys separated by os.linesep. + """ + return os.linesep.join(item for item in self.list_all()) + def delete(self, name: str) -> None: """ Delete data in S3 for given key name. @@ -81,7 +88,7 @@ def delete(self, name: str) -> None: vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, - key=self.key, + key=self.vault_key, prefix=self.prefix, profile=self.profile, ) @@ -97,7 +104,7 @@ def delete_many(self, names: Collection[str]) -> None: vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, - key=self.key, + key=self.vault_key, prefix=self.prefix, profile=self.profile, ) @@ -113,7 +120,7 @@ def exists(self, name: str) -> bool: vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, - key=self.key, + key=self.vault_key, prefix=self.prefix, profile=self.profile, ) @@ -152,7 +159,7 @@ def list_all(self) -> list[str]: vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, - key=self.key, + key=self.vault_key, prefix=self.prefix, profile=self.profile, ) @@ -168,7 +175,7 @@ def lookup(self, name: str) -> str: vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, - key=self.key, + key=self.vault_key, prefix=self.prefix, profile=self.profile, ) @@ -181,26 +188,26 @@ def stack_status(self) -> CloudFormationStackData: vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, - key=self.key, + key=self.vault_key, prefix=self.prefix, profile=self.profile, ) return CloudFormationStackData(**data) - def store(self, key: str, value: bytes | str) -> None: + def store(self, name: str, data: bytes | str) -> None: """ Store encrypted value with given key name in S3. """ - if isinstance(value, str): - value = value.encode("utf-8") + if isinstance(name, str): + name = name.encode("utf-8") return nitor_vault_rs.store( - key, - value, + name, + data, vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, - key=self.key, + key=self.vault_key, prefix=self.prefix, profile=self.profile, ) @@ -216,7 +223,7 @@ def update(self) -> StackUpdated | CloudFormationStackData: vault_stack=self.vault_stack, region=self.region, bucket=self.bucket, - key=self.key, + key=self.vault_key, prefix=self.prefix, profile=self.profile, ) diff --git a/python/n_vault/vault.py b/python/n_vault/vault.py index 81fbc90f..675ac367 100644 --- a/python/n_vault/vault.py +++ b/python/n_vault/vault.py @@ -207,6 +207,7 @@ def all(self): ret = "" for item in self.list_all(): ret = ret + item + os.linesep + return ret def list_all(self): @@ -216,6 +217,7 @@ def list_all(self): ret.append(next_object.key[:-17]) elif next_object.key.endswith(".encrypted") and next_object.key[:-10] not in ret: ret.append(next_object.key[:-10]) + return ret def get_key(self): From 7014b9d7739180ced91127b144770eca35ae329c Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 16:02:03 +0200 Subject: [PATCH 30/40] fix variable rename --- python-pyo3/python/n_vault/vault.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index c9382320..38a021ab 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -198,8 +198,8 @@ def store(self, name: str, data: bytes | str) -> None: """ Store encrypted value with given key name in S3. """ - if isinstance(name, str): - name = name.encode("utf-8") + if isinstance(data, str): + data = data.encode("utf-8") return nitor_vault_rs.store( name, From af6625d1f73e947d75a289ef4bbf7153f9e4edba Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 16:40:58 +0200 Subject: [PATCH 31/40] make type hinting compatible with python 3.9 --- python-pyo3/pyproject.toml | 2 +- python-pyo3/python/n_vault/vault.py | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/python-pyo3/pyproject.toml b/python-pyo3/pyproject.toml index 21e0a303..c2508a90 100644 --- a/python-pyo3/pyproject.toml +++ b/python-pyo3/pyproject.toml @@ -60,7 +60,7 @@ venv = ".venv" [tool.ruff] # https://docs.astral.sh/ruff/configuration/ include = ["*.py", "*.pyi", "**/pyproject.toml"] -target-version = "py311" +target-version = "py39" line-length = 120 [tool.ruff.lint] diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 38a021ab..2f4aeb37 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -16,6 +16,7 @@ from collections.abc import Collection from dataclasses import dataclass +from typing import Optional from n_vault import nitor_vault_rs @@ -25,11 +26,11 @@ class CloudFormationStackData: """Vault stack data from AWS CloudFormation describe stack.""" result: str - bucket: str | None - key: str | None - status: str | None - status_reason: str | None - version: int | None + bucket: Optional[str] + key: Optional[str] + status: Optional[str] + status_reason: Optional[str] + version: Optional[int] @dataclass @@ -37,9 +38,9 @@ class StackCreated: """Result data for vault init.""" result: str - stack_name: str | None - stack_id: str | None - region: str | None + stack_name: Optional[str] + stack_id: Optional[str] + region: Optional[str] @dataclass @@ -47,9 +48,9 @@ class StackUpdated: """Result data for vault update.""" result: str - stack_id: str | None - previous_version: int | None - new_version: int | None + stack_id: Optional[str] + previous_version: Optional[int] + new_version: Optional[int] class Vault: From 8079b2bdfcf31ee959e75f0b2a6f708cf9af8c8d Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 16:41:26 +0200 Subject: [PATCH 32/40] fix help output when no arguments are given --- rust/src/args.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/rust/src/args.rs b/rust/src/args.rs index d38c9199..6b7abba6 100644 --- a/rust/src/args.rs +++ b/rust/src/args.rs @@ -262,8 +262,17 @@ enum Command { }, } -/// Run Vault CLI with the given arguments -pub async fn run_cli_with_args(args: Vec) -> Result<()> { +/// Run Vault CLI with the given arguments. +/// +/// The argument list needs to include the binary name as the first element. +pub async fn run_cli_with_args(mut args: Vec) -> Result<()> { + // If args are empty, need to manually trigger the help output. + // `parse_from` does not do it automatically unlike `parse`. + if args.is_empty() { + args = vec!["vault".to_string(), "-h".to_string()]; + } else if args.len() == 1 { + args.push("-h".to_string()); + } let args = Args::parse_from(args); let quiet = args.quiet; @@ -279,7 +288,7 @@ pub async fn run_cli_with_args(args: Vec) -> Result<()> { Ok(()) } -/// Run Vault CLI +/// Run Vault CLI. pub async fn run_cli() -> Result<()> { let args = Args::parse(); let quiet = args.quiet; From bc045d86687d1002210ce4e84be8bfbc6c8b3772 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 17:13:46 +0200 Subject: [PATCH 33/40] rename and reorder vault class parameters to match previous version --- python-pyo3/python/n_vault/vault.py | 70 +++++++++++++++-------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 2f4aeb37..7d5e50b4 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -55,23 +55,27 @@ class StackUpdated: class Vault: """ - Nitor Vault wrapper around the Rust vault library. + Nitor Vault Python wrapper class around the Rust vault library. + + Note that initializing this class only saves the optional parameters, + but does *not* construct an actual vault instance. + Each method in this class creates its own Vault instance internally in the Rust library. """ def __init__( self, vault_stack: str = None, - region: str = None, - bucket: str = None, vault_key: str = None, - prefix: str = None, + vault_bucket: str = None, + vault_prefix: str = None, + vault_region: str = None, profile: str = None, ): self.vault_stack = vault_stack - self.region = region - self.bucket = bucket self.vault_key = vault_key - self.prefix = prefix + self.vault_bucket = vault_bucket + self.vault_prefix = vault_prefix + self.vault_region = vault_region self.profile = profile def all(self) -> str: @@ -87,10 +91,10 @@ def delete(self, name: str) -> None: return nitor_vault_rs.delete( name, vault_stack=self.vault_stack, - region=self.region, - bucket=self.bucket, + region=self.vault_region, + bucket=self.vault_bucket, key=self.vault_key, - prefix=self.prefix, + prefix=self.vault_prefix, profile=self.profile, ) @@ -103,10 +107,10 @@ def delete_many(self, names: Collection[str]) -> None: return nitor_vault_rs.delete_many( sorted(names), vault_stack=self.vault_stack, - region=self.region, - bucket=self.bucket, + region=self.vault_region, + bucket=self.vault_bucket, key=self.vault_key, - prefix=self.prefix, + prefix=self.vault_prefix, profile=self.profile, ) @@ -119,10 +123,10 @@ def exists(self, name: str) -> bool: return nitor_vault_rs.exists( name, vault_stack=self.vault_stack, - region=self.region, - bucket=self.bucket, + region=self.vault_region, + bucket=self.vault_bucket, key=self.vault_key, - prefix=self.prefix, + prefix=self.vault_prefix, profile=self.profile, ) @@ -138,8 +142,8 @@ def init(self) -> StackCreated | CloudFormationStackData: """ result = nitor_vault_rs.init( vault_stack=self.vault_stack, - region=self.region, - bucket=self.bucket, + region=self.vault_region, + bucket=self.vault_bucket, profile=self.profile, ) result_status = result.get("result") @@ -158,10 +162,10 @@ def list_all(self) -> list[str]: """ return nitor_vault_rs.list_all( vault_stack=self.vault_stack, - region=self.region, - bucket=self.bucket, + region=self.vault_region, + bucket=self.vault_bucket, key=self.vault_key, - prefix=self.prefix, + prefix=self.vault_prefix, profile=self.profile, ) @@ -174,10 +178,10 @@ def lookup(self, name: str) -> str: return nitor_vault_rs.lookup( name, vault_stack=self.vault_stack, - region=self.region, - bucket=self.bucket, + region=self.vault_region, + bucket=self.vault_bucket, key=self.vault_key, - prefix=self.prefix, + prefix=self.vault_prefix, profile=self.profile, ) @@ -187,10 +191,10 @@ def stack_status(self) -> CloudFormationStackData: """ data = nitor_vault_rs.stack_status( vault_stack=self.vault_stack, - region=self.region, - bucket=self.bucket, + region=self.vault_region, + bucket=self.vault_bucket, key=self.vault_key, - prefix=self.prefix, + prefix=self.vault_prefix, profile=self.profile, ) return CloudFormationStackData(**data) @@ -206,10 +210,10 @@ def store(self, name: str, data: bytes | str) -> None: name, data, vault_stack=self.vault_stack, - region=self.region, - bucket=self.bucket, + region=self.vault_region, + bucket=self.vault_bucket, key=self.vault_key, - prefix=self.prefix, + prefix=self.vault_prefix, profile=self.profile, ) @@ -222,10 +226,10 @@ def update(self) -> StackUpdated | CloudFormationStackData: """ result = nitor_vault_rs.update( vault_stack=self.vault_stack, - region=self.region, - bucket=self.bucket, + region=self.vault_region, + bucket=self.vault_bucket, key=self.vault_key, - prefix=self.prefix, + prefix=self.vault_prefix, profile=self.profile, ) result_status = result.get("result") From a1143fc2aecd161caceac4d32b5764718b801609 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 22:28:00 +0200 Subject: [PATCH 34/40] add direct encrypt and decrypt --- python-pyo3/python/n_vault/vault.py | 31 +++++++++++++++++++ python-pyo3/src/lib.rs | 46 +++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 7d5e50b4..6a3ce09b 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -114,6 +114,37 @@ def delete_many(self, names: Collection[str]) -> None: profile=self.profile, ) + def direct_decrypt(self, data: bytes) -> bytes: + """ + Decrypt data with KMS. + """ + return nitor_vault_rs.direct_decrypt( + data, + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + + def direct_encrypt(self, data: bytes | str) -> bytes: + """ + Encrypt data with KMS. + """ + if isinstance(data, str): + data = data.encode("utf-8") + + return nitor_vault_rs.direct_encrypt( + data, + vault_stack=self.vault_stack, + region=self.vault_region, + bucket=self.vault_bucket, + key=self.vault_key, + prefix=self.vault_prefix, + profile=self.profile, + ) + def exists(self, name: str) -> bool: """ Check if the given key name already exists in the S3 bucket. diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index 3e6cbe72..3e6a121e 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -77,6 +77,50 @@ fn delete_many( }) } +#[pyfunction(signature = (data, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn direct_decrypt( + data: &[u8], + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult> { + Runtime::new()?.block_on(async { + let result = Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .direct_decrypt(data) + .await + .map_err(vault_error_to_anyhow)?; + + Ok(result) + }) +} + +#[pyfunction(signature = (data, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] +fn direct_encrypt( + data: &[u8], + vault_stack: Option, + region: Option, + bucket: Option, + key: Option, + prefix: Option, + profile: Option, +) -> PyResult> { + Runtime::new()?.block_on(async { + let result = Vault::new(vault_stack, region, bucket, key, prefix, profile) + .await + .map_err(vault_error_to_anyhow)? + .direct_encrypt(data) + .await + .map_err(vault_error_to_anyhow)?; + + Ok(result) + }) +} + #[pyfunction(signature = (name, vault_stack=None, region=None, bucket=None, key=None, prefix=None, profile=None))] fn exists( name: &str, @@ -284,6 +328,8 @@ fn update( fn nitor_vault_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(delete, m)?)?; m.add_function(wrap_pyfunction!(delete_many, m)?)?; + m.add_function(wrap_pyfunction!(direct_decrypt, m)?)?; + m.add_function(wrap_pyfunction!(direct_encrypt, m)?)?; m.add_function(wrap_pyfunction!(exists, m)?)?; m.add_function(wrap_pyfunction!(init, m)?)?; m.add_function(wrap_pyfunction!(list_all, m)?)?; From d638ddaf7494450828ba7b9987cfccb55c952b00 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 22:45:42 +0200 Subject: [PATCH 35/40] return bytes from direct encrypt and decrypt --- python-pyo3/src/lib.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index 3e6a121e..21a24cdd 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -1,10 +1,13 @@ -use nitor_vault::cloudformation::CloudFormationStackData; -use nitor_vault::errors::VaultError; -use nitor_vault::{CreateStackResult, UpdateStackResult, Value, Vault}; +use std::borrow::Cow; + use pyo3::prelude::*; use pyo3::types::{IntoPyDict, PyDict}; use tokio::runtime::Runtime; +use nitor_vault::cloudformation::CloudFormationStackData; +use nitor_vault::errors::VaultError; +use nitor_vault::{CreateStackResult, UpdateStackResult, Value, Vault}; + /// Convert `VaultError` to `anyhow::Error` fn vault_error_to_anyhow(err: VaultError) -> anyhow::Error { err.into() @@ -86,7 +89,9 @@ fn direct_decrypt( key: Option, prefix: Option, profile: Option, -) -> PyResult> { +) -> PyResult> { + // Returns Cow<[u8]> instead of Vec since that will get mapped to bytes for the Python side + // https://pyo3.rs/main/conversions/tables#returning-rust-values-to-python Runtime::new()?.block_on(async { let result = Vault::new(vault_stack, region, bucket, key, prefix, profile) .await @@ -95,7 +100,7 @@ fn direct_decrypt( .await .map_err(vault_error_to_anyhow)?; - Ok(result) + Ok(result.into()) }) } @@ -108,7 +113,7 @@ fn direct_encrypt( key: Option, prefix: Option, profile: Option, -) -> PyResult> { +) -> PyResult> { Runtime::new()?.block_on(async { let result = Vault::new(vault_stack, region, bucket, key, prefix, profile) .await @@ -117,7 +122,7 @@ fn direct_encrypt( .await .map_err(vault_error_to_anyhow)?; - Ok(result) + Ok(result.into()) }) } From 8dfea46648ac66c4fdce4189c7c18fcba179f8c0 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 22:55:19 +0200 Subject: [PATCH 36/40] fix type hints for 3.9 :( --- python-pyo3/pyproject.toml | 4 ++-- python-pyo3/python/n_vault/vault.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python-pyo3/pyproject.toml b/python-pyo3/pyproject.toml index c2508a90..e0ff9397 100644 --- a/python-pyo3/pyproject.toml +++ b/python-pyo3/pyproject.toml @@ -30,8 +30,8 @@ classifiers = [ dependencies = [] [project.optional-dependencies] -build = ["maturin", "twine", "wheel"] -dev = ["ruff"] +build = ["maturin", "wheel"] +dev = ["maturin", "ruff"] [project.urls] Repository = "https://github.com/NitorCreations/vault" diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index 6a3ce09b..f2cad208 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -16,7 +16,7 @@ from collections.abc import Collection from dataclasses import dataclass -from typing import Optional +from typing import Optional, Union from n_vault import nitor_vault_rs @@ -128,7 +128,7 @@ def direct_decrypt(self, data: bytes) -> bytes: profile=self.profile, ) - def direct_encrypt(self, data: bytes | str) -> bytes: + def direct_encrypt(self, data: Union[bytes, str]) -> bytes: """ Encrypt data with KMS. """ @@ -161,7 +161,7 @@ def exists(self, name: str) -> bool: profile=self.profile, ) - def init(self) -> StackCreated | CloudFormationStackData: + def init(self) -> Union[StackCreated, CloudFormationStackData]: """ Initialize new Vault stack. @@ -230,7 +230,7 @@ def stack_status(self) -> CloudFormationStackData: ) return CloudFormationStackData(**data) - def store(self, name: str, data: bytes | str) -> None: + def store(self, name: str, data: Union[bytes, str]) -> None: """ Store encrypted value with given key name in S3. """ @@ -248,7 +248,7 @@ def store(self, name: str, data: bytes | str) -> None: profile=self.profile, ) - def update(self) -> StackUpdated | CloudFormationStackData: + def update(self) -> Union[StackUpdated, CloudFormationStackData]: """ Update the vault Cloudformation stack with the current template. From a903504d3142f060d8cd21c529114ab123ead013 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 22:55:25 +0200 Subject: [PATCH 37/40] ignore all venvs --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ddb1c942..18b61beb 100644 --- a/.gitignore +++ b/.gitignore @@ -79,7 +79,8 @@ celerybeat-schedule .env # virtualenv -venv/ +venv*/ +.venv*/ ENV/ # Spyder project settings @@ -112,4 +113,3 @@ dependency-reduced-pom.xml # Go binary /go/vault /go/vault -.venv From 91e94ed62d43fc0218c6597d6d9fe84bd5fd0892 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 23:13:49 +0200 Subject: [PATCH 38/40] match parameter name for direct decrypt --- python-pyo3/python/n_vault/vault.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-pyo3/python/n_vault/vault.py b/python-pyo3/python/n_vault/vault.py index f2cad208..51e5dad4 100644 --- a/python-pyo3/python/n_vault/vault.py +++ b/python-pyo3/python/n_vault/vault.py @@ -114,12 +114,12 @@ def delete_many(self, names: Collection[str]) -> None: profile=self.profile, ) - def direct_decrypt(self, data: bytes) -> bytes: + def direct_decrypt(self, encrypted_data: bytes) -> bytes: """ Decrypt data with KMS. """ return nitor_vault_rs.direct_decrypt( - data, + encrypted_data, vault_stack=self.vault_stack, region=self.vault_region, bucket=self.vault_bucket, From b287d34d4c97a079a5263a929a73cb9ed0181fb3 Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Wed, 13 Nov 2024 23:14:14 +0200 Subject: [PATCH 39/40] cargo update --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d450614e..f2977db6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -652,9 +652,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", "clap_derive", @@ -662,9 +662,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" dependencies = [ "anstream", "anstyle", @@ -674,9 +674,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.37" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11611dca53440593f38e6b25ec629de50b14cdfa63adc0fb856115a2c6d97595" +checksum = "d9647a559c112175f17cf724dc72d3645680a883c58481332779192b0d8e7a01" dependencies = [ "clap", ] @@ -695,9 +695,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" [[package]] name = "colorchoice" From 5f2996a0a8575ec7d7ce5e836bab1b71d810746f Mon Sep 17 00:00:00 2001 From: Akseli Lukkarila Date: Thu, 14 Nov 2024 11:20:02 +0200 Subject: [PATCH 40/40] fix typo in helper function name --- python-pyo3/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python-pyo3/src/lib.rs b/python-pyo3/src/lib.rs index 21a24cdd..82c34262 100644 --- a/python-pyo3/src/lib.rs +++ b/python-pyo3/src/lib.rs @@ -15,8 +15,8 @@ fn vault_error_to_anyhow(err: VaultError) -> anyhow::Error { /// Convert `CloudFormationStackData` to a Python dictionary. // Lifetime annotations are required due to `&str` usage, -// could be left out if passing a `String` for result message. -fn stack_data_to_to_pydict<'a>( +// could be left out if passing a `String` for the result message. +fn stack_data_to_pydict<'a>( py: Python<'a>, data: CloudFormationStackData, result: &'a str, @@ -162,11 +162,11 @@ fn init( })?; Python::with_gil(|py| match result { CreateStackResult::Exists { data } => { - let dict = stack_data_to_to_pydict(py, data, "EXISTS"); + let dict = stack_data_to_pydict(py, data, "EXISTS"); Ok(dict.into()) } CreateStackResult::ExistsWithFailedState { data } => { - let dict = stack_data_to_to_pydict(py, data, "EXISTS_WITH_FAILED_STATE"); + let dict = stack_data_to_pydict(py, data, "EXISTS_WITH_FAILED_STATE"); Ok(dict.into()) } CreateStackResult::Created { @@ -260,7 +260,7 @@ fn stack_status( })?; Python::with_gil(|py| { - let dict = stack_data_to_to_pydict(py, data, "SUCCESS"); + let dict = stack_data_to_pydict(py, data, "SUCCESS"); Ok(dict.into()) }) } @@ -308,7 +308,7 @@ fn update( Python::with_gil(|py| match result { UpdateStackResult::UpToDate { data } => { - let dict = stack_data_to_to_pydict(py, data, "UP_TO_DATE"); + let dict = stack_data_to_pydict(py, data, "UP_TO_DATE"); Ok(dict.into()) } UpdateStackResult::Updated {