diff --git a/.cookiecutter.json b/.cookiecutter.json index 8e969ef..5067585 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -21,7 +21,7 @@ "_drift_manager": { "template": "https://github.com/nautobot/cookiecutter-nautobot-app.git", "template_dir": "nautobot-app", - "template_ref": "refs/tags/nautobot-app-v2.1.0", + "template_ref": "refs/tags/nautobot-app-v2.2.0", "cookie_dir": "", "branch_prefix": "drift-manager", "pull_request_strategy": "create", @@ -29,7 +29,7 @@ "black" ], "draft": true, - "baked_commit_ref": "19d14aa7031336b5872b82806f71d7b1856d4fe1" + "baked_commit_ref": "07ab5a6da72c934c43f1f957ce073bedbfdbfc9a" } } } diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md index 688ff03..6881e27 100644 --- a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -31,5 +31,4 @@ - [ ] Attached Screenshots, Payload Example - [ ] Unit, Integration Tests - [ ] Documentation Updates (when adding/changing features) -- [ ] Example App Updates (when adding/changing features) - [ ] Outline Remaining Work, Constraints from Design diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 234771c..ed9d230 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" + uses: "networktocode/gh-action-setup-poetry-environment@v6" - name: "Linting: black" run: "poetry run invoke black" bandit: @@ -36,7 +36,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" + uses: "networktocode/gh-action-setup-poetry-environment@v6" - name: "Linting: bandit" run: "poetry run invoke bandit" ruff: @@ -47,7 +47,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" + uses: "networktocode/gh-action-setup-poetry-environment@v6" - name: "Linting: ruff" run: "poetry run invoke ruff" check-docs-build: @@ -58,7 +58,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" + uses: "networktocode/gh-action-setup-poetry-environment@v6" - name: "Check Docs Build" run: "poetry run invoke build-and-check-docs" flake8: @@ -69,7 +69,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" + uses: "networktocode/gh-action-setup-poetry-environment@v6" - name: "Linting: flake8" run: "poetry run invoke flake8" poetry: @@ -80,7 +80,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" + uses: "networktocode/gh-action-setup-poetry-environment@v6" - name: "Checking: poetry lock file" run: "poetry run invoke lock --check" yamllint: @@ -91,10 +91,10 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" + uses: "networktocode/gh-action-setup-poetry-environment@v6" - name: "Linting: yamllint" run: "poetry run invoke yamllint" - pylint: + check-in-docker: needs: - "black" - "bandit" @@ -116,7 +116,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" + uses: "networktocode/gh-action-setup-poetry-environment@v6" - name: "Set up Docker Buildx" id: "buildx" uses: "docker/setup-buildx-action@v3" @@ -138,53 +138,13 @@ jobs: run: "cp development/creds.example.env development/creds.env" - name: "Linting: pylint" run: "poetry run invoke pylint" - check-migrations: - needs: - - "bandit" - - "ruff" - - "flake8" - - "poetry" - - "yamllint" - - "black" - runs-on: "ubuntu-22.04" - strategy: - fail-fast: true - matrix: - python-version: ["3.11"] - nautobot-version: ["2.0.0"] - env: - INVOKE_NAUTOBOT_SECRETS_PROVIDERS_PYTHON_VER: "${{ matrix.python-version }}" - INVOKE_NAUTOBOT_SECRETS_PROVIDERS_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" - steps: - - name: "Check out repository code" - uses: "actions/checkout@v4" - - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" - - name: "Set up Docker Buildx" - id: "buildx" - uses: "docker/setup-buildx-action@v3" - - name: "Build" - uses: "docker/build-push-action@v5" - with: - builder: "${{ steps.buildx.outputs.name }}" - context: "./" - push: false - load: true - tags: "${{ env.APP_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" - file: "./development/Dockerfile" - cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" - cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" - build-args: | - NAUTOBOT_VER=${{ matrix.nautobot-version }} - PYTHON_VER=${{ matrix.python-version }} - - name: "Copy credentials" - run: "cp development/creds.example.env development/creds.env" + - name: "Checking: App Config" + run: "poetry run invoke validate-app-config" - name: "Checking: migrations" run: "poetry run invoke check-migrations" unittest: needs: - - "pylint" - - "check-migrations" + - "check-in-docker" strategy: fail-fast: true matrix: @@ -206,7 +166,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" + uses: "networktocode/gh-action-setup-poetry-environment@v6" - name: "Set up Docker Buildx" id: "buildx" uses: "docker/setup-buildx-action@v3" @@ -242,7 +202,7 @@ jobs: with: fetch-depth: "0" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" + uses: "networktocode/gh-action-setup-poetry-environment@v6" - name: "Check for changelog entry" run: | git fetch --no-tags origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }} @@ -259,7 +219,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Set up Python" - uses: "actions/setup-python@v4" + uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install Python Packages" @@ -294,7 +254,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Set up Python" - uses: "actions/setup-python@v4" + uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install Python Packages" diff --git a/FAQ.md b/FAQ.md deleted file mode 100644 index 318b08d..0000000 --- a/FAQ.md +++ /dev/null @@ -1 +0,0 @@ -# Frequently Asked Questions diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md deleted file mode 100644 index 13b21d8..0000000 --- a/GETTING_STARTED.md +++ /dev/null @@ -1,334 +0,0 @@ -# Decide On Development Environment - -- [Decide On Development Environment](#decide-on-development-environment) - - [Introduction](#introduction) - - [Poetry](#poetry) - - [Full Docker Development Environment](#full-docker-development-environment) - - [Invoke](#invoke) - - [Invoke - Building the Docker Image](#invoke---building-the-docker-image) - - [Invoke - Starting the Development Environment](#invoke---starting-the-development-environment) - - [Invoke - Creating a Superuser](#invoke---creating-a-superuser) - - [Invoke - Stopping the Development Environment](#invoke---stopping-the-development-environment) - - [Real-Time Updates? How Cool!](#real-time-updates-how-cool) - - [Docker Magic](#docker-magic) - - [Docker Logs](#docker-logs) - - [To Rebuild or Not to Rebuild](#to-rebuild-or-not-to-rebuild) - - [Updating Environment Variables](#updating-environment-variables) - - [Installing Additional Python Packages](#installing-additional-python-packages) - - [Installing Additional Nautobot Apps](#installing-additional-nautobot-apps) - - [Updating Python Version](#updating-python-version) - - [Updating Nautobot Version](#updating-nautobot-version) - - [Local Development Environment](#local-development-environment) - - [Other Miscellaneous Commands To Know](#other-miscellaneous-commands-to-know) - - [Python Shell](#python-shell) - - [Tests](#tests) - -## Introduction - -The cookie provides the ability to develop and manage the Nautobot server locally (with supporting services being *Dockerized*) or using only Docker containers to manage Nautobot. The main difference between the two environments is the ability to debug and use **pdb** when developing locally. Debugging with **pdb** within the Docker container is more complicated, but can still be accomplished by either exec'ing into the container or attaching your IDE to the container and running the Nautobot service manually within the container. - -The upside to having the Nautobot service handled by Docker rather than locally is that you do not have to manage the Nautobot server and the [Docker logs](#docker-logs) provide the majority information you will need to help troubleshoot while getting started quickly and not requiring you to perform several manual steps to get started and remember to have the Nautobot server running or having it run in a separate terminal while you develop. Ultimately, the decision is yours as to how you want to develop, but it was agreed it would be a good idea to provide pros and cons for each development environment. - -> The local environment still uses Docker containers for the supporting services (Postgres, Redis, and RQ Worker), but the Nautobot server is handled locally by you, the developer. - -Follow the directions below for the specific development environment that you choose. - -## Poetry - -Poetry is used in lieu of the "virtualenv" commands and is used for either environment. The virtual environment will provide most of the Python packages required to manage the development environment such as **Invoke**, but see the [Local Development Environment](#local-development-environment) section to see how to install Nautobot if you're going to be developing locally. To get started, run the following commands: - -```bash -➜ poetry install -➜ poetry shell -``` - -The first command creates the virtual environment through Poetry and installs all relevant dependencies, as outlined in the `pyproject.toml` file. - -The second command puts your shell session into the virtual environment, so all commands ran going forward are from within the virtual environment. (This is similar to running the `source venv/bin/activate` command with virtualenvs). - -## Full Docker Development Environment - -### Invoke - -The beauty of **Invoke** is that the Cookiecutter template provides several simple CLI commands to get developing fast. You'll use a few `invoke` commands to get your environment up and running. - -#### Invoke - Building the Docker Image - -The first thing you need to do is build the necessary Docker image for Nautobot that installs the specific **nautobot_ver**. The image is used for Nautobot and the RQ worker service used by Docker Compose. - -```bash -➜ invoke build -... -#14 exporting to image -#14 sha256:e8c613e07b0b7ff33893b694f7759a10d42e180f2b4dc349fb57dc6b71dcab00 -#14 exporting layers -#14 exporting layers 1.2s done -#14 writing image sha256:2d524bc1665327faa0d34001b0a9d2ccf450612bf8feeb969312e96a2d3e3503 done -#14 naming to docker.io/secrets/nautobot:1.2.0-py3.7 done -``` - -### Invoke - Starting the Development Environment - -Next, you need to start up your Docker containers. - -```bash -➜ invoke start -Starting Nautobot in detached mode... -Running docker-compose command "up --detach" -Creating network "nautobot_secrets_providers_default" with the default driver -Creating volume "nautobot_secrets_providers_postgres_data" with default driver -Creating nautobot_secrets_providers_redis_1 ... -Creating nautobot_secrets_providers_docs_1 ... -Creating nautobot_secrets_providers_postgres_1 ... -Creating nautobot_secrets_providers_postgres_1 ... done -Creating nautobot_secrets_providers_redis_1 ... done -Creating nautobot_secrets_providers_nautobot_1 ... -Creating nautobot_secrets_providers_docs_1 ... done -Creating nautobot_secrets_providers_nautobot_1 ... done -Creating nautobot_secrets_providers_worker_1 ... -Creating nautobot_secrets_providers_worker_1 ... done -Docker Compose is now in the Docker CLI, try `docker compose up` -``` - -This will start all of the Docker containers used for hosting Nautobot. Once the containers are up, you should be able to open up a web browser, and view the homepage at [http://localhost:8080](http://localhost:8080). - -> NOTE: Sometimes the containers take a minute to fully spin up. If the page doesn't load right away, wait a minute and try again. - -```bash -➜ docker ps -****CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -ee90fbfabd77 secrets/nautobot:1.2.0-py3.7 "nautobot-server rqw…" 16 seconds ago Up 13 seconds nautobot_secrets_providers_worker_1 -b8adb781d013 secrets/nautobot:1.2.0-py3.7 "/docker-entrypoint.…" 20 seconds ago Up 15 seconds 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp nautobot_secrets_providers_nautobot_1 -d64ebd60675d secrets/nautobot:1.2.0-py3.7 "mkdocs serve -v -a …" 25 seconds ago Up 18 seconds 0.0.0.0:8001->8080/tcp, :::8001->8080/tcp nautobot_secrets_providers_docs_1 -e72d63129b36 postgres:13-alpine "docker-entrypoint.s…" 25 seconds ago Up 19 seconds 0.0.0.0:5432->5432/tcp, :::5432->5432/tcp nautobot_secrets_providers_postgres_1 -96c6ff66997c redis:6-alpine "docker-entrypoint.s…" 25 seconds ago Up 21 seconds 0.0.0.0:6379->6379/tcp, :::6379->6379/tcp nautobot_secrets_providers_redis_1 -``` - -You should see the following containers running after running `invoke start` at this time of writing. - -### Invoke - Creating a Superuser - -The Nautobot development image will automatically provision a super user when specifying the following variables within `creds.env` which is the default when copying `creds.example.env` to `creds.env`. - -- **NAUTOBOT_CREATE_SUPERUSER=true** -- **NAUTOBOT_SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567** -- **NAUTOBOT_SUPERUSER_PASSWORD=admin** - -> NOTE: The default username is **admin**, but can be overridden by specifying **NAUTOBOT_SUPERUSER_USERNAME**. - -If you need to create additional superusers, run the follow commands. - -```bash -➜ invoke createsuperuser -Running docker-compose command "ps --services --filter status=running" -Running docker-compose command "exec nautobot nautobot-server createsuperuser --username admin" -Error: That username is already taken. -Username: ntc -Email address: ntc@networktocode.com -Password: -Password (again): -Superuser created successfully. -``` - -### Invoke - Stopping the Development Environment - -The last command to know for now is `invoke stop`. - -```bash -➜ invoke stop -Stopping Nautobot... -Running docker-compose command "down" -Stopping nautobot_secrets_providers_worker_1 ... -Stopping nautobot_secrets_providers_nautobot_1 ... -Stopping nautobot_secrets_providers_docs_1 ... -Stopping nautobot_secrets_providers_redis_1 ... -Stopping nautobot_secrets_providers_postgres_1 ... -Stopping nautobot_secrets_providers_worker_1 ... done -Stopping nautobot_secrets_providers_nautobot_1 ... done -Stopping nautobot_secrets_providers_postgres_1 ... done -Stopping nautobot_secrets_providers_redis_1 ... done -Stopping nautobot_secrets_providers_docs_1 ... done -Removing nautobot_secrets_providers_worker_1 ... -Removing nautobot_secrets_providers_nautobot_1 ... -Removing nautobot_secrets_providers_docs_1 ... -Removing nautobot_secrets_providers_redis_1 ... -Removing nautobot_secrets_providers_postgres_1 ... -Removing nautobot_secrets_providers_postgres_1 ... done -Removing nautobot_secrets_providers_docs_1 ... done -Removing nautobot_secrets_providers_worker_1 ... done -Removing nautobot_secrets_providers_redis_1 ... done -Removing nautobot_secrets_providers_nautobot_1 ... done -Removing network nautobot_secrets_providers_default -``` - -This will safely shut down all of your running Docker containers for this project. When you are ready to spin containers back up, it is as simple as running `invoke start` again like in [**Invoke - Starting the Development Environment**](#invoke---starting-the-development-environment). - -> NOTE: If you're wanting to reset the database and configuration settings, you can use the `invoke destroy` command, but it will result in data loss so make sure that is what you want to do. - -### Real-Time Updates? How Cool! - -Your environment should now be fully setup, all necessary Docker containers are created and running, and you're logged into Nautobot in your web browser. Now what? - -Now you can start developing your app in the folder generated for you by Cookiecutter. - -## Docker Magic - -The magic here is the root directory is mounted inside your Docker containers when built and ran, so **any** changes made to the files in here are directly updated to the Nautobot app code running in Docker. This means that as you modify the code in your `nautobot-app` folder (or whatever you named your app when generating it via Cookiecutter), the changes will be instantly updated in Nautobot. - -> NOTE: There are a few exceptions to this, as outlined in the section [To Rebuild or Not To Rebuild](#to-rebuild-or-not-to-rebuild). - -The backend Django process is setup to automatically reload itself (it only takes a couple of seconds) every time a file is updated (saved). So for example, if you were to update one of the files like `tables.py`, then save it, the changes will be visible right away in the web browser! - -> NOTE: You may get connection refused while Django reloads, but it should be refreshed fairly quickly. - -### Docker Logs - -When trying to debug an issue, one helpful thing you can look at are the logs within the Docker containers. - -```bash -➜ docker logs -f -``` - -> NOTE: The `-f` tag will keep the logs open, and output them in realtime as they are generated. - -So for example, our app is named `secrets`, the command would most likely be `docker logs nautobot_secrets_providers_nautobot_1 -f`. You can find the name of all running containers via `docker ps`. - -If you want to view the logs specific to the worker container, simply use the name of that container instead. - -## To Rebuild or Not to Rebuild - -Most of the time, you will not need to rebuild your images. Simply running `invoke start` and `invoke stop` is enough to keep your environment going. - -However there are a couple of instances when you will want to. - -### Updating Environment Variables - -To add environment variables to your containers, thus allowing Nautobot to use them, you will update/add them in the `development/dev.env` file. However, doing so is considered updating the underlying container shell, instead of Django (which auto restarts itself on changes). - -To get new environment variables to take effect, you will need stop any running images, rebuild the images, then restart them. This can easily be done with 3 commands: - -```bash -➜ invoke stop -➜ invoke build -➜ invoke start -``` - -Once completed, the new/updated environment variables should now be live. - -### Installing Additional Python Packages - -If you want your app to leverage another available Nautobot app or another Python package, you can easily add them into your Docker environment. - -```bash -➜ poetry shell -➜ poetry add netutils -``` - -Once the dependencies are resolved, stop the existing containers, rebuild the Docker image, and then start all containers again. - -```bash -➜ invoke stop -➜ invoke build -➜ invoke start -``` - -### Installing Additional Nautobot Apps - -Let's say for example you want the new app you're creating to integrate into Slack. To do this, you will want to integrate into the existing Nautobot ChatOps app. - -```bash -➜ poetry shell -➜ poetry add nautobot-chatops-app -``` - -Once you activate the virtual environment via Poetry, you then tell Poetry to install the new app. - -Before you continue, you'll need to update the file `development/nautobot_config.py` accordingly with the name of the new app under `PLUGINS` and any relevant settings as necessary for the app under `PLUGINS_CONFIG`. Since you're modifying the underlying OS (not just Django files), you need to rebuild the image. This is a similar process to updating environment variables, which was explained earlier. - -```bash -➜ invoke stop -➜ invoke build -➜ invoke start -``` - -Once the containers are up and running, you should now see the new app installed in your Nautobot instance. - -You can even launch an `ngrok` service locally on your laptop, pointing to port 8080 (such as for chatops development), and it will point traffic directly to your Docker images. How cool! - -### Updating Python Version - -To update the Python version, you can update it within `tasks.py`. - -```python -namespace = Collection("nautobot_secrets_providers") -namespace.configure( - { - "nautobot_secrets_providers": { - ... - "python_ver": "3.7", - ... - } - } -) -``` - -Or set the `INVOKE_NAUTOBOT_SECRETS_PROVIDERS_PYTHON_VER` variable - -### Updating Nautobot Version - -To update the Python version, you can update it within `tasks.py`. - -```python -namespace = Collection("nautobot_secrets_providers") -namespace.configure( - { - "nautobot_secrets_providers": { - ... - "nautobot_ver": "1.0.2", - ... - } - } -) -``` - -Or set the `INVOKE_NAUTOBOT_SECRETS_PROVIDERS_NAUTOBOT_VER` variable - -## Local Development Environment - -Refer back to the [README](./README.md) for developing locally. - -## Other Miscellaneous Commands To Know - -### Python Shell - -To drop into a Django shell for Nautobot (in the Docker container) run: - -```bash -➜ invoke nbshell -``` - -This is the same as running: - -```bash -➜ invoke cli -➜ nautobot-server nbshell -``` - -### Tests - -To run tests against your code, you can run all of the tests that TravisCI runs against any new PR with: - -```bash -➜ invoke tests -``` - -To run an individual test, you can run any or all of the following: - -```bash -➜ invoke unittest -➜ invoke bandit -➜ invoke black -➜ invoke flake8 -➜ invoke pydocstyle -➜ invoke pylint -``` diff --git a/README.md b/README.md index 949eac2..874e03f 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # Nautobot's Secrets Providers App

- -
@@ -11,9 +9,9 @@ An App for Nautobot.

-Nautobot Secrets Providers is a app for [Nautobot](https://github.com/nautobot/nautobot) 1.4.0 or higher that bundles Secrets Providers for integrating with popular secrets backends. Nautobot 1.4.0 added support for integrating with retrieving secrets from various secrets providers. +Nautobot Secrets Providers is an app for [Nautobot](https://github.com/nautobot/nautobot) 1.2.1 or higher that bundles Secrets Providers for integrating with popular secrets backends. Nautobot 1.2.0 added support for integrating with retrieving secrets from various secrets providers. -This app publishes secrets providers that are not included in the within the Nautobot core software package so that it will be easier to maintain and extend support for various secrets providers without waiting on Nautobot software releases. +This app publishes secrets providers that are not included in the Nautobot core software package so that it will be easier to maintain and extend support for various secrets providers without waiting on Nautobot software releases. ## Supported Secrets Backends @@ -52,192 +50,7 @@ This app supports the following popular secrets backends: ## Installation -> Nautobot Secrets Providers is compatible with Nautobot 1.4.0 and higher. Support for Thycotic Secret Server requires Python 3.7 or later. - -The package is available as a Python package in PyPI and can be installed with `pip`: - -```no-highlight -pip install nautobot-secrets-providers -``` - -You may quickly install all of the dependent libraries using the following command, however, this is *not recommended for production deployments* as it will install extras that you may not need: - -```no-highlight -pip install nautobot-secrets-providers[all] -``` - -### Dependencies - -For this app to operate you must install at least one of the dependent libraries required by the secrets providers. - -**You must install the dependencies for at least one of the supported secrets providers or a `RuntimeError` will be raised.** - -#### AWS - -AWS Secrets Manager and Systems Manager Parameter Store are supported. Both providers require the `boto3` library. This can be easily installed along with the app using the following command: - -```no-highlight -pip install nautobot-secrets-providers[aws] -``` - -#### HashiCorp Vault - -The HashiCorp Vault provider requires the `hvac` library. This can easily be installed along with the app using the following command: - -```no-highlight -pip install nautobot-secrets-providers[hashicorp] -``` - -#### Delinea/Thycotic Secret Server - -The Delinea/Thycotic Secret Server provider requires the `python-tss-sdk` library. This can easily be installed along with the app using the following command: - -```no-highlight -pip install nautobot-secrets-providers[thycotic] -``` - -### Enabling Secrets Providers - -To ensure Nautobot Secrets Providers is automatically re-installed during future upgrades, create a file named `local_requirements.txt` (if not already existing) in the Nautobot root directory (alongside `requirements.txt`) and list the `secrets` package: - -```no-highlight -echo nautobot-secrets-providers >> local_requirements.txt -``` - -Once installed, the app needs to be enabled in your `nautobot_config.py`: - -```python -# In your nautobot_config.py -PLUGINS = ["nautobot_secrets_providers"] - -# PLUGINS_CONFIG = { -# "nautobot_secrets_providers": { -# See below for how to configure Nautobot for each secrets provider! -# } -# } -``` - -## Usage - -Before you proceed, you must have **at least one** of the dependent libaries installed as detailed above. - -Please do not enable this app until you are able to install the dependencies, as it will block Nautobot from starting. - -### AWS - -#### Authentication - -No configuration is needed within Nautobot for this provider to operate. Instead you must provide [AWS credentials in one of the methods supported by the `boto3` library](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials). - -Boto3 credentials can be configured in multiple ways (eight as of this writing) that are mirrored here: - -1. Passing credentials as parameters in the `boto.client()` method -2. Passing credentials as parameters when creating a Session object -3. Environment variables -4. Shared credential file (`~/.aws/credentials`) -5. AWS config file (`~/.aws/config`) -6. Assume Role provider -7. Boto2 config file (`/etc/boto.cfg` and `~/.boto`) -8. Instance metadata service on an Amazon EC2 instance that has an IAM role configured. - -**The AWS providers only support methods 3-8. Methods 1 and 2 ARE NOT SUPPORTED at this time.** - -We highly recommend you defer to using environment variables for your deployment as specified in the credentials documentation linked above. The values specified in the linked documentation should be [set within your `~.bashrc`](https://nautobot.readthedocs.io/en/latest/installation/nautobot/#update-the-nautobot-bashrc) (or similar profile) on your system. - -#### Configuration - -This is an example based on our recommended deployment pattern in the section above (item 3) that is using [environment variables](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#environment-variables). You will need to set these in the environment prior to starting Nautobot: - -```no-highlight -export AWS_ACCESS_KEY_ID=foo # The access key for your AWS account. -export AWS_SECRET_ACCESS_KEY=bar # The secret key for your AWS account. -``` - -Please refer to the [Nautobot documentation on updating your `.bashrc`](https://nautobot.readthedocs.io/en/latest/installation/nautobot/#update-the-nautobot-bashrc) for how to do this for production Nautobot deployments. - -### HashiCorp Vault - -#### Configuration - -You must provide a mapping in `PLUGINS_CONFIG` within your `nautobot_config.py`, for example: - -```python -PLUGINS_CONFIG = { - "nautobot_secrets_providers": { - "hashicorp_vault": { - "url": "http://localhost:8200", - "auth_method": "token", - "token": os.getenv("NAUTOBOT_HASHICORP_VAULT_TOKEN"), - } - }, -} -``` - -- `url` - (required) The URL to the HashiCorp Vault instance (e.g. `http://localhost:8200`). -- `auth_method` - (optional / defaults to "token") The method used to authenticate against the HashiCorp Vault instance. Either `"approle"`, `"aws"`, `"kubernetes"` or `"token"`. For information on using AWS authentication with vault see the [authentication](#authentication) section above. -- `ca_cert` - (optional) Path to a PEM formatted CA certificate to use when verifying the Vault connection. Can alternatively be set to `False` to ignore SSL verification (not recommended) or `True` to use the system certificates. -- `default_mount_point` - (optional / defaults to "secret") The default mount point of the K/V Version 2 secrets engine within Hashicorp Vault. -- `kv_version` - (optional / defaults to "v2") The version of the KV engine to use, can be `v1` or `v2` -- `k8s_token_path` - (optional) Path to the kubernetes service account token file. Defaults to "/var/run/secrets/kubernetes.io/serviceaccount/token". -- `token` - (optional) Required when `"auth_method": "token"` or `auth_method` is not supplied. The token for authenticating the client with the HashiCorp Vault instance. As with other sensitive service credentials, we recommend that you provide the token value as an environment variable and retrieve it with `{"token": os.getenv("NAUTOBOT_HASHICORP_VAULT_TOKEN")}` rather than hard-coding it in your `nautobot_config.py`. -- `role_name` - (optional) Required when `"auth_method": "kubernetes"`, optional when `"auth_method": "aws"`. The Vault Kubernetes role or Vault AWS role to assume which the pod's service account has access to. -- `role_id` - (optional) Required when `"auth_method": "approle"`. As with other sensitive service credentials, we recommend that you provide the role_id value as an environment variable and retrieve it with `{"role_id": os.getenv("NAUTOBOT_HASHICORP_VAULT_ROLE_ID")}` rather than hard-coding it in your `nautobot_config.py`. -- `secret_id` - (optional) Required when `"auth_method": "approle"`.As with other sensitive service credentials, we recommend that you provide the secret_id value as an environment variable and retrieve it with `{"secret_id": os.getenv("NAUTOBOT_HASHICORP_VAULT_SECRET_ID")}` rather than hard-coding it in your `nautobot_config.py`. -- `login_kwargs` - (optional) Additional optional parameters to pass to the login method for [`approle`](https://hvac.readthedocs.io/en/stable/source/hvac_api_auth_methods.html#hvac.api.auth_methods.AppRole.login), [`aws`](https://hvac.readthedocs.io/en/stable/source/hvac_api_auth_methods.html#hvac.api.auth_methods.Aws.iam_login) and [`kubernetes`](https://hvac.readthedocs.io/en/stable/source/hvac_api_auth_methods.html#hvac.api.auth_methods.Kubernetes.login) authentication methods. -- `namespace` - (optional) Namespace to use for the [`X-Vault-Namespace` header](https://github.com/hvac/hvac/blob/main/hvac/adapters.py#L287) on all hvac client requests. Required when the [`Namespaces`](https://developer.hashicorp.com/vault/docs/enterprise/namespaces#usage) feature is enabled in Vault Enterprise. - -### Delinea/Thycotic Secret Server (TSS) - -The Delinea/Thycotic Secret Server app includes two providers: - -- **`Thycotic Secret Server by ID`** - - This provider uses the `Secret ID` to specifiy the secret that is selected. The `Secret ID` is displayed in the browser's URL field if you `Edit` the data in Thycotic Secret Server. - - - Example: - - The url is: _https://pw.example.local/SecretServer/app/#/secret/**1234**/general_ - - In this example the value for `Secret ID` is **1234**. - -- **`Thycotic Secret Server by Path`** - - This provider allows to select the secret by folder-path and secret-name. The path delimiter is a '\\'. - - The `Secret path` is displayed as page header when `Edit` a secret. - - - Example: - - The header is: **NET-Automation > Nautobot > My-Secret** - - In this example the value for `Secret path` is **`\NET-Automation\Nautobot\My-Secret`**. - -#### Configuration - -```python -PLUGINS_CONFIG = { - "nautobot_secrets_providers": { - "thycotic": { # https://github.com/thycotic/python-tss-sdk - "base_url": os.getenv("SECRET_SERVER_BASE_URL", None), - "ca_bundle_path": os.getenv("REQUESTS_CA_BUNDLE", None), - "cloud_based": is_truthy(os.getenv("SECRET_SERVER_IS_CLOUD_BASED", "False")), - "domain": os.getenv("SECRET_SERVER_DOMAIN", None), - "password": os.getenv("SECRET_SERVER_PASSWORD", None), - "tenant": os.getenv("SECRET_SERVER_TENANT", None), - "token": os.getenv("SECRET_SERVER_TOKEN", None), - "username": os.getenv("SECRET_SERVER_USERNAME", None), - }, - } -} -``` -- `base_url` - (required) The Secret Server base_url. _e.g.'https://pw.example.local/SecretServer'_ -- `ca_bundle_path` - (optional) When using self-signed certificates this variable must be set to a file containing the trusted certificates (in .pem format). _e.g. '/etc/ssl/certs/ca-bundle.trust.crt'_. -- `cloud_based` - (optional) Set to "True" if Secret Server Cloud should be used. (Default: "False"). -- `domain` - (optional) Required for 'Domain Authorization' -- `password` - (optional) Required for 'Secret Server Cloud', 'Password Authorization', 'Domain Authorization'. -- `tenant` - (optional) Required for 'Domain Authorization'. -- `token` - (optional) Required for 'Access Token Authorization'. -- `username` - (optional) Required for 'Secret Server Cloud', 'Password Authorization', 'Domain Authorization'. +See the [installation documentation](https://docs.nautobot.com/projects/secrets-providers/en/latest/admin/install/) for detailed instructions on installing the Nautobot Secrets Providers app. ## Contributing @@ -252,353 +65,7 @@ The project is following Network to Code software development guidelines and is ### Development Environment -The development environment can be used in 2 ways. First, with a local poetry environment if you wish to develop outside of Docker with the caveat of using external services provided by Docker for PostgreSQL and Redis. Second, all services are spun up using Docker and a local mount so you can develop locally, but Nautobot is spun up within the Docker container. - -Below is a quick start guide if you're already familiar with the development environment provided, but if you're not familiar, please read the [Getting Started Guide](GETTING_STARTED.md). - -#### Invoke - -The [PyInvoke](http://www.pyinvoke.org/) library is used to provide some helper commands based on the environment. There are a few configuration parameters which can be passed to PyInvoke to override the default configuration: - -* `nautobot_ver`: the version of Nautobot to use as a base for any built docker containers (default: 1.4.10) -* `project_name`: the default docker compose project name (default: nautobot_secrets_providers) -* `python_ver`: the version of Python to use as a base for any built docker containers (default: 3.8) -* `local`: a boolean flag indicating if invoke tasks should be run on the host or inside the docker containers (default: False, commands will be run in docker containers) -* `compose_dir`: the full path to a directory containing the project compose files -* `compose_files`: a list of compose files applied in order (see [Multiple Compose files](https://docs.docker.com/compose/extends/#multiple-compose-files) for more information) - -Using **PyInvoke** these configuration options can be overridden using [several methods](http://docs.pyinvoke.org/en/stable/concepts/configuration.html). Perhaps the simplest is simply setting an environment variable `INVOKE_NAUTOBOT_SECRETS_PROVIDERS_VARIABLE_NAME` where `VARIABLE_NAME` is the variable you are trying to override. The only exception is `compose_files`, because it is a list it must be overridden in a yaml file. There is an example `invoke.yml` (`invoke.example.yml`) in this directory which can be used as a starting point. - -#### Local Poetry Development Environment - -1. Copy `development/creds.example.env` to `development/creds.env` (This file will be ignored by Git and Docker) -2. Uncomment the `POSTGRES_HOST`, `REDIS_HOST`, and `NAUTOBOT_ROOT` variables in `development/creds.env` -3. Create an `invoke.yml` file with the following contents at the root of the repo (you can also `cp invoke.example.yml invoke.yml` and edit as necessary): - -```yaml ---- -nautobot_secrets_providers: - local: true - compose_files: - - "docker-compose.requirements.yml" -``` - -3. Run the following commands: - -```shell -poetry shell -poetry install --extras nautobot -export $(cat development/dev.env | xargs) -export $(cat development/creds.env | xargs) -invoke start && sleep 5 -nautobot-server migrate -``` - -> If you want to develop on the latest develop branch of Nautobot, run the following command: `poetry add --optional git+https://github.com/nautobot/nautobot@develop`. After the `@` symbol must match either a branch or a tag. - -4. You can now run nautobot-server commands as you would from the [Nautobot documentation](https://nautobot.readthedocs.io/en/latest/) for example to start the development server: - -```shell -nautobot-server runserver 0.0.0.0:8080 --insecure -``` - -Nautobot server can now be accessed at [http://localhost:8080](http://localhost:8080). - -It is typically recommended to launch the Nautobot **runserver** command in a separate shell so you can keep developing and manage the webserver separately. - -#### Docker Development Environment - -This project is managed by [Python Poetry](https://python-poetry.org/) and has a few requirements to setup your development environment: - -1. Install Poetry, see the [Poetry Documentation](https://python-poetry.org/docs/#installation) for your operating system. -2. Install Docker, see the [Docker documentation](https://docs.docker.com/get-docker/) for your operating system. - -Once you have Poetry and Docker installed you can run the following commands to install all other development dependencies in an isolated python virtual environment: - -```shell -poetry shell -poetry install -invoke start -``` - -Nautobot server can now be accessed at [http://localhost:8080](http://localhost:8080). - -To either stop or destroy the development environment use the following options. - -- **invoke stop** - Stop the containers, but keep all underlying systems intact -- **invoke destroy** - Stop and remove all containers, volumes, etc. (This results in data loss due to the volume being deleted) - -### CLI Helper Commands - -The project is coming with a CLI helper based on [invoke](http://www.pyinvoke.org/) to help setup the development environment. The commands are listed below in 3 categories `dev environment`, `utility` and `testing`. - -Each command can be executed with `invoke `. Environment variables `INVOKE_NAUTOBOT_SECRETS_PROVIDERS_PYTHON_VER` and `INVOKE_NAUTOBOT_SECRETS_PROVIDERS_NAUTOBOT_VER` may be specified to override the default versions. Each command also has its own help `invoke --help` - -#### Docker dev environment - -```no-highlight - build Build all docker images. - debug Start Nautobot and its dependencies in debug mode. - destroy Destroy all containers and volumes. - restart Restart Nautobot and its dependencies. - start Start Nautobot and its dependencies in detached mode. - stop Stop Nautobot and its dependencies. -``` - -#### Utility - -```no-highlight - cli Launch a bash shell inside the running Nautobot container. - create-user Create a new user in django (default: admin), will prompt for password. - makemigrations Run Make Migration in Django. - nbshell Launch a nbshell session. -``` - -#### Testing - -```no-highlight - bandit Run bandit to validate basic static code security analysis. - black Run black to check that Python files adhere to its style standards. - flake8 This will run flake8 for the specified name and Python version. - pydocstyle Run pydocstyle to validate docstring formatting adheres to NTC defined standards. - pylint Run pylint code analysis. - tests Run all tests for this app. - unittest Run Django unit tests for the app. -``` - -### Project Documentation - -Project documentation is generated by [mkdocs](https://www.mkdocs.org/) from the documentation located in the docs folder. You can configure [readthedocs.io](https://readthedocs.io/) to point at this folder in your repo. For development purposes a `docker-compose.docs.yml` is also included. A container hosting the docs will be started using the invoke commands on [http://localhost:8001](http://localhost:8001), as changes are saved the docs will be automatically reloaded. - -### Developing Against Secrets Backends - -#### AWS Secrets Manager - -This assumes you are logged into the AWS Console. - -- Navigate to AWS Console -- Navigate to AWS Secrets Manager -- Click "Store a new secret" - - Select “Other type of secrets” - - Use Secret key/value - - Enter `hello=world` - - Use "DefaultEncryptionKey" for now - - Click "Next" - - Under "Secret name" fill out `hello` - - Click "Next" - - Under "Configure automatic rotation" - - Leave it as "Disable automatic rotation" - - On "Store a new secret" - - Copy the sample code (see below) - - Click "Store" -- END - -##### Install the AWS CLI - -Next, install the [AWS CLI](https://aws.amazon.com/cli/). - -On MacOS, this can be done using `brew install awscli`: - -``` -brew install awscli -``` - -On Linux, you will need to run a `curl` command (This assumes x86. Please see the docs for [AWS CLI on -Linux](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) for ARM and other options): - -``` -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -unzip awscliv2.zip -sudo ./aws/install -``` - -##### Configure the AWS CLI - -After installing the AWS CLI, you will need to configure it for authentication. - -You may use an existing AWS access key or create a new one. For these instructions we cover the need to create a new access key that can be used for this. - -- Navigate to AWS Console -- Click your username - - Click "My security credentials" - - Click "create access key" -- Save your "Access key ID" and "Secret access key" for use when configuring the AWS CLI - -Now configure the CLI: - -- Run `aws configure` -- Enter your credentials from above -- Choose your region -- Use output format: `json` - -Example: - -```no-highlight -$ aws configure -AWS Access Key ID [None]: AKIAIOSFODNN7EXAMPLE -AWS Secret Access Key [None]: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY -Default region name [None]: us-east-2 -Default output format [None]: json -``` - -Now you are ready to use the sample code to retrieve your secret from AWS Secrets Manager! - -##### Sample Code - -Make sure that the `boto3` client is installed: - -```no-highlight -poetry install --extras aws -``` - -Next, save this as `aws_secrets.py`: - -```python -# Use this code snippet in your app. -# If you need more information about configurations or implementing the sample code, visit the AWS docs: -# https://aws.amazon.com/developers/getting-started/python/ - -import boto3 -import base64 -from botocore.exceptions import ClientError - - -def get_secret(): - - secret_name = "hello" - region_name = "us-east-2" - - # Create a Secrets Manager client - session = boto3.session.Session() - client = session.client( - service_name='secretsmanager', - region_name=region_name - ) - - # In this sample we only handle the specific exceptions for the 'GetSecretValue' API. - # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html - # We rethrow the exception by default. - - try: - get_secret_value_response = client.get_secret_value( - SecretId=secret_name - ) - except ClientError as e: - if e.response['Error']['Code'] == 'DecryptionFailureException': - # Secrets Manager can't decrypt the protected secret text using the provided KMS key. - # Deal with the exception here, and/or rethrow at your discretion. - raise e - elif e.response['Error']['Code'] == 'InternalServiceErrorException': - # An error occurred on the server side. - # Deal with the exception here, and/or rethrow at your discretion. - raise e - elif e.response['Error']['Code'] == 'InvalidParameterException': - # You provided an invalid value for a parameter. - # Deal with the exception here, and/or rethrow at your discretion. - raise e - elif e.response['Error']['Code'] == 'InvalidRequestException': - # You provided a parameter value that is not valid for the current state of the resource. - # Deal with the exception here, and/or rethrow at your discretion. - raise e - elif e.response['Error']['Code'] == 'ResourceNotFoundException': - # We can't find the resource that you asked for. - # Deal with the exception here, and/or rethrow at your discretion. - raise e - else: - # Decrypts secret using the associated KMS CMK. - # Depending on whether the secret is a string or binary, one of these fields will be populated. - if 'SecretString' in get_secret_value_response: - secret = get_secret_value_response['SecretString'] - else: - decoded_binary_secret = base64.b64decode(get_secret_value_response['SecretBinary']) - - # Your code goes here. - -# ^ Above was generated by AWS. - -# This was added by us so you can run this as a script: -if __name__ == "__main__": - secret = get_secret() - print(f"Secret = {secret}") -``` - -Run it with `python aws_secrets.py`: - -``` -$ python aws_secrets.py -Secret = {"hello":"world"}. -``` - -Note that this blob is JSON and will also need to be decoded if you want to extract the value. - -#### HashiCorp Vault - -Make sure that the `hvac` client is installed: - -```no-highlight -poetry install --extras hashicorp -``` - -##### Start Services with Docker - -```no-highlight -invoke start -``` - -##### Set an alias to work with `vault` - -This will allow you to easily run the CLI command from within the container: - -```no-highlight -alias vault="docker exec -it nautobot_secrets_providers_vault_1 vault" -``` - -Interact with the Vault vi CLI (via `docker exec` into the container from localhost): - -```no-highlight -$ vault status -Key Value ---- ----- -Seal Type shamir -Initialized true -Sealed false -Total Shares 1 -Threshold 1 -Version 1.8.2 -Storage Type inmem -Cluster Name vault-cluster-35c5d319 -Cluster ID 2611f99c-a6de-a883-1fcc-bfffdc0217bc -HA Enabled false -``` - -##### Using the Python `hvac` Library - -This establishes a client, creates a basic key/value secret (`hello=world`) at the path `hello`, and then retrieves the data from the `hello` key at the secret path `hello`. - -> This is equivalent to the command `vault kv get -field hello secret/hello`. - -```python -In [1]: import hvac - -In [2]: client = hvac.Client(url="http://localhost:8200", token="nautobot") - -In [3]: client.secrets.kv.create_or_update_secret(path="hello", secret=dict(hello="world")) -Out[3]: -{'request_id': 'c4709868-c08f-4cb1-ab8c-605c58b82f3f', - 'lease_id': '', - 'renewable': False, - 'lease_duration': 0, - 'data': {'created_time': '2021-09-16T23:21:07.5564132Z', - 'deletion_time': '', - 'destroyed': False, - 'version': 2}, - 'wrap_info': None, - 'warnings': None, - 'auth': None} - -In [4]: client.secrets.kv.read_secret(path="hello")["data"]["data"]["hello"] -Out[4]: 'world' -``` +For information on setting up a local development environment, see the [documentation](https://docs.nautobot.com/projects/secrets-providers/en/latest/dev/dev_environment/). ## Questions diff --git a/changes/8.housekeeping b/changes/8.housekeeping new file mode 100644 index 0000000..653c54a --- /dev/null +++ b/changes/8.housekeeping @@ -0,0 +1 @@ +Re-baked from the latest template. diff --git a/development/Dockerfile b/development/Dockerfile index 7d3c41e..c30ef27 100644 --- a/development/Dockerfile +++ b/development/Dockerfile @@ -28,7 +28,7 @@ ENV INVOKE_NAUTOBOT_SECRETS_PROVIDERS_LOCAL=true # Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary # We also don't need virtual environments in container RUN which poetry || curl -sSL https://install.python-poetry.org | python3 - && \ - poetry config virtualenvs.create false + poetry config virtualenvs.create false # !!! USE CAUTION WHEN MODIFYING LINES ABOVE # ------------------------------------------------------------------------------------- diff --git a/development/app_config_schema.py b/development/app_config_schema.py new file mode 100644 index 0000000..4700995 --- /dev/null +++ b/development/app_config_schema.py @@ -0,0 +1,64 @@ +"""App Config Schema Generator and Validator.""" +import json +from importlib import import_module +from os import getenv +from pathlib import Path +from urllib.parse import urlparse + +import jsonschema +import toml +from django.conf import settings +from to_json_schema.to_json_schema import SchemaBuilder + + +def _enrich_object_schema(schema, defaults, required): + schema["additionalProperties"] = False + for key, value in schema["properties"].items(): + if required and key in required: + value["required"] = True + default_value = defaults and defaults.get(key, None) + if value["type"] == "object" and "properties" in value: + _enrich_object_schema(value, default_value, None) + elif default_value is not None: + value["default"] = default_value + + +def _main(): + pyproject = toml.loads(Path("pyproject.toml").read_text()) + url = urlparse(pyproject["tool"]["poetry"]["repository"]) + _, owner, repository = url.path.split("/") + package_name = pyproject["tool"]["poetry"]["packages"][0]["include"] + app_config = settings.PLUGINS_CONFIG[package_name] # type: ignore + schema_path = Path(package_name) / "app-config-schema.json" + command = getenv("APP_CONFIG_SCHEMA_COMMAND", "") + if command == "generate": + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": f"https://raw.githubusercontent.com/{owner}/{repository}/develop/{package_name}/app-config-schema.json", + "$comment": "TBD: Update $id, replace `develop` with the future release tag", + **SchemaBuilder().to_json_schema(app_config), # type: ignore + } + app_config = import_module(package_name).config + _enrich_object_schema(schema, app_config.default_settings, app_config.required_settings) + schema_path.write_text(json.dumps(schema, indent=4) + "\n") + print(f"\n==================\nGenerated schema:\n\n{schema_path}\n") + print( + "WARNING: Review and edit the generated file before committing.\n" + "\n" + "Its content is inferred from:\n" + "\n" + "- The current configuration in `PLUGINS_CONFIG`\n" + "- `NautobotAppConfig.default_settings`\n" + "- `NautobotAppConfig.required_settings`" + ) + elif command == "validate": + schema = json.loads(schema_path.read_text()) + jsonschema.validate(app_config, schema) + print( + f"\n==================\nValidated configuration using the schema:\n{schema_path}\nConfiguration is valid." + ) + else: + raise RuntimeError(f"Unknown command: {command}") + + +_main() diff --git a/development/nautobot_config.py b/development/nautobot_config.py index 4b988eb..d61ed30 100644 --- a/development/nautobot_config.py +++ b/development/nautobot_config.py @@ -92,11 +92,11 @@ "disable_existing_loggers": False, "formatters": { "normal": { - "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s :\n %(message)s", + "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s : %(message)s", "datefmt": "%H:%M:%S", }, "verbose": { - "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() :\n %(message)s", + "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() : %(message)s", "datefmt": "%H:%M:%S", }, }, diff --git a/development/towncrier_template.j2 b/development/towncrier_template.j2 index d2ec61b..b0f6e12 100644 --- a/development/towncrier_template.j2 +++ b/development/towncrier_template.j2 @@ -1,6 +1,6 @@ {% if render_title %} -## v{{ versiondata.version }} ({{ versiondata.date }}) +## [v{{ versiondata.version }} ({{ versiondata.date }})]({{ cookiecutter.repo_url }}/releases/tag/v{{ versiondata.version}}) {% endif %} {% for section, _ in sections.items() %} diff --git a/docs/admin/compatibility_matrix.md b/docs/admin/compatibility_matrix.md index 7303164..c118ab1 100644 --- a/docs/admin/compatibility_matrix.md +++ b/docs/admin/compatibility_matrix.md @@ -1,9 +1,7 @@ # Compatibility Matrix -!!! warning "Developer Note - Remove Me!" - Explain how the release models of the app and of Nautobot work together, how releases are supported, how features and older releases are deprecated etc. - | Nautobot's Secrets Providers App Version | Nautobot First Support Version | Nautobot Last Support Version | -| ------------- | -------------------- | ------------- | -| 1.0.X | 1.4.0 | 1.99.99 | -| 2.0.X | 2.0.0 | 2.99.99 | +| ---------------------------------------- | ------------------------------ | ----------------------------- | +| 1.0.0 - 1.3.0 | 1.2.1 | 1.2.99 | +| 1.4.X | 1.4.0 | 1.99.99 | +| 2.0.X | 2.0.0 | 2.99.99 | diff --git a/docs/admin/install.md b/docs/admin/install.md new file mode 100644 index 0000000..637ea26 --- /dev/null +++ b/docs/admin/install.md @@ -0,0 +1,212 @@ +# Installing the App in Nautobot + +Here you will find detailed instructions on how to **install** and **configure** the App within your Nautobot environment. + +## Prerequisites + +- This app is compatible with Nautobot 1.2.1 and higher. +- Support for Thycotic Secret Server requires Python 3.7 or later. +- Databases supported: PostgreSQL, MySQL + +!!! note + Please check the [dedicated page](compatibility_matrix.md) for a full compatibility matrix and the deprecation policy. + +Before you proceed, you must have **at least one** of the dependent libaries installed as detailed below. + +Please do not enable this app until you are able to install the dependencies, as it will block Nautobot from starting. + +### Dependencies + +For this app to operate you must install at least one of the dependent libraries required by the secrets providers. + +**You must install the dependencies for at least one of the supported secrets providers or a `RuntimeError` will be raised.** + +#### AWS + +AWS Secrets Manager and Systems Manager Parameter Store are supported. Both providers require the `boto3` library. This can be easily installed along with the app using the following command: + +```no-highlight +pip install nautobot-secrets-providers[aws] +``` + +#### HashiCorp Vault + +The HashiCorp Vault provider requires the `hvac` library. This can easily be installed along with the app using the following command: + +```no-highlight +pip install nautobot-secrets-providers[hashicorp] +``` + +#### Delinea/Thycotic Secret Server + +The Delinea/Thycotic Secret Server provider requires the `python-tss-sdk` library. This can easily be installed along with the app using the following command: + +```no-highlight +pip install nautobot-secrets-providers[thycotic] +``` + +### Access Requirements + +There are no special access requirements to install the app. + +## Install Guide + +!!! note + Apps can be installed from the [Python Package Index](https://pypi.org/) or locally. See the [Nautobot documentation](https://docs.nautobot.com/projects/core/en/stable/user-guide/administration/installation/app-install/) for more details. The pip package name for this app is [`nautobot-secrets-providers`](https://pypi.org/project/nautobot-secrets-providers/). + +The app is available as a Python package via PyPI and can be installed with `pip`: + +```shell +pip install nautobot-secrets-providers +``` + +To ensure Nautobot's Secrets Providers App is automatically re-installed during future upgrades, create a file named `local_requirements.txt` (if not already existing) in the Nautobot root directory (alongside `requirements.txt`) and list the `nautobot-secrets-providers` package: + +```shell +echo nautobot-secrets-providers >> local_requirements.txt +``` + +Once installed, the app needs to be enabled in your Nautobot configuration. The following block of code below shows the additional configuration required to be added to your `nautobot_config.py` file: + +- Append `"nautobot_secrets_providers"` to the `PLUGINS` list. +- Append the `"nautobot_secrets_providers"` dictionary to the `PLUGINS_CONFIG` dictionary and override any defaults. + +```python +# In your nautobot_config.py +PLUGINS = ["nautobot_secrets_providers"] + +# PLUGINS_CONFIG = { +# "nautobot_secrets_providers": { +# ADD YOUR SETTINGS HERE +# } +# } +``` + +Once the Nautobot configuration is updated, run the Post Upgrade command (`nautobot-server post_upgrade`) to run migrations and clear any cache: + +```shell +nautobot-server post_upgrade +``` + +Then restart (if necessary) the Nautobot services which may include: + +- Nautobot +- Nautobot Workers +- Nautobot Scheduler + +```shell +sudo systemctl restart nautobot nautobot-worker nautobot-scheduler +``` + +## App Configuration + +### AWS + +#### Authentication + +No configuration is needed within Nautobot for this provider to operate. Instead you must provide [AWS credentials in one of the methods supported by the `boto3` library](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials). + +Boto3 credentials can be configured in multiple ways (eight as of this writing) that are mirrored here: + +1. Passing credentials as parameters in the `boto.client()` method +2. Passing credentials as parameters when creating a Session object +3. Environment variables +4. Shared credential file (`~/.aws/credentials`) +5. AWS config file (`~/.aws/config`) +6. Assume Role provider +7. Boto2 config file (`/etc/boto.cfg` and `~/.boto`) +8. Instance metadata service on an Amazon EC2 instance that has an IAM role configured. + +**The AWS providers only support methods 3-8. Methods 1 and 2 ARE NOT SUPPORTED at this time.** + +We highly recommend you defer to using environment variables for your deployment as specified in the credentials documentation linked above. The values specified in the linked documentation should be [set within your `~.bashrc`](https://nautobot.readthedocs.io/en/latest/installation/nautobot/#update-the-nautobot-bashrc) (or similar profile) on your system. + +#### Configuration + +This is an example based on our recommended deployment pattern in the section above (item 3) that is using [environment variables](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#environment-variables). You will need to set these in the environment prior to starting Nautobot: + +```no-highlight +export AWS_ACCESS_KEY_ID=foo # The access key for your AWS account. +export AWS_SECRET_ACCESS_KEY=bar # The secret key for your AWS account. +``` + +Please refer to the [Nautobot documentation on updating your `.bashrc`](https://nautobot.readthedocs.io/en/latest/installation/nautobot/#update-the-nautobot-bashrc) for how to do this for production Nautobot deployments. + +### HashiCorp Vault + +#### Configuration + +You must provide a mapping in `PLUGINS_CONFIG` within your `nautobot_config.py`, for example: + +```python +PLUGINS_CONFIG = { + "nautobot_secrets_providers": { + "hashicorp_vault": { + "url": "http://localhost:8200", + "auth_method": "token", + "token": os.getenv("NAUTOBOT_HASHICORP_VAULT_TOKEN"), + } + }, +} +``` + +- `url` - (required) The URL to the HashiCorp Vault instance (e.g. `http://localhost:8200`). +- `auth_method` - (optional / defaults to "token") The method used to authenticate against the HashiCorp Vault instance. Either `"approle"`, `"aws"`, `"kubernetes"` or `"token"`. For information on using AWS authentication with vault see the [authentication](#authentication) section above. +- `ca_cert` - (optional) Path to a PEM formatted CA certificate to use when verifying the Vault connection. Can alternatively be set to `False` to ignore SSL verification (not recommended) or `True` to use the system certificates. +- `default_mount_point` - (optional / defaults to "secret") The default mount point of the K/V Version 2 secrets engine within Hashicorp Vault. +- `kv_version` - (optional / defaults to "v2") The version of the KV engine to use, can be `v1` or `v2` +- `k8s_token_path` - (optional) Path to the kubernetes service account token file. Defaults to "/var/run/secrets/kubernetes.io/serviceaccount/token". +- `token` - (optional) Required when `"auth_method": "token"` or `auth_method` is not supplied. The token for authenticating the client with the HashiCorp Vault instance. As with other sensitive service credentials, we recommend that you provide the token value as an environment variable and retrieve it with `{"token": os.getenv("NAUTOBOT_HASHICORP_VAULT_TOKEN")}` rather than hard-coding it in your `nautobot_config.py`. +- `role_name` - (optional) Required when `"auth_method": "kubernetes"`, optional when `"auth_method": "aws"`. The Vault Kubernetes role or Vault AWS role to assume which the pod's service account has access to. +- `role_id` - (optional) Required when `"auth_method": "approle"`. As with other sensitive service credentials, we recommend that you provide the role_id value as an environment variable and retrieve it with `{"role_id": os.getenv("NAUTOBOT_HASHICORP_VAULT_ROLE_ID")}` rather than hard-coding it in your `nautobot_config.py`. +- `secret_id` - (optional) Required when `"auth_method": "approle"`.As with other sensitive service credentials, we recommend that you provide the secret_id value as an environment variable and retrieve it with `{"secret_id": os.getenv("NAUTOBOT_HASHICORP_VAULT_SECRET_ID")}` rather than hard-coding it in your `nautobot_config.py`. +- `login_kwargs` - (optional) Additional optional parameters to pass to the login method for [`approle`](https://hvac.readthedocs.io/en/stable/source/hvac_api_auth_methods.html#hvac.api.auth_methods.AppRole.login), [`aws`](https://hvac.readthedocs.io/en/stable/source/hvac_api_auth_methods.html#hvac.api.auth_methods.Aws.iam_login) and [`kubernetes`](https://hvac.readthedocs.io/en/stable/source/hvac_api_auth_methods.html#hvac.api.auth_methods.Kubernetes.login) authentication methods. +- `namespace` - (optional) Namespace to use for the [`X-Vault-Namespace` header](https://github.com/hvac/hvac/blob/main/hvac/adapters.py#L287) on all hvac client requests. Required when the [`Namespaces`](https://developer.hashicorp.com/vault/docs/enterprise/namespaces#usage) feature is enabled in Vault Enterprise. + +### Delinea/Thycotic Secret Server (TSS) + +The Delinea/Thycotic Secret Server app includes two providers: + +#### Thycotic Secret Server by ID + +This provider uses the `Secret ID` to specifiy the secret that is selected. The `Secret ID` is displayed in the browser's URL field if you `Edit` the data in Thycotic Secret Server. + +!!! example + The url is: _https://pw.example.local/SecretServer/app/#/secret/**1234**/general_ + In this example the value for `Secret ID` is **1234**. + +#### Thycotic Secret Server by Path + +This provider allows to select the secret by folder-path and secret-name. The path delimiter is a '\\'. The `Secret path` is displayed as page header when `Edit` a secret. + +!!! example + The header is: **NET-Automation > Nautobot > My-Secret** + In this example the value for `Secret path` is **`\NET-Automation\Nautobot\My-Secret`**. + +#### Configuration + +```python +PLUGINS_CONFIG = { + "nautobot_secrets_providers": { + "thycotic": { # https://github.com/thycotic/python-tss-sdk + "base_url": os.getenv("SECRET_SERVER_BASE_URL", None), + "ca_bundle_path": os.getenv("REQUESTS_CA_BUNDLE", None), + "cloud_based": is_truthy(os.getenv("SECRET_SERVER_IS_CLOUD_BASED", "False")), + "domain": os.getenv("SECRET_SERVER_DOMAIN", None), + "password": os.getenv("SECRET_SERVER_PASSWORD", None), + "tenant": os.getenv("SECRET_SERVER_TENANT", None), + "token": os.getenv("SECRET_SERVER_TOKEN", None), + "username": os.getenv("SECRET_SERVER_USERNAME", None), + }, + } +} +``` + +- `base_url` - (required) The Secret Server base_url. _e.g.'https://pw.example.local/SecretServer'_ +- `ca_bundle_path` - (optional) When using self-signed certificates this variable must be set to a file containing the trusted certificates (in .pem format). _e.g. '/etc/ssl/certs/ca-bundle.trust.crt'_. +- `cloud_based` - (optional) Set to "True" if Secret Server Cloud should be used. (Default: "False"). +- `domain` - (optional) Required for 'Domain Authorization' +- `password` - (optional) Required for 'Secret Server Cloud', 'Password Authorization', 'Domain Authorization'. +- `tenant` - (optional) Required for 'Domain Authorization'. +- `token` - (optional) Required for 'Access Token Authorization'. +- `username` - (optional) Required for 'Secret Server Cloud', 'Password Authorization', 'Domain Authorization'. diff --git a/docs/dev/contributing.md b/docs/dev/contributing.md new file mode 100644 index 0000000..855cc71 --- /dev/null +++ b/docs/dev/contributing.md @@ -0,0 +1,72 @@ +# Contributing to the App + +The project is packaged with a light [development environment](dev_environment.md) based on `docker-compose` to help with the local development of the project and to run tests. + +The project is following Network to Code software development guidelines and is leveraging the following: + +- Python linting and formatting: `black`, `pylint`, `bandit`, `flake8`, and `ruff`. +- YAML linting is done with `yamllint`. +- Django unit test to ensure the app is working properly. + +Documentation is built using [mkdocs](https://www.mkdocs.org/). The [Docker based development environment](dev_environment.md#docker-development-environment) automatically starts a container hosting a live version of the documentation website on [http://localhost:8001](http://localhost:8001) that auto-refreshes when you make any changes to your local files. + +## Creating Changelog Fragments + +All pull requests to `next` or `develop` must include a changelog fragment file in the `./changes` directory. To create a fragment, use your GitHub issue number and fragment type as the filename. For example, `2362.added`. Valid fragment types are `added`, `changed`, `deprecated`, `fixed`, `removed`, and `security`. The change summary is added to the file in plain text. Change summaries should be complete sentences, starting with a capital letter and ending with a period, and be in past tense. Each line of the change fragment will generate a single change entry in the release notes. Use multiple lines in the same file if your change needs to generate multiple release notes in the same category. If the change needs to create multiple entries in separate categories, create multiple files. + +!!! example + + **Wrong** + ```plaintext title="changes/1234.fixed" + fix critical bug in documentation + ``` + + **Right** + ```plaintext title="changes/1234.fixed" + Fixed critical bug in documentation. + ``` + +!!! example "Multiple Entry Example" + + This will generate 2 entries in the `fixed` category and one entry in the `changed` category. + + ```plaintext title="changes/1234.fixed" + Fixed critical bug in documentation. + Fixed release notes generation. + ``` + + ```plaintext title="changes/1234.changed" + Changed release notes generation. + ``` + +## Branching Policy + +The branching policy includes the following tenets: + +- The `develop` branch is the branch of the next major and minor paired version planned. +- PRs intended to add new features should be sourced from the `develop` branch. +- PRs intended to fix issues in the Nautobot LTM compatible release should be sourced from the latest `ltm-` branch instead of `develop`. + +Nautobot's Secrets Providers App will observe semantic versioning, as of 1.0. This may result in a quick turnaround in minor versions to keep pace with an ever growing feature set. + +## Release Policy + +Nautobot's Secrets Providers App has currently no intended scheduled release schedule, and will release new features in minor versions. + +When a new release, from `develop` to `main`, is created the following should happen. + +- A release PR is created from `develop` with: + - Update the release notes in `docs/admin/release_notes/version_..md` file to reflect the changes. + - Change the version from `..-beta` to `..` in `pyproject.toml`. + - Set the PR to the `main` branch. +- Ensure the tests for the PR pass. +- Merge the PR. +- Create a new tag: + - The tag should be in the form of `v..`. + - The title should be in the form of `v..`. + - The description should be the changes that were added to the `version_..md` document. +- If merged into `main`, then push from `main` to `develop`, in order to retain the merge commit created when the PR was merged +- A post release PR is created with: + - Change the version from `..` to `..-beta` in both `pyproject.toml` and `nautobot.__init__.__version__`. + - Set the PR to the proper branch, `develop`. + - Once tests pass, merge. diff --git a/docs/dev/dev_environment.md b/docs/dev/dev_environment.md index 98b2b80..6dbec8c 100644 --- a/docs/dev/dev_environment.md +++ b/docs/dev/dev_environment.md @@ -29,21 +29,18 @@ Using **Invoke** these configuration options can be overridden using [several me This project is managed by [Python Poetry](https://python-poetry.org/) and has a few requirements to setup your development environment: -1. Install Poetry, see the [Poetry documentation](https://python-poetry.org/docs/#installation) for your operating system. +1. Install Poetry, see the [Poetry Documentation](https://python-poetry.org/docs/#installation) for your operating system. 2. Install Docker, see the [Docker documentation](https://docs.docker.com/get-docker/) for your operating system. -3. Install Docker-compose, see the [Docker-compose documentation](https://github.com/docker/compose) for your operation system. -Once you have Poetry and Docker installed you can run the following commands (in the root of the repository) to install all other development dependencies in an isolated Python virtual environment: +Once you have Poetry and Docker installed you can run the following commands to install all other development dependencies in an isolated python virtual environment: ```shell poetry shell poetry install -cp development/creds.example.env development/creds.env -invoke build invoke start ``` -The Nautobot server can now be accessed at [http://localhost:8080](http://localhost:8080) and the live documentation at [http://localhost:8001](http://localhost:8001). +Nautobot server can now be accessed at [http://localhost:8080](http://localhost:8080). To either stop or destroy the development environment use the following options. @@ -52,7 +49,9 @@ To either stop or destroy the development environment use the following options. ### Local Poetry Development Environment -- Create an `invoke.yml` file with the following contents at the root of the repo and edit as necessary +1. Copy `development/creds.example.env` to `development/creds.env` (This file will be ignored by Git and Docker) +2. Uncomment the `POSTGRES_HOST`, `REDIS_HOST`, and `NAUTOBOT_ROOT` variables in `development/creds.env` +3. Create an `invoke.yml` file with the following contents at the root of the repo (you can also `cp invoke.example.yml invoke.yml` and edit as necessary): ```yaml --- @@ -60,21 +59,20 @@ nautobot_secrets_providers: local: true ``` -Run the following commands: +3. Run the following commands: ```shell poetry shell poetry install --extras nautobot -export $(cat development/development.env | xargs) +export $(cat development/dev.env | xargs) export $(cat development/creds.env | xargs) invoke start && sleep 5 nautobot-server migrate ``` -!!! note - If you want to develop on the latest develop branch of Nautobot, run the following command: `poetry add --optional git+https://github.com/nautobot/nautobot@develop`. After the `@` symbol must match either a branch or a tag. +> If you want to develop on the latest develop branch of Nautobot, run the following command: `poetry add --optional git+https://github.com/nautobot/nautobot@develop`. After the `@` symbol must match either a branch or a tag. -You can now run `nautobot-server` commands as you would from the [Nautobot documentation](https://nautobot.readthedocs.io/en/latest/) for example to start the development server: +4. You can now run nautobot-server commands as you would from the [Nautobot documentation](https://nautobot.readthedocs.io/en/latest/) for example to start the development server: ```shell nautobot-server runserver 0.0.0.0:8080 --insecure @@ -132,6 +130,237 @@ Each command can be executed with `invoke `. All commands support the a unittest Run Django unit tests for the app. ``` +### Developing Against Secrets Backends + +#### AWS Secrets Manager + +This assumes you are logged into the AWS Console. + +- Navigate to AWS Console +- Navigate to AWS Secrets Manager +- Click "Store a new secret" + - Select “Other type of secrets” + - Use Secret key/value + - Enter `hello=world` + - Use "DefaultEncryptionKey" for now + - Click "Next" + - Under "Secret name" fill out `hello` + - Click "Next" + - Under "Configure automatic rotation" + - Leave it as "Disable automatic rotation" + - On "Store a new secret" + - Copy the sample code (see below) + - Click "Store" +- END + +##### Install the AWS CLI + +Next, install the [AWS CLI](https://aws.amazon.com/cli/). + +On MacOS, this can be done using `brew install awscli`: + +``` +brew install awscli +``` + +On Linux, you will need to run a `curl` command (This assumes x86. Please see the docs for [AWS CLI on +Linux](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) for ARM and other options): + +``` +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip awscliv2.zip +sudo ./aws/install +``` + +##### Configure the AWS CLI + +After installing the AWS CLI, you will need to configure it for authentication. + +You may use an existing AWS access key or create a new one. For these instructions we cover the need to create a new access key that can be used for this. + +- Navigate to AWS Console +- Click your username + - Click "My security credentials" + - Click "create access key" +- Save your "Access key ID" and "Secret access key" for use when configuring the AWS CLI + +Now configure the CLI: + +- Run `aws configure` +- Enter your credentials from above +- Choose your region +- Use output format: `json` + +Example: + +```no-highlight +$ aws configure +AWS Access Key ID [None]: AKIAIOSFODNN7EXAMPLE +AWS Secret Access Key [None]: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +Default region name [None]: us-east-2 +Default output format [None]: json +``` + +Now you are ready to use the sample code to retrieve your secret from AWS Secrets Manager! + +##### Sample Code + +Make sure that the `boto3` client is installed: + +```no-highlight +poetry install --extras aws +``` + +Next, save this as `aws_secrets.py`: + +```python +# Use this code snippet in your app. +# If you need more information about configurations or implementing the sample code, visit the AWS docs: +# https://aws.amazon.com/developers/getting-started/python/ + +import boto3 +import base64 +from botocore.exceptions import ClientError + + +def get_secret(): + + secret_name = "hello" + region_name = "us-east-2" + + # Create a Secrets Manager client + session = boto3.session.Session() + client = session.client( + service_name='secretsmanager', + region_name=region_name + ) + + # In this sample we only handle the specific exceptions for the 'GetSecretValue' API. + # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html + # We rethrow the exception by default. + + try: + get_secret_value_response = client.get_secret_value( + SecretId=secret_name + ) + except ClientError as e: + if e.response['Error']['Code'] == 'DecryptionFailureException': + # Secrets Manager can't decrypt the protected secret text using the provided KMS key. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response['Error']['Code'] == 'InternalServiceErrorException': + # An error occurred on the server side. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response['Error']['Code'] == 'InvalidParameterException': + # You provided an invalid value for a parameter. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response['Error']['Code'] == 'InvalidRequestException': + # You provided a parameter value that is not valid for the current state of the resource. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response['Error']['Code'] == 'ResourceNotFoundException': + # We can't find the resource that you asked for. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + else: + # Decrypts secret using the associated KMS CMK. + # Depending on whether the secret is a string or binary, one of these fields will be populated. + if 'SecretString' in get_secret_value_response: + secret = get_secret_value_response['SecretString'] + else: + decoded_binary_secret = base64.b64decode(get_secret_value_response['SecretBinary']) + + # Your code goes here. + +# ^ Above was generated by AWS. + +# This was added by us so you can run this as a script: +if __name__ == "__main__": + secret = get_secret() + print(f"Secret = {secret}") +``` + +Run it with `python aws_secrets.py`: + +``` +$ python aws_secrets.py +Secret = {"hello":"world"}. +``` + +Note that this blob is JSON and will also need to be decoded if you want to extract the value. + +#### HashiCorp Vault + +Make sure that the `hvac` client is installed: + +```no-highlight +poetry install --extras hashicorp +``` + +##### Start Services with Docker + +```no-highlight +invoke start +``` + +##### Set an alias to work with `vault` + +This will allow you to easily run the CLI command from within the container: + +```no-highlight +alias vault="docker exec -it nautobot_secrets_providers_vault_1 vault" +``` + +Interact with the Vault vi CLI (via `docker exec` into the container from localhost): + +```no-highlight +$ vault status +Key Value +--- ----- +Seal Type shamir +Initialized true +Sealed false +Total Shares 1 +Threshold 1 +Version 1.8.2 +Storage Type inmem +Cluster Name vault-cluster-35c5d319 +Cluster ID 2611f99c-a6de-a883-1fcc-bfffdc0217bc +HA Enabled false +``` + +##### Using the Python `hvac` Library + +This establishes a client, creates a basic key/value secret (`hello=world`) at the path `hello`, and then retrieves the data from the `hello` key at the secret path `hello`. + +> This is equivalent to the command `vault kv get -field hello secret/hello`. + +```python +In [1]: import hvac + +In [2]: client = hvac.Client(url="http://localhost:8200", token="nautobot") + +In [3]: client.secrets.kv.create_or_update_secret(path="hello", secret=dict(hello="world")) +Out[3]: +{'request_id': 'c4709868-c08f-4cb1-ab8c-605c58b82f3f', + 'lease_id': '', + 'renewable': False, + 'lease_duration': 0, + 'data': {'created_time': '2021-09-16T23:21:07.5564132Z', + 'deletion_time': '', + 'destroyed': False, + 'version': 2}, + 'wrap_info': None, + 'warnings': None, + 'auth': None} + +In [4]: client.secrets.kv.read_secret(path="hello")["data"]["data"]["hello"] +Out[4]: 'world' +``` + + ## Project Overview This project provides the ability to develop and manage the Nautobot server locally (with supporting services being *Dockerized*) or by using only Docker containers to manage Nautobot. The main difference between the two environments is the ability to debug and use **pdb** when developing locally. Debugging with **pdb** within the Docker container is more complicated, but can still be accomplished by either entering into the container (via `docker exec`) or attaching your IDE to the container and running the Nautobot service manually within the container. @@ -470,3 +699,21 @@ To run an individual test, you can run any or all of the following: ➜ invoke ruff ➜ invoke pylint ``` + +### App Configuration Schema + +In the package source, there is the `nautobot_secrets_providers/app-config-schema.json` file, conforming to the [JSON Schema](https://json-schema.org/) format. This file is used to validate the configuration of the app in CI pipelines. + +If you make changes to `PLUGINS_CONFIG` or the configuration schema, you can run the following command to validate the schema: + +```bash +invoke validate-app-config +``` + +To generate the `app-config-schema.json` file based on the current `PLUGINS_CONFIG` configuration, run the following command: + +```bash +invoke generate-app-config-schema +``` + +This command can only guess the schema, so it's up to the developer to manually update the schema as needed. diff --git a/docs/index.md b/docs/index.md index dbfdad4..6399752 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,17 +1,35 @@ -# NautobotSecretsProviders +# Nautobot Secrets Providers App -TODO: Write app documentation, the outline here is provided as a guide and should be expanded upon. If more detail is required you are encouraged to expand on the table of contents (TOC) in `mkdocs.yml` to add additional pages. +This document provides an overview of the App including critical information and important considerations when applying it to your Nautobot environment. + +!!! note + Throughout this documentation, the terms "app" and "plugin" will be used interchangeably. ## Description -## Installation +Nautobot Secrets Providers is an app for [Nautobot](https://github.com/nautobot/nautobot) 1.2.1 or higher that bundles Secrets Providers for integrating with popular secrets backends. Nautobot 1.2.0 added support for integrating with retrieving secrets from various secrets providers. + +This app publishes secrets providers that are not included in the Nautobot core software package so that it will be easier to maintain and extend support for various secrets providers without waiting on Nautobot software releases. + +### Supported Secrets Backends + +This app supports the following popular secrets backends: + +| Secrets Backend | Supported Secret Types | Supported Authentication Methods | +| ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | +| [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/) | [Other: Key/value pairs](https://docs.aws.amazon.com/secretsmanager/latest/userguide/manage_create-basic-secret.html) | [AWS credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html) (see Usage section below) | +| [AWS Systems Manager Parameter Store](https://aws.amazon.com/secrets-manager/) | [Other: Key/value pairs](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html) | [AWS credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html) (see Usage section below) | +| [HashiCorp Vault](https://www.vaultproject.io) | [K/V Version 2](https://www.vaultproject.io/docs/secrets/kv/kv-v2)
[K/V Version 1](https://developer.hashicorp.com/vault/docs/secrets/kv/kv-v1) | [Token](https://www.vaultproject.io/docs/auth/token)
[AppRole](https://www.vaultproject.io/docs/auth/approle)
[AWS](https://www.vaultproject.io/docs/auth/aws)
[Kubernetes](https://www.vaultproject.io/docs/auth/kubernetes) | +| [Delinea/Thycotic Secret Server](https://delinea.com/products/secret-server) | [Secret Server Cloud](https://github.com/DelineaXPM/python-tss-sdk#secret-server-cloud)
[Secret Server (on-prem)](https://github.com/DelineaXPM/python-tss-sdk#initializing-secretserver)| [Access Token Authorization](https://github.com/DelineaXPM/python-tss-sdk#access-token-authorization)
[Domain Authorization](https://github.com/DelineaXPM/python-tss-sdk#domain-authorization)
[Password Authorization](https://github.com/DelineaXPM/python-tss-sdk#password-authorization)
| + +## Audience (User Personas) - Who should use this App? -## Configuration +- Anyone who needs to retrieve secrets from Hashicorp Vault, AWS, or Delinea/Thycotic and use them in Nautobot -## Usage +## Authors and Maintainers -## API +- Nautobot Core Team -## Views +## Nautobot Features Used -## Models +- Secrets Providers diff --git a/mkdocs.yml b/mkdocs.yml index ad67a3c..39f03bf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,7 +81,11 @@ markdown_extensions: anchor_linenums: true - "pymdownx.inlinehilite" - "pymdownx.snippets" - - "pymdownx.superfences" + - "pymdownx.superfences": + custom_fences: + - name: "mermaid" + class: "mermaid" + format: !!python/name:pymdownx.superfences.fence_code_format - "footnotes" plugins: - "search" @@ -98,3 +102,11 @@ watch: nav: - Overview: "index.md" + - Administrator Guide: + - Install and Configure: "admin/install.md" + - Uninstall: "admin/uninstall.md" + - Compatibility Matrix: "admin/compatibility_matrix.md" + - Developer Guide: + - Contributing to the App: "dev/contributing.md" + - Development Environment: "dev/dev_environment.md" + - Nautobot Docs Home ↗︎: "https://docs.nautobot.com" diff --git a/nautobot_secrets_providers/app-config-schema.json b/nautobot_secrets_providers/app-config-schema.json new file mode 100644 index 0000000..98df0b5 --- /dev/null +++ b/nautobot_secrets_providers/app-config-schema.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/nautobot/nautobot-app-secrets-providers/develop/nautobot_secrets_providers/app-config-schema.json", + "type": "object", + "properties": { + "hashicorp_vault": { + "type": "object", + "properties": { + "auth_method": { + "type": "string", + "default": "token", + "oneOf": [ + "approle", + "aws", + "kubernetes", + "token" + ] + }, + "ca_cert": { + "type": ["string", "boolean"] + }, + "default_kv_version": { + "type": "string", + "default": "v2", + "oneOf": ["v1", "v2"] + }, + "default_mount_point": { + "type": "string", + "default": "secret" + }, + "k8s_token_path": { + "type": "string", + "default": "/var/run/secrets/kubernetes.io/serviceaccount/token" + }, + "kv_version": { + "type": "string", + "default": "v2", + "oneOf": ["v1", "v2"] + }, + "login_kwargs": { + "type": "object" + }, + "namespace": { + "type": "string" + }, + "role_id": { + "type": "string" + }, + "role_name": { + "type": "string" + }, + "secret_id": { + "type": "string" + }, + "token": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": ["url"] + }, + "thycotic": { + "type": "object", + "properties": { + "base_url": { + "type": "string" + }, + "ca_bundle_path": { + "type": "string" + }, + "cloud_based": { + "type": "boolean" + }, + "domain": { + "type": "string" + }, + "password": { + "type": "string" + }, + "tenant": { + "type": "string" + }, + "token": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "required": ["base_url"] + } + } +} diff --git a/poetry.lock b/poetry.lock index f254005..99fe40d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "amqp" @@ -1713,6 +1713,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -2730,6 +2740,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2737,8 +2748,16 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2755,6 +2774,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2762,6 +2782,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2971,23 +2992,22 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"] [[package]] name = "responses" -version = "0.23.3" +version = "0.25.0" description = "A utility library for mocking out the `requests` Python library." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "responses-0.23.3-py3-none-any.whl", hash = "sha256:e6fbcf5d82172fecc0aa1860fd91e58cbfd96cee5e96da5b63fa6eb3caa10dd3"}, - {file = "responses-0.23.3.tar.gz", hash = "sha256:205029e1cb334c21cb4ec64fc7599be48b859a0fd381a42443cdd600bfe8b16a"}, + {file = "responses-0.25.0-py3-none-any.whl", hash = "sha256:2f0b9c2b6437db4b528619a77e5d565e4ec2a9532162ac1a131a83529db7be1a"}, + {file = "responses-0.25.0.tar.gz", hash = "sha256:01ae6a02b4f34e39bffceb0fc6786b67a25eae919c6368d05eabc8d9576c2a66"}, ] [package.dependencies] pyyaml = "*" requests = ">=2.30.0,<3.0" -types-PyYAML = "*" urllib3 = ">=1.25.10,<3.0" [package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-PyYAML", "types-requests"] [[package]] name = "rich" @@ -3317,6 +3337,20 @@ files = [ {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, ] +[[package]] +name = "to-json-schema" +version = "1.0.1" +description = "" +optional = false +python-versions = "*" +files = [ + {file = "to_json_schema-1.0.1-py3-none-any.whl", hash = "sha256:5708663f1c81815e4ff01fce910ac32ee3964d0c6b3587fd4fff2e38d5c9aa7b"}, + {file = "to_json_schema-1.0.1.tar.gz", hash = "sha256:ec747bd5129256dd571105f66a7bc9a4546228cd5e5fbf5e06dc9776e255400e"}, +] + +[package.extras] +testing = ["pytest", "pytest-cov", "setuptools"] + [[package]] name = "toml" version = "0.10.2" @@ -3387,17 +3421,6 @@ files = [ docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<7.5)", "pytest-mock", "pytest-mypy-testing"] -[[package]] -name = "types-pyyaml" -version = "6.0.12.12" -description = "Typing stubs for PyYAML" -optional = false -python-versions = "*" -files = [ - {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"}, - {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"}, -] - [[package]] name = "typing-extensions" version = "4.8.0" @@ -3510,13 +3533,13 @@ files = [ [[package]] name = "werkzeug" -version = "2.3.7" +version = "3.0.1" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.8" files = [ - {file = "werkzeug-2.3.7-py3-none-any.whl", hash = "sha256:effc12dba7f3bd72e605ce49807bbe692bd729c3bb122a3b91747a6ae77df528"}, - {file = "werkzeug-2.3.7.tar.gz", hash = "sha256:2b8c0e447b4b9dbcc85dd97b6eeb4dcbaf6c8b6c3be0bd654e25553e0a2157d8"}, + {file = "werkzeug-3.0.1-py3-none-any.whl", hash = "sha256:90a285dc0e42ad56b34e696398b8122ee4c681833fb35b8334a095d82c56da10"}, + {file = "werkzeug-3.0.1.tar.gz", hash = "sha256:507e811ecea72b18a404947aded4b3390e1db8f826b494d76550ef45bb3b1dcc"}, ] [package.dependencies] @@ -3663,4 +3686,4 @@ thycotic = ["python-tss-sdk"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "8bd5c274369bdf4572b41d07b7961cd14ef9fc0120ede7c974fbc132dfd14572" +content-hash = "778aa31d640036f5e9b0cdda10add84f8fa425aeea14126070ba1227b88932be" diff --git a/pyproject.toml b/pyproject.toml index ce7463e..0797950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ license = "Apache-2.0" readme = "README.md" homepage = "https://github.com/nautobot/nautobot-app-secrets-providers" repository = "https://github.com/nautobot/nautobot-app-secrets-providers" +documentation = "https://docs.nautobot.com/projects/secrets-providers/en/latest/" keywords = ["nautobot", "nautobot-app", "nautobot-plugin"] classifiers = [ "Intended Audience :: Developers", @@ -57,6 +58,8 @@ mkdocs-version-annotations = "1.0.0" mkdocstrings = "0.22.0" mkdocstrings-python = "1.5.2" towncrier = "~23.6.0" +to-json-schema = "*" +jsonschema = "*" moto = "~2.2.11" pyOpenSSL = "23.2.0" requests-mock = "1.9.3" @@ -173,7 +176,7 @@ requires = ["poetry_core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.towncrier] -package = "nautobot" +package = "nautobot_secrets_providers" directory = "changes" filename = "docs/admin/release_notes/version_X.Y.md" template = "development/towncrier_template.j2" diff --git a/tasks.py b/tasks.py index c257d86..6c08350 100644 --- a/tasks.py +++ b/tasks.py @@ -149,15 +149,27 @@ def docker_compose(context, command, **kwargs): def run_command(context, command, **kwargs): """Wrapper to run a command locally or inside the nautobot container.""" if is_truthy(context.nautobot_secrets_providers.local): + if "command_env" in kwargs: + kwargs["env"] = { + **kwargs.get("env", {}), + **kwargs.pop("command_env"), + } context.run(command, **kwargs) else: # Check if nautobot is running, no need to start another nautobot container to run a command docker_compose_status = "ps --services --filter status=running" results = docker_compose(context, docker_compose_status, hide="out") if "nautobot" in results.stdout: - compose_command = f"exec nautobot {command}" + compose_command = "exec" else: - compose_command = f"run --rm --entrypoint '{command}' nautobot" + compose_command = "run --rm --entrypoint=''" + + if "command_env" in kwargs: + command_env = kwargs.pop("command_env") + for key, value in command_env.items(): + compose_command += f' --env="{key}={value}"' + + compose_command += f" -- nautobot {command}" pty = kwargs.pop("pty", True) @@ -327,15 +339,22 @@ def logs(context, service="", follow=False, tail=0): # ------------------------------------------------------------------------------ # ACTIONS # ------------------------------------------------------------------------------ -@task(help={"file": "Python file to execute"}) -def nbshell(context, file=""): +@task( + help={ + "file": "Python file to execute", + "env": "Environment variables to pass to the command", + "plain": "Flag to run nbshell in plain mode (default: False)", + }, +) +def nbshell(context, file="", env={}, plain=False): """Launch an interactive nbshell session.""" command = [ "nautobot-server", "nbshell", + "--plain" if plain else "", f"< '{file}'" if file else "", ] - run_command(context, " ".join(command), pty=not bool(file)) + run_command(context, " ".join(command), pty=not bool(file), command_env=env) @task @@ -672,7 +691,7 @@ def pylint(context): def autoformat(context): """Run code autoformatting.""" black(context, autoformat=True) - ruff(context, action="both", fix=True) + ruff(context, fix=True) @task( @@ -800,8 +819,33 @@ def tests(context, failfast=False, keepdb=False, lint_only=False): pylint(context) print("Running mkdocs...") build_and_check_docs(context) + print("Checking app config schema...") + validate_app_config(context) if not lint_only: print("Running unit tests...") unittest(context, failfast=failfast, keepdb=keepdb) unittest_coverage(context) print("All tests have passed!") + + +@task +def generate_app_config_schema(context): + """Generate the app config schema from the current app config. + + WARNING: Review and edit the generated file before committing. + + Its content is inferred from: + + - The current configuration in `PLUGINS_CONFIG` + - `NautobotAppConfig.default_settings` + - `NautobotAppConfig.required_settings` + """ + start(context, service="nautobot") + nbshell(context, file="development/app_config_schema.py", env={"APP_CONFIG_SCHEMA_COMMAND": "generate"}) + + +@task +def validate_app_config(context): + """Validate the app config based on the app config schema.""" + start(context, service="nautobot") + nbshell(context, plain=True, file="development/app_config_schema.py", env={"APP_CONFIG_SCHEMA_COMMAND": "validate"})