From 56df3279a601afa227cd1e5767eb96bedc3939d6 Mon Sep 17 00:00:00 2001 From: kurokobo Date: Fri, 24 May 2024 02:40:51 +0900 Subject: [PATCH] feat: implement extra_settings_files (#1836) * feat: implement extra_settings_files * fix: reduce duplicated code blocks by templates * docs: update docs for extra settings * docs: simplify the commands * docs: add notes for duplicated keys in setting files --- config/crd/bases/awx.ansible.com_awxs.yaml | 22 ++++ .../awx-operator.clusterserviceversion.yaml | 5 + .../custom-volume-and-volume-mount-options.md | 48 +++---- .../advanced-configuration/extra-settings.md | 123 +++++++++++++++--- roles/installer/defaults/main.yml | 2 + .../extra_settings_files.yaml.j2 | 16 +++ .../volumes/extra_settings_files.yaml.j2 | 20 +++ .../cronjobs/metrics-utility-gather.yaml.j2 | 2 + .../cronjobs/metrics-utility-report.yaml.j2 | 2 + .../templates/deployments/task.yaml.j2 | 4 + .../templates/deployments/web.yaml.j2 | 3 + .../templates/jobs/migration.yaml.j2 | 2 + 12 files changed, 199 insertions(+), 50 deletions(-) create mode 100644 roles/installer/templates/common/volume_mounts/extra_settings_files.yaml.j2 create mode 100644 roles/installer/templates/common/volumes/extra_settings_files.yaml.j2 diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index cced65bc2..5b0a5c0f5 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -1904,6 +1904,28 @@ spec: x-kubernetes-preserve-unknown-fields: true type: object type: array + extra_settings_files: + description: Extra ConfigMaps or Secrets of settings files to specify for AWX + properties: + configmaps: + items: + properties: + name: + type: string + key: + type: string + type: object + type: array + secrets: + items: + properties: + name: + type: string + key: + type: string + type: object + type: array + type: object no_log: description: Configure no_log for no_log tasks type: boolean diff --git a/config/manifests/bases/awx-operator.clusterserviceversion.yaml b/config/manifests/bases/awx-operator.clusterserviceversion.yaml index 268dba6c7..d1f599136 100644 --- a/config/manifests/bases/awx-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/awx-operator.clusterserviceversion.yaml @@ -966,6 +966,11 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:hidden + - displayName: Extra Settings Files + path: extra_settings_files + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:hidden - displayName: No Log Configuration path: no_log x-descriptors: diff --git a/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md b/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md index ef0ae75aa..d3f2a99b1 100644 --- a/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md +++ b/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md @@ -1,4 +1,4 @@ -#### Custom Volume and Volume Mount Options +# Custom Volume and Volume Mount Options In a scenario where custom volumes and volume mounts are required to either overwrite defaults or mount configuration files. @@ -12,7 +12,6 @@ In a scenario where custom volumes and volume mounts are required to either over | init_container_extra_volume_mounts | Specify volume mounts to be added to Init container | '' | | init_container_extra_commands | Specify additional commands for Init container | '' | - !!! warning The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec). @@ -31,10 +30,8 @@ data: remote_tmp = /tmp [ssh_connection] ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s - custom.py: | - INSIGHTS_URL_BASE = "example.org" - AWX_CLEANUP_PATHS = True ``` + Example spec file for volumes and volume mounts ```yaml @@ -49,13 +46,6 @@ spec: - key: ansible.cfg path: ansible.cfg name: -extra-config - - name: custom-py - configMap: - defaultMode: 420 - items: - - key: custom.py - path: custom.py - name: -extra-config - name: shared-volume persistentVolumeClaim: claimName: my-external-volume-claim @@ -73,24 +63,13 @@ spec: - name: ansible-cfg mountPath: /etc/ansible/ansible.cfg subPath: ansible.cfg - - web_extra_volume_mounts: | - - name: custom-py - mountPath: /etc/tower/conf.d/custom.py - subPath: custom.py - - task_extra_volume_mounts: | - - name: custom-py - mountPath: /etc/tower/conf.d/custom.py - subPath: custom.py - - name: shared-volume - mountPath: /shared ``` !!! warning **Volume and VolumeMount names cannot contain underscores(_)** -##### Custom UWSGI Configuration +## Custom UWSGI Configuration + We allow the customization of two UWSGI parameters: * [processes](https://uwsgi-docs.readthedocs.io/en/latest/Options.html#processes) with `uwsgi_processes` (default 5) @@ -110,7 +89,7 @@ requests (more than 128) tend to come in a short period of time, but can all be handled before any other time outs may apply. Also see related nginx configuration. -##### Custom Nginx Configuration +## Custom Nginx Configuration Using the [extra_volumes feature](#custom-volume-and-volume-mount-options), it is possible to extend the nginx.conf. @@ -131,13 +110,13 @@ may allow the web pods to handle more "bursty" request patterns if many requests (more than 128) tend to come in a short period of time, but can all be handled before any other time outs may apply. Also see related uwsgi configuration. + * [worker_processes](http://nginx.org/en/docs/ngx_core_module.html#worker_processes) with `nginx_worker_processes` (default of 1) * [worker_cpu_affinity](http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity) with `nginx_worker_cpu_affinity` (default "auto") * [worker_connections](http://nginx.org/en/docs/ngx_core_module.html#worker_connections) with `nginx_worker_connections` (minimum of 1024) * [listen](https://nginx.org/en/docs/http/ngx_http_core_module.html#listen) with `nginx_listen_queue_size` (default same as uwsgi listen queue size) - -##### Custom Logos +## Custom Logos You can use custom volume mounts to mount in your own logos to be displayed instead of the AWX logo. There are two different logos, one to be displayed on page headers, and one for the login screen. @@ -145,8 +124,8 @@ There are two different logos, one to be displayed on page headers, and one for First, create configmaps for the logos from local `logo-login.svg` and `logo-header.svg` files. ```bash -$ kubectl create configmap logo-login-configmap --from-file logo-login.svg -$ kubectl create configmap logo-header-configmap --from-file logo-header.svg +kubectl create configmap logo-login-configmap --from-file logo-login.svg +kubectl create configmap logo-header-configmap --from-file logo-header.svg ``` Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec @@ -179,15 +158,14 @@ spec: subPath: logo-header.svg ``` - -##### Custom Favicon +## Custom Favicon You can also use custom volume mounts to mount in your own favicon to be displayed in your AWX browser tab. First, create the configmap from a local `favicon.ico` file. ```bash -$ kubectl create configmap favicon-configmap --from-file favicon.ico +kubectl create configmap favicon-configmap --from-file favicon.ico ``` Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec @@ -209,3 +187,7 @@ spec: mountPath: /var/lib/awx/public/static/media/favicon.ico subPath: favicon.ico ``` + +## Custom AWX Configuration + +Refer to the [Extra Settings](./extra-settings.md) documentation for customizing the AWX configuration. diff --git a/docs/user-guide/advanced-configuration/extra-settings.md b/docs/user-guide/advanced-configuration/extra-settings.md index 319cdfd28..021665bd5 100644 --- a/docs/user-guide/advanced-configuration/extra-settings.md +++ b/docs/user-guide/advanced-configuration/extra-settings.md @@ -1,30 +1,119 @@ -#### Extra Settings +# Extra Settings -With`extra_settings`, you can pass multiple custom settings via the `awx-operator`. The parameter `extra_settings` will be appended to the `/etc/tower/settings.py` and can be an alternative to the `extra_volumes` parameter. +With `extra_settings` and `extra_settings_files`, you can pass multiple custom settings to AWX via the AWX Operator. -| Name | Description | Default | -| -------------- | -------------- | ------- | -| extra_settings | Extra settings | '' | +!!! note + Parameters configured in `extra_settings` or `extra_settings_files` are set as read-only settings in AWX. As a result, they cannot be changed in the UI after deployment. -**Note:** Parameters configured in `extra_settings` are set as read-only settings in AWX. As a result, they cannot be changed in the UI after deployment. If you need to change the setting after the initial deployment, you need to change it on the AWX CR spec. + If you need to change the setting after the initial deployment, you need to change it on the AWX CR spec (for `extra_settings`) or corresponding ConfigMap or Secret (for `extra_settings_files`). After updating ConfigMap or Secret, you need to restart the AWX pods to apply the changes. + +!!! note + If the same setting is set in both `extra_settings` and `extra_settings_files`, the setting in `extra_settings_files` will take precedence. + +## Add extra settings with `extra_settings` + +You can pass extra settings by specifying the pair of the setting name and value as the `extra_settings` parameter. + +The settings passed via `extra_settings` will be appended to the `/etc/tower/settings.py`. + +| Name | Description | Default | +| -------------- | -------------- | --------- | +| extra_settings | Extra settings | `[]` | Example configuration of `extra_settings` parameter ```yaml - spec: - extra_settings: - - setting: MAX_PAGE_SIZE - value: "500" +spec: + extra_settings: + - setting: MAX_PAGE_SIZE + value: "500" - - setting: AUTH_LDAP_BIND_DN - value: "cn=admin,dc=example,dc=com" + - setting: AUTH_LDAP_BIND_DN + value: "cn=admin,dc=example,dc=com" - - setting: LOG_AGGREGATOR_LEVEL - value: "'DEBUG'" + - setting: LOG_AGGREGATOR_LEVEL + value: "'DEBUG'" ``` Note for some settings, such as `LOG_AGGREGATOR_LEVEL`, the value may need double quotes. -!!! tip - Alternatively, you can pass any additional settings by mounting ConfigMaps or Secrets of the python files (`*.py`) that contain custom settings to under `/etc/tower/conf.d/` in the web and task pods. - See the example of `custom.py` in the [Custom Volume and Volume Mount Options](custom-volume-and-volume-mount-options.md) section. +## Add extra settings with `extra_settings_files` + +You can pass extra settings by specifying the additional settings files in the ConfigMaps or Secrets as the `extra_settings_files` parameter. + +The settings files passed via `extra_settings_files` will be mounted as the files under the `/etc/tower/conf.d`. + +| Name | Description | Default | +| -------------------- | -------------------- | --------- | +| extra_settings_files | Extra settings files | `{}` | + +!!! note + If the same setting is set in multiple files in `extra_settings_files`, it would be difficult to predict which would be adopted since these files are loaded in arbitrary order that [`glob`](https://docs.python.org/3/library/glob.html) returns. For a reliable setting, do not include the same key in more than one file. + +Create ConfigMaps or Secrets that contain custom settings files (`*.py`). + +```python title="custom_job_settings.py" +AWX_TASK_ENV = { + "HTTPS_PROXY": "http://proxy.example.com:3128", + "HTTP_PROXY": "http://proxy.example.com:3128", + "NO_PROXY": "127.0.0.1,localhost,.example.com" +} +GALAXY_TASK_ENV = { + "ANSIBLE_FORCE_COLOR": "false", + "GIT_SSH_COMMAND": "ssh -o StrictHostKeyChecking=no", +} +``` + +```python title="custom_system_settings.py" +REMOTE_HOST_HEADERS = [ + "HTTP_X_FORWARDED_FOR", + "REMOTE_ADDR", + "REMOTE_HOST", +] +``` + +```python title="custom_passwords.py" +SUBSCRIPTIONS_PASSWORD = "my-super-secure-subscription-password123!" +REDHAT_PASSWORD = "my-super-secure-redhat-password123!" +``` + +```bash title="Create ConfigMap and Secret" +# Create ConfigMap +kubectl create configmap my-custom-settings \ + --from-file /PATH/TO/YOUR/custom_job_settings.py \ + --from-file /PATH/TO/YOUR/custom_system_settings.py + +# Create Secret +kubectl create secret generic my-custom-passwords \ + --from-file /PATH/TO/YOUR/custom_passwords.py +``` + +Then specify them in the AWX CR spec. Here is an example configuration of `extra_settings_files` parameter. + +```yaml +spec: + extra_settings_files: + configmaps: + - name: my-custom-settings # The name of the ConfigMap + key: custom_job_settings.py # The key in the ConfigMap, which means the file name + - name: my-custom-settings + key: custom_system_settings.py + secrets: + - name: my-custom-passwords # The name of the Secret + key: custom_passwords.py # The key in the Secret, which means the file name +``` + +!!! Warning "Restriction" + There are some restrictions on the ConfigMaps or Secrets used in `extra_settings_files`. + + - The keys in ConfigMaps or Secrets MUST be the name of python files and MUST end with `.py` + - The keys in ConfigMaps or Secrets MUST consists of alphanumeric characters, `-`, `_` or `.` + - The keys in ConfigMaps or Secrets are converted to the following strings, which MUST not exceed 63 characters + - Keys in ConfigMaps: `--configmap` + - Keys in Secrets: `--secret` + - Following keys are reserved and MUST NOT be used in ConfigMaps or Secrets + - `credentials.py` + - `execution_environments.py` + - `ldap.py` + + Refer to the Kubernetes documentations ([[1]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/config-map-v1/), [[2]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/secret-v1/), [[3]](https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/volume/), [[4]](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/)) for more information about character types and length restrictions. diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 303b987e1..4efe94c39 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -491,3 +491,5 @@ nginx_worker_processes: 1 nginx_worker_connections: "{{ uwsgi_listen_queue_size }}" nginx_worker_cpu_affinity: 'auto' nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}" + +extra_settings_files: {} diff --git a/roles/installer/templates/common/volume_mounts/extra_settings_files.yaml.j2 b/roles/installer/templates/common/volume_mounts/extra_settings_files.yaml.j2 new file mode 100644 index 000000000..2f8dbdce8 --- /dev/null +++ b/roles/installer/templates/common/volume_mounts/extra_settings_files.yaml.j2 @@ -0,0 +1,16 @@ +{% if extra_settings_files.configmaps is defined and extra_settings_files.configmaps | length %} +{% for configmap in extra_settings_files.configmaps %} +- name: {{ ansible_operator_meta.name }}-{{ configmap.key | replace('_', '-') | replace('.', '-') | lower }}-configmap + mountPath: "/etc/tower/conf.d/{{ configmap.key }}" + subPath: {{ configmap.key }} + readOnly: true +{% endfor %} +{% endif %} +{% if extra_settings_files.secrets is defined and extra_settings_files.secrets | length %} +{% for secret in extra_settings_files.secrets %} +- name: {{ ansible_operator_meta.name }}-{{ secret.key | replace('_', '-') | replace('.', '-') | lower }}-secret + mountPath: "/etc/tower/conf.d/{{ secret.key }}" + subPath: {{ secret.key }} + readOnly: true +{% endfor %} +{% endif %} diff --git a/roles/installer/templates/common/volumes/extra_settings_files.yaml.j2 b/roles/installer/templates/common/volumes/extra_settings_files.yaml.j2 new file mode 100644 index 000000000..f0d8146df --- /dev/null +++ b/roles/installer/templates/common/volumes/extra_settings_files.yaml.j2 @@ -0,0 +1,20 @@ +{% if extra_settings_files.configmaps is defined and extra_settings_files.configmaps | length %} +{% for configmap in extra_settings_files.configmaps %} +- name: {{ ansible_operator_meta.name }}-{{ configmap.key | replace('_', '-') | replace('.', '-') | lower }}-configmap + configMap: + name: {{ configmap.name }} + items: + - key: {{ configmap.key }} + path: {{ configmap.key }} +{% endfor %} +{% endif %} +{% if extra_settings_files.secrets is defined and extra_settings_files.secrets | length %} +{% for secret in extra_settings_files.secrets %} +- name: {{ ansible_operator_meta.name }}-{{ secret.key | replace('_', '-') | replace('.', '-') | lower }}-secret + secret: + secretName: {{ secret.name }} + items: + - key: {{ secret.key }} + path: {{ secret.key }} +{% endfor %} +{% endif %} diff --git a/roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2 b/roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2 index 3867ae942..b78770229 100644 --- a/roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2 +++ b/roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2 @@ -67,6 +67,7 @@ spec: mountPath: /etc/tower/settings.py subPath: settings.py readOnly: true + {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2") | indent(width=12) | trim }} volumes: - name: {{ ansible_operator_meta.name }}-metrics-utility persistentVolumeClaim: @@ -90,4 +91,5 @@ spec: items: - key: settings path: settings.py + {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2") | indent(width=10) | trim }} restartPolicy: OnFailure \ No newline at end of file diff --git a/roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2 b/roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2 index abfd0f33e..a7c53b2ee 100644 --- a/roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2 +++ b/roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2 @@ -64,6 +64,7 @@ spec: mountPath: /etc/tower/settings.py subPath: settings.py readOnly: true + {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2") | indent(width=12) | trim }} volumes: - name: {{ ansible_operator_meta.name }}-metrics-utility persistentVolumeClaim: @@ -87,4 +88,5 @@ spec: items: - key: settings path: settings.py + {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2") | indent(width=10) | trim }} restartPolicy: OnFailure \ No newline at end of file diff --git a/roles/installer/templates/deployments/task.yaml.j2 b/roles/installer/templates/deployments/task.yaml.j2 index 29da6f511..a785cb6d8 100644 --- a/roles/installer/templates/deployments/task.yaml.j2 +++ b/roles/installer/templates/deployments/task.yaml.j2 @@ -95,6 +95,7 @@ spec: mountPath: "/etc/tower/settings.py" subPath: settings.py readOnly: true + {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2") | indent(width=12) | trim }} {% if development_mode | bool %} - name: awx-devel mountPath: "/awx_devel" @@ -279,6 +280,7 @@ spec: mountPath: /etc/tower/settings.py subPath: settings.py readOnly: true + {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2") | indent(width=12) | trim }} - name: {{ ansible_operator_meta.name }}-redis-socket mountPath: "/var/run/redis" - name: rsyslog-socket @@ -428,6 +430,7 @@ spec: mountPath: "/etc/tower/settings.py" subPath: settings.py readOnly: true + {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2") | indent(width=12) | trim }} - name: {{ ansible_operator_meta.name }}-redis-socket mountPath: "/var/run/redis" - name: rsyslog-socket @@ -588,6 +591,7 @@ spec: items: - key: redis_conf path: redis.conf + {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2") | indent(width=8) | trim }} - name: {{ ansible_operator_meta.name }}-redis-socket emptyDir: {} - name: {{ ansible_operator_meta.name }}-redis-data diff --git a/roles/installer/templates/deployments/web.yaml.j2 b/roles/installer/templates/deployments/web.yaml.j2 index 64b32177a..115bbcbd6 100644 --- a/roles/installer/templates/deployments/web.yaml.j2 +++ b/roles/installer/templates/deployments/web.yaml.j2 @@ -231,6 +231,7 @@ spec: mountPath: /etc/tower/settings.py subPath: settings.py readOnly: true + {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2") | indent(width=12) | trim }} - name: {{ ansible_operator_meta.name }}-nginx-conf mountPath: /etc/nginx/nginx.conf subPath: nginx.conf @@ -307,6 +308,7 @@ spec: mountPath: "/etc/tower/settings.py" subPath: settings.py readOnly: true + {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2") | indent(width=12) | trim }} - name: {{ ansible_operator_meta.name }}-redis-socket mountPath: "/var/run/redis" - name: rsyslog-socket @@ -438,6 +440,7 @@ spec: items: - key: redis_conf path: redis.conf + {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2") | indent(width=8) | trim }} - name: {{ ansible_operator_meta.name }}-uwsgi-config configMap: name: {{ ansible_operator_meta.name }}-{{ deployment_type }}-configmap diff --git a/roles/installer/templates/jobs/migration.yaml.j2 b/roles/installer/templates/jobs/migration.yaml.j2 index becb02412..b3ad759e5 100644 --- a/roles/installer/templates/jobs/migration.yaml.j2 +++ b/roles/installer/templates/jobs/migration.yaml.j2 @@ -29,6 +29,7 @@ spec: mountPath: "/etc/tower/settings.py" subPath: settings.py readOnly: true + {{ lookup("template", "common/volume_mounts/extra_settings_files.yaml.j2") | indent(width=12) | trim }} {% if development_mode | bool %} - name: awx-devel mountPath: "/awx_devel" @@ -94,6 +95,7 @@ spec: items: - key: settings path: settings.py + {{ lookup("template", "common/volumes/extra_settings_files.yaml.j2") | indent(width=8) | trim }} {% if development_mode | bool %} - name: awx-devel hostPath: