Merge branch 'main' into mep-no-apache

.gitignore

@@ -9,6 +9,9 @@
 # Ansible files
 *.retry
 
+# ignore roles installed from ansible galaxy
+roles/geerlingguy.supervisor
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -19,35 +19,21 @@ The overall structure of this repository can be broken down as follows:
 
 ### Requirements
   - Python virtual environment.
-    - See `.python-version` for the recommended version of Python.
+    - See `.python-version` for the recommended version of Python; we recommend [pyenv](https://github.com/pyenv/pyenv) for managing python versions.
     - If you use `env` or `venv`, the `.gitignore` will exclude it.
+    - Install python dependencies: `pip install -r requirements.txt`
 
-  -  Install required Ansible galaxy collections:
+  -  Install required Ansible galaxy collections and roles:
       - `ansible-galaxy install -r requirements.yml`
 
-  - The CDH Ansible vault key. This can be referenced on the command line, but it is
-  recommende to set it as an environment variable; e.g., for BASH
-   `export ANSIBLE_VAULT_PASSWORD_FILE=/path/to/.passwd`
+  - The CDH Ansible vault keys are stored in LastPass. You need to be added to the appropriate LastPass share and install [lastpass-cli](https://github.com/lastpass/lastpass-cli).  There are two command-line scripts in the `bin/` directory to call `lpass` to retrieve the vault keys, and the default configuration is set in `ansible.cfg`. See below for more details on the vault setup.
   - A GitHub [personal access token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) for any playbook that uses the `create_deployment` and `close_deployment` tasks. You can set this as an environment variable
   as `ANSIBLE_GITHUB_TOKEN` or pass it on the command line as `-e github_token=`
-  - The CDH deploy bot key. This can be added to ssh-agent or in `~/.ssh/config`. All production deploys must be on the campus network (including VPN) and proxy through the QA server to production, with an ssh config stanza that looks something like:
-  ```
-  Host derridas-margins.princeton.edu
-      User deploy
-      Proxycommand ssh deploy@QASERVERHOST -W %h:%p
-      Identityfile ~/.ssh/key_for_qa_server
-  ```
-
-  And for deploying to the QA server:
-  ```
-  Host test-*.cdh.princeton.edu
-      User deploy
-      Identityfile ~/.ssh/key_for_qa_server
-  ```
+
 
 ### Precommit hook
 
-If you plan to contribute to this repository, you should install the configured pre-commit hooks:
+If you plan to contribute to this repository, you should install the configured pre-commit hooks. (If you installed python dependencies, pre-commit should already be installed)
 
 ```{bash}
 pre-commit install
@@ -77,16 +63,19 @@ The playbook will run, noting success and failures. The `-v` flag adjusts verbos
 
 ### Skip setup tasks
 
-By default, initial provisioning and setup tasks are configured to be skipped.  Tasks or groups of tasks should be tagged as `setup`.
+By default, initial provisioning and setup tasks are configured to be skipped.  Tasks or groups of tasks should be tagged with both `setup` and `never`.
 
-To run playbook without skipping setup tasks, override the default skip tag configuration:
+To run playbook without skipping setup tasks, pass the `setup` and `all` tags, so untagged tasks and tasks tagged `setup` run:
 
 ```{bash}
-env ANSIBLE_SKIP_TAGS= ansible-playbook playbooks/name_of_playbook.yml
+ansible-playbook --tags=all,setup playbooks/name_of_playbook.yml
 ```
 
-Note that `--skip-tags=[]` doesn't work because the skip tags setting in
-`ansible.cfg` takes precedence over command line options.
+### Skip deployment tasks
+
+By default, most playbooks create a [GitHub deployment](https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28), which is used to track which version of the code is deployed to which environment. Through Slack/GitHub integration, GitHub deployments can be used to notify team members when a deploy is taking place and whether or not it succeeds or fails.  (This can be very noisy when working on or troubleshooting a deploy.)
+
+The tasks for creating and closing the GitHub deployment are tagged with `gh_deploy`. If you want to run a playbook without deploying code, pass `--skip-tags gh_deploy`.
 
 ## Pause before finalizing deploy
 
@@ -100,7 +89,7 @@ ansible-playbook --tags=all,final-pause playbooks/playbook.yml
 When running with setup steps enabled:
 
 ```{bash}
-env ANSIBLE_SKIP_TAGS= ansible-playbook --tags=all,final-pause playbooks/playbook.yml
+ansible-playbook --tags=all,setup,final-pause playbooks/playbook.yml
 ```
 
 ## Revert last deploy
@@ -170,12 +159,12 @@ ansible-playbook playbooks/replicate.yml --limit=geniza_qa
 ```
 
 Currently replication consists of:
-- dumping the production database, restoring it to qa, and running 
+- dumping the production database, restoring it to qa, and running
   django migrations in the current deploy
 - update django sites in the database to match the qa environment
 - backing up and restoring any user-uploaded media files and setting correct ownership and permissions
 
-Replication does not yet include restoring Solr indexing or support replication to dev environments. 
+Replication does not yet include restoring Solr indexing or support replication to dev environments.
 
 ### Setting up replication for a new project
 
@@ -219,4 +208,3 @@ GENIZA_DEPLOY_ONLY=1
 ```
 
 Note that you will not be able to run setup tasks or decrypt setup vault secrets.
-

ansible.cfg

@@ -10,6 +10,3 @@ vault_identity_list=default@bin/lpass_default.sh,geniza@bin/lpass_geniza.sh
 
 [ssh-connection]
 pipelining=true
-
-[tags]
-skip=setup

group_vars/all/vars.yml

@@ -90,8 +90,8 @@ csp_enforce_uri: '{{ vault_csp_enforce_uri  | default("") }}'
 django_csp_reportonly_uri: '{{ vault_csp_reportonly_uri | default("") }}'
 django_csp_enforce_uri: '{{ vault_csp_enforce_uri | default("") }}'
 
-# Postgres configuration (default version is 13)
-postgres_host: lib-postgres-prod3.princeton.edu
+# Postgres configuration (version 15)
+postgres_host: lib-postgres-prod1.princeton.edu
 # postgres admin user
 postgres_admin_user: postgres
 

group_vars/cdhweb/vars.yml

@@ -17,7 +17,7 @@ python_app: "{{ app_name }}"
 django_app: "{{ app_name }}"
 
 # python version
-python_version: "3.8"  # can't go past 3.8 until we get to newer ubuntu
+python_version: "3.8" # can't go past 3.8 until we get to newer ubuntu
 # symlink to current deploy in /var/www
 symlink: cdhweb
 # use PUL deploy user of conan
@@ -37,13 +37,13 @@ application_db_name: "{{ vault_db_name }}"
 application_db_host: "{{ postgres_host }}"
 
 # configure app-specific local settings
-django_local_settings_template: "cdhweb_settings.py"
+django_local_settings_template: "cdhweb_settings.py.j2"
 
 # Database backup location — revert to default, with timestamp but not app version
 # db_backup_path: "/home/{{ deploy_user }}/backups/pre-{{ version }}-{{ short_hash }}.sql"
 # app-specific system dependencies
 app_dependencies:
-  - python-opencv  # for wagtail image feature detection
+  - python-opencv # for wagtail image feature detection
 
 passenger_app_root: "/var/www/{{ app_name }}"
 passenger_server_name: "cdh.princeton.edu"
@@ -65,4 +65,3 @@ dest_backup_path: "{{ dest_backup_dir }}{{ db_backup_filename }}"
 passenger_extra_config: |
   rewrite "^/media/uploads/(.*/)?([^/]+\.(jpg|jpeg|gif|png))$" "https://{{ passenger_server_name }}/media/original_images/$2" permanent;
   rewrite "^/grants/(.*)$" "https://{{ passenger_server_name }}/engage/grants/$1" permanent;
-

group_vars/geniza/vars.yml

@@ -25,7 +25,7 @@ clone_root: "/home/{{ deploy_user }}/repos"
 # for Geniza, we don't distinguish between qa/prod paths
 install_root: "/srv/www/{{ app_name }}"
 # geniza local settings have a slightly different path
-django_local_settings_dest: "{{ deploy }}/{{ django_app }}/settings/local_settings.py"
+django_local_settings_dest: "{{ deploy }}/{{ django_app }}/settings/local_settings.py.j2"
 # configure app-specific local settings
 django_local_settings_template: "geniza_settings.py"
 # uploaded media directory; must end with trailing slash for nginx
@@ -107,11 +107,11 @@ font_require_referrer:
   - geniza.cdh.princeton.edu
   - geniza.princeton.edu
   - test-geniza.cdh.princeton.edu
-  - "*.percy.io"   # for visual testing
+  - "*.percy.io" # for visual testing
 
 # override webpack build command
-webpack_build_qa: 'build'
-webpack_build_prod: 'build'
+webpack_build_qa: "build"
+webpack_build_prod: "build"
 
 # solr settings
 solr_collection: geniza
@@ -134,10 +134,10 @@ crontab:
     state: absent
     # disable until TEI migration is complete
   - name: "{{ app_name }} sync annotation export"
-    minute: "*/5"   # run every five minutes
+    minute: "*/5" # run every five minutes
     job: "env DJANGO_ENV=staging bin/cron-wrapper {{ passenger_app_root }}/env/bin/python{{ python_version }} {{ passenger_app_root }}/manage.py sync_annotation_export -v 0 >> {{ logging_dir }}/sync_export.log 2>&1"
     state: present
   - name: "{{ app_name }} metadata export "
-    minute: "*/30"   # run every thirty minutes for now (footnote export takes ~10 min)
+    minute: "*/30" # run every thirty minutes for now (footnote export takes ~10 min)
     job: "env DJANGO_ENV=staging bin/cron-wrapper {{ passenger_app_root }}/env/bin/python{{ python_version }} {{ passenger_app_root }}/manage.py export_metadata -ws -v 0 >> {{ logging_dir }}/sync_metadata_export.log 2>&1"
     state: present

group_vars/prosody/vars.yml

@@ -3,7 +3,7 @@
 ###
 
 # Github repository
-repo: 'Princeton-CDH/ppa-django'
+repo: "Princeton-CDH/ppa-django"
 app_name: ppa
 # name of django application
 python_app: "{{ app_name }}"
@@ -32,7 +32,6 @@ passenger_server_name: "prosody.princeton.edu"
 passenger_startup_file: "{{ app_name }}/wsgi.py"
 passenger_python: "{{ passenger_app_root }}/env/bin/python"
 
-
 # base data dir — easier if NOT under conan home directory
 data_path: "/srv/www/data"
 # path to hathitree pairtree data
@@ -44,7 +43,6 @@ technical_contact: [email protected]
 # flag that csp is enabled
 csp_enabled: true
 
-
 # django database backend
 db_backend: "postgresql"
 db_host: "{{ postgres_host }}"
@@ -56,7 +54,7 @@ application_db_name: cdh_ppa
 application_db_host: "{{ postgres_host }}"
 
 # app-specific local settings
-django_local_settings_template: "prosody_settings.py"
+django_local_settings_template: "prosody_settings.py.j2"
 
 # - solr settings
 solr_collection: cdh_ppa
@@ -67,9 +65,6 @@ solr_configset: cdh_ppa
 # #   - "Python unit tests"
 #     - "Python unit tests (3.6, 8.6, postgresql)"
 
-# default media root
-anag
 # app-specific system dependencies
 app_dependencies:
   - libmysqlclient-dev # for mysql installation during the migration
-

group_vars/qa/vars.yml

@@ -12,8 +12,8 @@ solr_admin_url: '{{ solr_url }}admin/cores'
 qa: qa
 # allow any deploy contexts
 deploy_contexts: []
-# PUL staging VM (default version is postgres 13)
-postgres_host: lib-postgres-staging3.princeton.edu
+# PUL staging VM (postgres v15)
+postgres_host: lib-postgres-staging1.princeton.edu
 
 # in QA, grant postgres app user permission to create db for replication
 application_dbuser_role_attr_flags: "CREATEDB"

group_vars/sandbox/simrisk.yml

@@ -14,13 +14,15 @@ gitref: '{{ ref | default("main") }}'
 
 # use python 3.9
 python_version: "3.9"
-# for simulatingrisk, don't differentiate qa/prod paths
+# where to install the application code
 install_root: '/srv/www/{{ app_name }}'
 
+
 # configure to run via supervisor
+# runs at a non top-level path when on the sandbox
 supervisor_programs:
   - name: 'simulatingrisk'
-    command: "{{ install_root }}/current/env/bin/solara run --host 0.0.0.0 --port 8765 --root-path /{{ app_name }} {{ install_root }}/current/simulatingrisk/app.py"
+    command: "{{ install_root }}/current/env/bin/solara run --host 0.0.0.0 --port 8765 {% if inventory_hostname in groups['sandbox'] %}--root-path /{{ app_name }}{% endif %} {{ install_root }}/current/simulatingrisk/app.py"
     state: present
     configuration: |
       directory={{ install_root }}/current

group_vars/sandbox/vars.yml

@@ -0,0 +1,6 @@
+# unlike most deploys, we may put multiple apps on this vm
+single_app: false
+
+alias_apps:
+  simrisk: /srv/www/simulatingrisk
+

group_vars/shxco/vars.yml

@@ -2,7 +2,7 @@
 # Common variables for Shakespeare & Company Project django application
 ###
 # Github repository
-repo: 'Princeton-CDH/mep-django'
+repo: "Princeton-CDH/mep-django"
 # application name
 app_name: mep
 # name of django application
@@ -32,10 +32,10 @@ passenger_python: "{{ passenger_app_root }}/env/bin/python"
 deploy_user: "conan"
 
 # geonames and mapbox usernames, map-related configs
-geonames_username: '{{ vault_geonames_username }}'
-mapbox_token: '{{ vault_mapbox_token }}'
-mapbox_basemap: 'light-v10'
-paris_overlay: 'https://tiles.arcgis.com/tiles/4Ko8f1mCWFLyY4NV/arcgis/rest/services/Paris_1943/MapServer'
+geonames_username: "{{ vault_geonames_username }}"
+mapbox_token: "{{ vault_mapbox_token }}"
+mapbox_basemap: "light-v10"
+paris_overlay: "https://tiles.arcgis.com/tiles/4Ko8f1mCWFLyY4NV/arcgis/rest/services/Paris_1943/MapServer"
 
 # django database backend
 db_backend: "postgresql"
@@ -48,7 +48,7 @@ application_db_name: cdh_shxco
 application_db_host: "{{ postgres_host }}"
 
 # app-specific local settings
-django_local_settings_template: "shxco_settings.py"
+django_local_settings_template: "shxco_settings.py.j2"
 
 # - solr settings
 solr_collection: cdh_shxco
@@ -60,13 +60,13 @@ solr_configset: cdh_shxco
 #     - "Python unit tests (3.6, 8.6, postgresql)"
 
 # default media root
-media_root: '/srv/www/media/'
+media_root: "/srv/www/media/"
 
 technical_contact: [email protected]
 
 # app-specific system dependencies
 app_dependencies:
-  - at  # for 100 years twitter bot scheduled tweets
+  - at # for 100 years twitter bot scheduled tweets
   - libmysqlclient-dev # for mysql installation during the migration
 
 # 2to3 support needed for installing rdflib-jsonld

hosts

@@ -74,6 +74,10 @@ prosody_prod
 [sandbox]
 cdh-dev-sandbox1.princeton.edu
 
+# must be configured in ssh config with bastion proxy command
+[simrisk]
+simrisk
+
 ### PUL solr servers
 
 [solr_staging]
@@ -111,6 +115,7 @@ derrida_crawl_qa
 derrida_archive_qa
 prosody_qa
 sandbox
+simrisk
 
 [dev]
 localhost

playbooks/db_setup.yml

@@ -0,0 +1,33 @@
+# Run database setup and update application local settings.
+# The application must already be deployed.
+#
+# Use this playbook for database migrations to do database setup
+# (create new database and db user, ensure app server has access),
+# and then update the local settings to reference the new database.
+#
+# For proper behavior, you MUST specify setup and app_config tags
+# and skip db_backup:
+#
+#    ansible-playbook playbooks/db_setup.yml -t setup,app_config
+#
+# Use the --limit option to run on a single host or host group.
+#
+- hosts: geniza_qa, cdhweb_qa, prosody_qa, shxco_qa, geniza_prod, cdhweb_prod, prosody_prod, shxco_prod
+  connection: ssh
+  remote_user: pulsys
+  become: true
+  vars:
+    # update currently deployed config,
+    # instead of getting path from app version and git hash
+    deploy: "{{ install_root }}/current"
+  tasks:
+    - name: Run postgresql setup tasks (ensure access, create db and user)
+      ansible.builtin.include_role:
+        name: postgresql
+      tags: setup
+    - name: Update django local settings
+      # need to use import to get variables and path for template
+      # use app_config tag to only run local settings task
+      ansible.builtin.import_role:
+        name: django
+

playbooks/simulatingrisk_dev.yml

@@ -1,4 +1,4 @@
-- hosts: sandbox
+- hosts: sandbox, simrisk
   vars_files:
     - ../group_vars/sandbox/simrisk.yml
   connection: ssh