From 7709199199bdd03b2ba23c85580548c983a03c2f Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Thu, 29 Aug 2024 14:58:54 +1000 Subject: [PATCH 01/34] Add unit test for static webpages --- webapp/home/tests.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/webapp/home/tests.py b/webapp/home/tests.py index ffd85e45..1143147d 100644 --- a/webapp/home/tests.py +++ b/webapp/home/tests.py @@ -21,6 +21,20 @@ from webapp.test import TestCase TEST_DATA_DIR = Path(__file__).parent / 'test/data' + +# Endpoints to test for HTTP 200 +TEST_ENDPOINTS = [ + '/about', + '/request', + '/request/tool', + '/request/quota', + '/request/support', + '/request/access', + '/list-of-institutions.html', +] + + +# Galaxy Labs TEST_LAB_NAME = 'Antarctica' TEST_LAB_LAB_NAME = 'Galaxy Lab Pages'.upper() TEST_LAB_NATIONALITY = 'Antarctican' @@ -172,6 +186,11 @@ def test_aaf_webpage(self): 'University of Queensland' ) + def test_webpages(self): + for endpoint in TEST_ENDPOINTS: + response = self.client.get(endpoint) + self.assertLess(response.status_code, 300) + def test_utility_institution(self): assert institution.is_institution_email('johndoe@uq.edu.au') assert not institution.is_institution_email('johndoe@gmail.com') From 3b1f20a180428f737dd291bc48c6cfb02648612a Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Wed, 11 Sep 2024 11:44:43 +1000 Subject: [PATCH 02/34] Add hostname to slack errors --- webapp/utils/slack.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/webapp/utils/slack.py b/webapp/utils/slack.py index a4145816..f038095c 100644 --- a/webapp/utils/slack.py +++ b/webapp/utils/slack.py @@ -11,11 +11,15 @@ def post(message): key = os.environ.get("SLACK_API_KEY") user_id = os.environ.get("SLACK_MENTION_USER_ID") channel_id = os.environ.get("SLACK_CHANNEL_ID") + message = "" if key is None: return if user_id: - message = f'<@{user_id}> {message}' + message += f'<@{user_id}>' + + message += f' [hostname: {os.getenv('HOSTNAME')}]' + message += f' {message}' requests.post( SLACK_URL, From aa65fbe3ddb1b6f79f91846b421ad6ba34c9b9b0 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Wed, 11 Sep 2024 12:14:36 +1000 Subject: [PATCH 03/34] Fix syntax error --- webapp/utils/slack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/utils/slack.py b/webapp/utils/slack.py index f038095c..1cba9c19 100644 --- a/webapp/utils/slack.py +++ b/webapp/utils/slack.py @@ -18,7 +18,7 @@ def post(message): if user_id: message += f'<@{user_id}>' - message += f' [hostname: {os.getenv('HOSTNAME')}]' + message += f' [hostname: {os.getenv("HOSTNAME")}]' message += f' {message}' requests.post( From 56bf30ab957d5950c50f63db34d1961faa95e967 Mon Sep 17 00:00:00 2001 From: Patrick Capon <137233925+PatCapon39@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:19:05 +1000 Subject: [PATCH 04/34] Typo in about.html Underpined -> Underpinned --- webapp/home/templates/home/snippets/about/about.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/home/templates/home/snippets/about/about.html b/webapp/home/templates/home/snippets/about/about.html index a849553c..c9c07c91 100644 --- a/webapp/home/templates/home/snippets/about/about.html +++ b/webapp/home/templates/home/snippets/about/about.html @@ -81,7 +81,7 @@

-Galaxy Australia is underpined by computational resources provided by +Galaxy Australia is underpinned by computational resources provided by AARNet, ARDC Nectar Research Cloud, the University of Melbourne, From 99cbff52e3acaa9890a6103ea271e54104f5a64c Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Mon, 30 Sep 2024 06:56:14 +1000 Subject: [PATCH 05/34] Rename role webapp to galaxy_media_site --- deploy/ansible/.gitignore | 2 +- deploy/ansible/dev.yml | 7 +++++-- deploy/ansible/group_vars/webservers.yml | 4 ++-- deploy/ansible/host_vars/site.usegalaxy.org.au.yml | 1 - deploy/ansible/prod.yml | 7 +++++-- .../defaults/main.yml | 0 .../files/webapp.socket | 0 .../{webapp => galaxy_media_site}/tasks/main.yml | 14 +++++++++++--- .../templates/.env.j2 | 0 .../templates/gunicorn.py.j2 | 0 .../templates/nginx.vhost.j2 | 0 .../templates/setup.sh.j2 | 6 +++--- .../templates/update.sh.j2 | 0 .../templates/webapp.service.j2 | 0 14 files changed, 27 insertions(+), 14 deletions(-) delete mode 100644 deploy/ansible/host_vars/site.usegalaxy.org.au.yml rename deploy/ansible/roles/{webapp => galaxy_media_site}/defaults/main.yml (100%) rename deploy/ansible/roles/{webapp => galaxy_media_site}/files/webapp.socket (100%) rename deploy/ansible/roles/{webapp => galaxy_media_site}/tasks/main.yml (89%) rename deploy/ansible/roles/{webapp => galaxy_media_site}/templates/.env.j2 (100%) rename deploy/ansible/roles/{webapp => galaxy_media_site}/templates/gunicorn.py.j2 (100%) rename deploy/ansible/roles/{webapp => galaxy_media_site}/templates/nginx.vhost.j2 (100%) rename deploy/ansible/roles/{webapp => galaxy_media_site}/templates/setup.sh.j2 (100%) rename deploy/ansible/roles/{webapp => galaxy_media_site}/templates/update.sh.j2 (100%) rename deploy/ansible/roles/{webapp => galaxy_media_site}/templates/webapp.service.j2 (100%) diff --git a/deploy/ansible/.gitignore b/deploy/ansible/.gitignore index a4d3d49a..793cd85d 100644 --- a/deploy/ansible/.gitignore +++ b/deploy/ansible/.gitignore @@ -1,2 +1,2 @@ roles/* -!roles/webapp \ No newline at end of file +!roles/galaxy_media_site \ No newline at end of file diff --git a/deploy/ansible/dev.yml b/deploy/ansible/dev.yml index 2664227b..d3f28a96 100644 --- a/deploy/ansible/dev.yml +++ b/deploy/ansible/dev.yml @@ -6,7 +6,10 @@ - group_vars/secrets.yml - group_vars/webservers.yml roles: - - geerlingguy.nginx + - role: geerlingguy.nginx + tags: init - role: geerlingguy.postgresql become: true - - webapp + tags: init,postgresql + - role: galaxy_media_site + tags: gms diff --git a/deploy/ansible/group_vars/webservers.yml b/deploy/ansible/group_vars/webservers.yml index 42630787..b67da733 100644 --- a/deploy/ansible/group_vars/webservers.yml +++ b/deploy/ansible/group_vars/webservers.yml @@ -16,8 +16,8 @@ gunicorn: web_workers: 4 # Postgres database conf -postgres_db_name: webapp -postgres_db_user: webapp +postgres_db_name: gms +postgres_db_user: gms postgresql_locales: - 'en_US.UTF-8' diff --git a/deploy/ansible/host_vars/site.usegalaxy.org.au.yml b/deploy/ansible/host_vars/site.usegalaxy.org.au.yml deleted file mode 100644 index edaf9995..00000000 --- a/deploy/ansible/host_vars/site.usegalaxy.org.au.yml +++ /dev/null @@ -1 +0,0 @@ -hostname: usegalaxy-au.neoformit.com diff --git a/deploy/ansible/prod.yml b/deploy/ansible/prod.yml index 938695bb..8c74c6d3 100644 --- a/deploy/ansible/prod.yml +++ b/deploy/ansible/prod.yml @@ -6,7 +6,10 @@ - group_vars/secrets.yml - group_vars/webservers.yml roles: - - geerlingguy.nginx + - role: geerlingguy.nginx + tags: init - role: geerlingguy.postgresql become: true - - webapp + tags: init,postgresql + - role: galaxy_media_site + tags: gms diff --git a/deploy/ansible/roles/webapp/defaults/main.yml b/deploy/ansible/roles/galaxy_media_site/defaults/main.yml similarity index 100% rename from deploy/ansible/roles/webapp/defaults/main.yml rename to deploy/ansible/roles/galaxy_media_site/defaults/main.yml diff --git a/deploy/ansible/roles/webapp/files/webapp.socket b/deploy/ansible/roles/galaxy_media_site/files/webapp.socket similarity index 100% rename from deploy/ansible/roles/webapp/files/webapp.socket rename to deploy/ansible/roles/galaxy_media_site/files/webapp.socket diff --git a/deploy/ansible/roles/webapp/tasks/main.yml b/deploy/ansible/roles/galaxy_media_site/tasks/main.yml similarity index 89% rename from deploy/ansible/roles/webapp/tasks/main.yml rename to deploy/ansible/roles/galaxy_media_site/tasks/main.yml index ac5f719e..833654d8 100644 --- a/deploy/ansible/roles/webapp/tasks/main.yml +++ b/deploy/ansible/roles/galaxy_media_site/tasks/main.yml @@ -19,11 +19,19 @@ name: - gcc - python3.12 - - python3.12-pip - - python3.12-venv + - python3-pip + - python3-virtualenv - python3.12-dev - python3-certbot-nginx +- name: Grant database privileges to postgres_db_user + ansible.builtin.shell: > + sudo -u postgres psql -c + "GRANT ALL ON SCHEMA public TO {{ postgres_db_user }}; + GRANT USAGE, CREATE ON SCHEMA public TO {{ postgres_db_user }}; + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO gms;" + become: yes + - name: clone git repository for galaxy-content-site ansible.builtin.git: repo: https://github.com/neoformit/galaxy-content-site.git @@ -31,7 +39,7 @@ clone: yes force: yes -- name: link webapp to /srv/sites +- name: link webapp to web_root ansible.builtin.file: src: "{{ project_root }}/webapp" dest: "{{ web_root }}" diff --git a/deploy/ansible/roles/webapp/templates/.env.j2 b/deploy/ansible/roles/galaxy_media_site/templates/.env.j2 similarity index 100% rename from deploy/ansible/roles/webapp/templates/.env.j2 rename to deploy/ansible/roles/galaxy_media_site/templates/.env.j2 diff --git a/deploy/ansible/roles/webapp/templates/gunicorn.py.j2 b/deploy/ansible/roles/galaxy_media_site/templates/gunicorn.py.j2 similarity index 100% rename from deploy/ansible/roles/webapp/templates/gunicorn.py.j2 rename to deploy/ansible/roles/galaxy_media_site/templates/gunicorn.py.j2 diff --git a/deploy/ansible/roles/webapp/templates/nginx.vhost.j2 b/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 similarity index 100% rename from deploy/ansible/roles/webapp/templates/nginx.vhost.j2 rename to deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 diff --git a/deploy/ansible/roles/webapp/templates/setup.sh.j2 b/deploy/ansible/roles/galaxy_media_site/templates/setup.sh.j2 similarity index 100% rename from deploy/ansible/roles/webapp/templates/setup.sh.j2 rename to deploy/ansible/roles/galaxy_media_site/templates/setup.sh.j2 index e361f906..fd5bb481 100644 --- a/deploy/ansible/roles/webapp/templates/setup.sh.j2 +++ b/deploy/ansible/roles/galaxy_media_site/templates/setup.sh.j2 @@ -8,14 +8,14 @@ cd {{ project_root }}/webapp # Set Django settings export DJANGO_SETTINGS_MODULE=webapp.settings.prod +# Collect static files for Nginx +python manage.py collectstatic --noinput + {% if not skip_database_migration %} # Migrate database python manage.py migrate {% endif %} -# Collect static files for Nginx -python manage.py collectstatic --noinput - # Restart services sudo systemctl daemon-reload sudo systemctl enable webapp.service diff --git a/deploy/ansible/roles/webapp/templates/update.sh.j2 b/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 similarity index 100% rename from deploy/ansible/roles/webapp/templates/update.sh.j2 rename to deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 diff --git a/deploy/ansible/roles/webapp/templates/webapp.service.j2 b/deploy/ansible/roles/galaxy_media_site/templates/webapp.service.j2 similarity index 100% rename from deploy/ansible/roles/webapp/templates/webapp.service.j2 rename to deploy/ansible/roles/galaxy_media_site/templates/webapp.service.j2 From 7804b62ccc027228be9ff11adbaec9ab9dfde567 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Mon, 30 Sep 2024 06:57:46 +1000 Subject: [PATCH 06/34] Fix ansible postgres permissions --- deploy/ansible/group_vars/webservers.yml | 9 +++++---- deploy/ansible/roles/galaxy_media_site/tasks/main.yml | 6 ------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/deploy/ansible/group_vars/webservers.yml b/deploy/ansible/group_vars/webservers.yml index b67da733..1c36b911 100644 --- a/deploy/ansible/group_vars/webservers.yml +++ b/deploy/ansible/group_vars/webservers.yml @@ -30,6 +30,7 @@ postgresql_users: - name: "{{ postgres_db_user }}" password: "{{ postgres_db_password }}" db: "{{ postgres_db_name }}" + priv: ALL postgres_users_no_log: false @@ -37,16 +38,16 @@ postgres_users_no_log: false # ----------------------------------------------------------------------------- # Will git clone galaxy-content-site to here -project_root: /home/ubuntu/galaxy-content-site +project_root: /srv/sites/galaxy-media-site # This is where service files will be created -server_root: /home/ubuntu/serve +server_root: /srv/config # App will be served from here -web_root: /srv/webapp +web_root: "{{ project_root }}/webapp" # Virtual environment will be created here -venv_root: "{{ project_root }}/.venv" +venv_root: "{{ project_root }}/venv" # Admin user login for the admin portal admin_user: diff --git a/deploy/ansible/roles/galaxy_media_site/tasks/main.yml b/deploy/ansible/roles/galaxy_media_site/tasks/main.yml index 833654d8..456fa069 100644 --- a/deploy/ansible/roles/galaxy_media_site/tasks/main.yml +++ b/deploy/ansible/roles/galaxy_media_site/tasks/main.yml @@ -39,12 +39,6 @@ clone: yes force: yes -- name: link webapp to web_root - ansible.builtin.file: - src: "{{ project_root }}/webapp" - dest: "{{ web_root }}" - state: link - - name: create webapp media directory file: path: "{{ web_root }}/webapp/media" From 3d276443bccfe61817e8f354dfe253369d3625d0 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Mon, 30 Sep 2024 06:58:05 +1000 Subject: [PATCH 07/34] New dev_webservers host --- deploy/ansible/hosts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy/ansible/hosts b/deploy/ansible/hosts index 52e97fbb..621d02e8 100644 --- a/deploy/ansible/hosts +++ b/deploy/ansible/hosts @@ -1,5 +1,6 @@ [webservers] -site.usegalaxy.org.au ansible_connection=ssh ansible_user=ubuntu +site.usegalaxy.org.au ansible_connection=ssh ansible_user=ubuntu ansible_ssh_private_key_file=~/.ssh/galaxy [dev_webservers] -dev-site.gvl.org.au ansible_connection=ssh ansible_user=ubuntu +# dev-site.gvl.org.au ansible_connection=ssh ansible_user=ubuntu +gms.neoformit.com ansible_connection=ssh ansible_user=ubuntu ansible_ssh_private_key_file=~/.ssh/galaxy From b50902fb13f7445bb7da00c9183706cf3e6ec6a7 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Mon, 30 Sep 2024 06:58:47 +1000 Subject: [PATCH 08/34] Replace web_root with django_root and drop soft-linked web root --- deploy/ansible/README.md | 2 +- deploy/ansible/group_vars/webservers.yml | 2 +- deploy/ansible/group_vars/webservers.yml.sample | 2 +- .../roles/galaxy_media_site/defaults/main.yml | 2 +- .../ansible/roles/galaxy_media_site/tasks/main.yml | 14 +++++++------- .../galaxy_media_site/templates/nginx.vhost.j2 | 6 +++--- .../galaxy_media_site/templates/webapp.service.j2 | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index f5a501b0..649902ec 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -37,7 +37,7 @@ python -m pip install ansible - *Optional* - Host installation paths: - `project_root` - where this git repository will be cloned - `server_root` - where server configuration will be saved - - `web_root` - where the application will be served from + - `django_root` - where the application will be served from - `venv_root` - where the virtual env will be created diff --git a/deploy/ansible/group_vars/webservers.yml b/deploy/ansible/group_vars/webservers.yml index 1c36b911..a80ec2e6 100644 --- a/deploy/ansible/group_vars/webservers.yml +++ b/deploy/ansible/group_vars/webservers.yml @@ -44,7 +44,7 @@ project_root: /srv/sites/galaxy-media-site server_root: /srv/config # App will be served from here -web_root: "{{ project_root }}/webapp" +django_root: "{{ project_root }}/webapp" # Virtual environment will be created here venv_root: "{{ project_root }}/venv" diff --git a/deploy/ansible/group_vars/webservers.yml.sample b/deploy/ansible/group_vars/webservers.yml.sample index 8ec68d6c..22d63949 100644 --- a/deploy/ansible/group_vars/webservers.yml.sample +++ b/deploy/ansible/group_vars/webservers.yml.sample @@ -46,7 +46,7 @@ project_root: /home/ubuntu/galaxy-content-site server_root: /home/ubuntu/serve # App will be served from here -web_root: /srv/webapp +django_root: /srv/webapp # Virtual environment will be created here venv_root: "{{ project_root }}/.venv" diff --git a/deploy/ansible/roles/galaxy_media_site/defaults/main.yml b/deploy/ansible/roles/galaxy_media_site/defaults/main.yml index edcbd089..51612fa1 100644 --- a/deploy/ansible/roles/galaxy_media_site/defaults/main.yml +++ b/deploy/ansible/roles/galaxy_media_site/defaults/main.yml @@ -1,7 +1,7 @@ --- project_root: /home/ubuntu/galaxy-content-site server_root: /home/ubuntu/server -web_root: /srv/webapp +django_root: /srv/webapp venv_root: "{{ project_root }}/venv" webapp: diff --git a/deploy/ansible/roles/galaxy_media_site/tasks/main.yml b/deploy/ansible/roles/galaxy_media_site/tasks/main.yml index 456fa069..11a3e252 100644 --- a/deploy/ansible/roles/galaxy_media_site/tasks/main.yml +++ b/deploy/ansible/roles/galaxy_media_site/tasks/main.yml @@ -41,12 +41,12 @@ - name: create webapp media directory file: - path: "{{ web_root }}/webapp/media" + path: "{{ django_root }}/webapp/media" state: directory - name: create webapp logs directory file: - path: "{{ web_root }}/webapp/logs" + path: "{{ django_root }}/webapp/logs" state: directory - name: template webserver configuration @@ -82,7 +82,7 @@ - name: Ensure webapp superuser login shell: | - cd {{ web_root }} && \ + cd {{ django_root }} && \ export DJANGO_SUPERUSER_PASSWORD={{ admin_user.password }} && \ {{ venv_root }}/bin/python manage.py createsuperuser --noinput \ --email {{ admin_user.email }} \ @@ -100,10 +100,10 @@ group: ubuntu - name: update media file ownership - file: dest={{ web_root }}/webapp/media owner=www-data group=www-data mode=u=rwX,g=rwX,o=rwX recurse=yes + file: dest={{ django_root }}/webapp/media owner=www-data group=www-data mode=u=rwX,g=rwX,o=rwX recurse=yes - name: update log file ownership - file: dest={{ web_root }}/webapp/logs owner=www-data group=www-data mode=u=rwX,g=rwX,o=rwX recurse=yes + file: dest={{ django_root }}/webapp/logs owner=www-data group=www-data mode=u=rwX,g=rwX,o=rwX recurse=yes - name: make update.sh executable file: dest=/home/ubuntu/update.sh mode=a+x @@ -111,7 +111,7 @@ - name: check whether news scraper in GMS version ansible.builtin.shell: "{{ venv_root }}/bin/python manage.py scrape_news --help" args: - chdir: "{{ web_root }}" + chdir: "{{ django_root }}" register: check_scraper - name: add cron job to scrape news feed @@ -120,5 +120,5 @@ user: ubuntu hour: 9 minute: 0 - job: "cd {{ web_root }} && {{ venv_root }}/bin/python manage.py scrape_news" + job: "cd {{ django_root }} && {{ venv_root }}/bin/python manage.py scrape_news" when: check_scraper.rc == 0 diff --git a/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 b/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 index 7ee21fbc..00886a22 100644 --- a/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 +++ b/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 @@ -24,15 +24,15 @@ server { } location = /favicon.ico { - alias {{ web_root }}/webapp/static/favicon/favicon.ico; + alias {{ django_root }}/webapp/static/favicon/favicon.ico; } location /static { - root {{ web_root }}/webapp; + root {{ django_root }}/webapp; } location /media { - root {{ web_root }}/webapp; + root {{ django_root }}/webapp; } location / { diff --git a/deploy/ansible/roles/galaxy_media_site/templates/webapp.service.j2 b/deploy/ansible/roles/galaxy_media_site/templates/webapp.service.j2 index 03dbb852..bdd974d0 100644 --- a/deploy/ansible/roles/galaxy_media_site/templates/webapp.service.j2 +++ b/deploy/ansible/roles/galaxy_media_site/templates/webapp.service.j2 @@ -8,7 +8,7 @@ PIDFile=/run/gunicorn/pid User = www-data Group = www-data RuntimeDirectory=gunicorn -WorkingDirectory={{ web_root }} +WorkingDirectory={{ django_root }} Environment="PATH={{ venv_root }}/.venv/bin:$PATH" ExecStart={{ venv_root }}/bin/gunicorn --pid /run/gunicorn/pid \ --bind unix:/run/webapp.sock \ From 18da003697fdcbe1627f42335d2d55c127695c30 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Mon, 30 Sep 2024 07:30:48 +1000 Subject: [PATCH 09/34] Ansible - rename 'webapp' to 'gms' --- .env.sample | 4 ++-- deploy/ansible/files/webapp.socket | 2 +- deploy/ansible/group_vars/webservers.yml | 2 +- deploy/ansible/group_vars/webservers.yml.sample | 4 ++-- .../roles/galaxy_media_site/defaults/main.yml | 16 ++++++++-------- .../files/{webapp.socket => gms.socket} | 2 +- .../roles/galaxy_media_site/tasks/main.yml | 12 ++++++------ .../{webapp.service.j2 => gms.service.j2} | 6 +++--- .../galaxy_media_site/templates/nginx.vhost.j2 | 8 ++++---- .../galaxy_media_site/templates/setup.sh.j2 | 10 +++++----- .../galaxy_media_site/templates/update.sh.j2 | 2 +- deploy/nginx.conf.tmpl | 12 ++++++------ deploy/webapp.service.tmpl | 6 +++--- deploy/webapp.socket | 2 +- 14 files changed, 44 insertions(+), 44 deletions(-) rename deploy/ansible/roles/galaxy_media_site/files/{webapp.socket => gms.socket} (74%) rename deploy/ansible/roles/galaxy_media_site/templates/{webapp.service.j2 => gms.service.j2} (78%) diff --git a/.env.sample b/.env.sample index 71583f9d..3079eedb 100644 --- a/.env.sample +++ b/.env.sample @@ -7,8 +7,8 @@ HOSTNAME=localhost:8000 CSRF_COOKIE_DOMAIN=.mysite.com # Postgres database credentials -DB_NAME=webapp -DB_USER=webapp +DB_NAME=gms +DB_USER=gms DB_PASSWORD="secret" # Credentials for your SMTP mail server diff --git a/deploy/ansible/files/webapp.socket b/deploy/ansible/files/webapp.socket index 4eb3bbe1..094d1e5c 100644 --- a/deploy/ansible/files/webapp.socket +++ b/deploy/ansible/files/webapp.socket @@ -1,6 +1,6 @@ [Unit] Description=Galaxy content site socket [Socket] -ListenStream=/run/webapp.sock +ListenStream=/run/gms.sock [Install] WantedBy=sockets.target diff --git a/deploy/ansible/group_vars/webservers.yml b/deploy/ansible/group_vars/webservers.yml index a80ec2e6..35488d7f 100644 --- a/deploy/ansible/group_vars/webservers.yml +++ b/deploy/ansible/group_vars/webservers.yml @@ -49,7 +49,7 @@ django_root: "{{ project_root }}/webapp" # Virtual environment will be created here venv_root: "{{ project_root }}/venv" -# Admin user login for the admin portal +# Login for django admin portal admin_user: first_name: admin last_name: admin diff --git a/deploy/ansible/group_vars/webservers.yml.sample b/deploy/ansible/group_vars/webservers.yml.sample index 22d63949..aa1ecb17 100644 --- a/deploy/ansible/group_vars/webservers.yml.sample +++ b/deploy/ansible/group_vars/webservers.yml.sample @@ -19,8 +19,8 @@ gunicorn: web_workers: 4 # Postgres database conf -postgres_db_name: webapp -postgres_db_user: webapp +postgres_db_name: gms +postgres_db_user: gms postgresql_locales: - 'en_US.UTF-8' diff --git a/deploy/ansible/roles/galaxy_media_site/defaults/main.yml b/deploy/ansible/roles/galaxy_media_site/defaults/main.yml index 51612fa1..f1127c1e 100644 --- a/deploy/ansible/roles/galaxy_media_site/defaults/main.yml +++ b/deploy/ansible/roles/galaxy_media_site/defaults/main.yml @@ -1,15 +1,15 @@ --- -project_root: /home/ubuntu/galaxy-content-site -server_root: /home/ubuntu/server -django_root: /srv/webapp +project_root: /srv/galaxy-media-site +server_root: /srv/config +django_root: "{{ project_root }}/webapp" venv_root: "{{ project_root }}/venv" -webapp: +gms: templates: - src: "{{ role_path }}/templates/gunicorn.py.j2" dest: "{{ server_root }}/gunicorn.py" - - src: "{{ role_path }}/templates/webapp.service.j2" - dest: /etc/systemd/system/webapp.service + - src: "{{ role_path }}/templates/gms.service.j2" + dest: /etc/systemd/system/gms.service - src: "{{ role_path }}/templates/setup.sh.j2" dest: "{{ server_root }}/setup.sh" - src: "{{ role_path }}/templates/nginx.vhost.j2" @@ -20,8 +20,8 @@ webapp: dest: "/home/ubuntu/update.sh" files: - - src: "{{ role_path }}/files/webapp.socket" - dest: /etc/systemd/system/webapp.socket + - src: "{{ role_path }}/files/gms.socket" + dest: /etc/systemd/system/gms.socket certbot_ssl: true diff --git a/deploy/ansible/roles/galaxy_media_site/files/webapp.socket b/deploy/ansible/roles/galaxy_media_site/files/gms.socket similarity index 74% rename from deploy/ansible/roles/galaxy_media_site/files/webapp.socket rename to deploy/ansible/roles/galaxy_media_site/files/gms.socket index 4eb3bbe1..094d1e5c 100644 --- a/deploy/ansible/roles/galaxy_media_site/files/webapp.socket +++ b/deploy/ansible/roles/galaxy_media_site/files/gms.socket @@ -1,6 +1,6 @@ [Unit] Description=Galaxy content site socket [Socket] -ListenStream=/run/webapp.sock +ListenStream=/run/gms.sock [Install] WantedBy=sockets.target diff --git a/deploy/ansible/roles/galaxy_media_site/tasks/main.yml b/deploy/ansible/roles/galaxy_media_site/tasks/main.yml index 11a3e252..549dd94b 100644 --- a/deploy/ansible/roles/galaxy_media_site/tasks/main.yml +++ b/deploy/ansible/roles/galaxy_media_site/tasks/main.yml @@ -39,12 +39,12 @@ clone: yes force: yes -- name: create webapp media directory +- name: create gms media directory file: path: "{{ django_root }}/webapp/media" state: directory -- name: create webapp logs directory +- name: create gms logs directory file: path: "{{ django_root }}/webapp/logs" state: directory @@ -53,13 +53,13 @@ template: src: "{{ item.src }}" dest: "{{ item.dest }}" - loop: "{{ webapp.templates }}" + loop: "{{ gms.templates }}" - name: copy webserver configuration copy: src: "{{ item.src }}" dest: "{{ item.dest }}" - loop: "{{ webapp.files }}" + loop: "{{ gms.files }}" - name: link to enable nginx virtualhost ansible.builtin.file: @@ -73,14 +73,14 @@ virtualenv: "{{ venv_root }}" virtualenv_python: python3.12 -- name: run webapp setup script +- name: run gms setup script command: "bash {{ server_root }}/setup.sh" - name: create and install SSL certificate with letsencrypt command: "certbot --nginx --noninteractive --agree-tos --redirect --email {{ certbot_renew_email }} -d {{ inventory_hostname }}" when: certbot_ssl -- name: Ensure webapp superuser login +- name: Ensure gms superuser login shell: | cd {{ django_root }} && \ export DJANGO_SUPERUSER_PASSWORD={{ admin_user.password }} && \ diff --git a/deploy/ansible/roles/galaxy_media_site/templates/webapp.service.j2 b/deploy/ansible/roles/galaxy_media_site/templates/gms.service.j2 similarity index 78% rename from deploy/ansible/roles/galaxy_media_site/templates/webapp.service.j2 rename to deploy/ansible/roles/galaxy_media_site/templates/gms.service.j2 index bdd974d0..97947793 100644 --- a/deploy/ansible/roles/galaxy_media_site/templates/webapp.service.j2 +++ b/deploy/ansible/roles/galaxy_media_site/templates/gms.service.j2 @@ -1,6 +1,6 @@ [Unit] Description=Gunicorn for Galaxy Media Site -Requires=webapp.socket +Requires=gms.socket After=network.target [Service] @@ -9,9 +9,9 @@ User = www-data Group = www-data RuntimeDirectory=gunicorn WorkingDirectory={{ django_root }} -Environment="PATH={{ venv_root }}/.venv/bin:$PATH" +Environment="PATH={{ venv_root }}/bin:$PATH" ExecStart={{ venv_root }}/bin/gunicorn --pid /run/gunicorn/pid \ - --bind unix:/run/webapp.sock \ + --bind unix:/run/gms.sock \ -c {{ server_root }}/gunicorn.py webapp.wsgi ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s TERM $MAINPID diff --git a/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 b/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 index 00886a22..547a3b1e 100644 --- a/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 +++ b/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 @@ -1,4 +1,4 @@ -# Server config for {{ inventory_hostname }} (Galaxy content site) +# Server config for {{ inventory_hostname }} (Galaxy media site) server { # redirect www to non-www @@ -11,8 +11,8 @@ server { listen 80; server_name {{ inventory_hostname }}; - access_log /var/log/nginx/webapp.access.log; - error_log /var/log/nginx/webapp.error.log; + access_log /var/log/nginx/gms.access.log; + error_log /var/log/nginx/gms.error.log; if ( $host !~* ^({{ inventory_hostname }})$ ) { # Reject requests with incorrect host header @@ -36,7 +36,7 @@ server { } location / { - proxy_pass http://unix:/run/webapp.sock; + proxy_pass http://unix:/run/gms.sock; proxy_redirect off; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/deploy/ansible/roles/galaxy_media_site/templates/setup.sh.j2 b/deploy/ansible/roles/galaxy_media_site/templates/setup.sh.j2 index fd5bb481..eae35408 100644 --- a/deploy/ansible/roles/galaxy_media_site/templates/setup.sh.j2 +++ b/deploy/ansible/roles/galaxy_media_site/templates/setup.sh.j2 @@ -1,9 +1,9 @@ -# Set up webapp and restart web services +# Set up gms and restart web services set -e source {{ venv_root }}/bin/activate -cd {{ project_root }}/webapp +cd {{ django_root }} # Set Django settings export DJANGO_SETTINGS_MODULE=webapp.settings.prod @@ -18,6 +18,6 @@ python manage.py migrate # Restart services sudo systemctl daemon-reload -sudo systemctl enable webapp.service -sudo systemctl enable webapp.socket -sudo service webapp restart +sudo systemctl enable gms.service +sudo systemctl enable gms.socket +sudo service gms restart diff --git a/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 b/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 index 8a2d8bb7..0d12c7ff 100644 --- a/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 +++ b/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 @@ -28,7 +28,7 @@ sudo chown -R www-data:www-data webapp/logs sudo chown -R www-data:www-data webapp/media # Restart the GMS web service -sudo service webapp restart +sudo service gms restart echo "" echo "~~~ Restarted Galaxy Media Site ~~~" diff --git a/deploy/nginx.conf.tmpl b/deploy/nginx.conf.tmpl index ce9702f4..bfd29130 100644 --- a/deploy/nginx.conf.tmpl +++ b/deploy/nginx.conf.tmpl @@ -11,8 +11,8 @@ server { listen 80; server_name {{ HOSTNAME }}; - access_log /var/log/nginx/webapp.access.log; - error_log /var/log/nginx/webapp.error.log; + access_log /var/log/nginx/gms.access.log; + error_log /var/log/nginx/gms.error.log; if ( $host !~* ^({{ HOSTNAME }})$ ) { # Reject requests with incorrect host header @@ -24,19 +24,19 @@ server { } location = /favicon.ico { - alias /srv/sites/webapp/webapp/static/favicon/favicon.ico; + alias /srv/galaxy-media-site/webapp/gms/static/favicon/favicon.ico; } location /static { - root /srv/sites/webapp/webapp; + root /srv/galaxy-media-site/webapp/gms; } location /media { - root /srv/sites/webapp/webapp; + root /srv/galaxy-media-site/webapp/gms; } location / { - proxy_pass http://unix:/run/webapp.sock; + proxy_pass http://unix:/run/gms.sock; proxy_redirect off; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/deploy/webapp.service.tmpl b/deploy/webapp.service.tmpl index 35c3dfee..9ca6a661 100644 --- a/deploy/webapp.service.tmpl +++ b/deploy/webapp.service.tmpl @@ -1,6 +1,6 @@ [Unit] Description=Gunicorn for Galaxy Media Site -Requires=webapp.socket +Requires=gms.socket After=network.target [Service] @@ -8,10 +8,10 @@ PIDFile=/run/gunicorn/pid User = ubuntu Group = www-data RuntimeDirectory=gunicorn -WorkingDirectory=/srv/sites/webapp +WorkingDirectory=/srv/galaxy-media-site/webapp Environment="PATH={{ PWD }}/.venv/bin:$PATH" ExecStart={{ PWD }}/.venv/bin/gunicorn --pid /run/gunicorn/pid \ - --bind unix:/run/webapp.sock \ + --bind unix:/run/gms.sock \ -c {{ PWD }}/gunicorn.py webapp.wsgi ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s TERM $MAINPID diff --git a/deploy/webapp.socket b/deploy/webapp.socket index 4eb3bbe1..094d1e5c 100644 --- a/deploy/webapp.socket +++ b/deploy/webapp.socket @@ -1,6 +1,6 @@ [Unit] Description=Galaxy content site socket [Socket] -ListenStream=/run/webapp.sock +ListenStream=/run/gms.sock [Install] WantedBy=sockets.target From 82b4dc3752221ad23f1338aa8139ad3b608fe11d Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Mon, 30 Sep 2024 07:31:13 +1000 Subject: [PATCH 10/34] Wiki --- wiki | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki b/wiki index 2e306323..5a248a6d 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 2e3063237ae34f2085c5cfbfbcf7459be6d474e1 +Subproject commit 5a248a6d2cbdbe1581608d7961e5b8271b4a7b9d From 3cce8e46411de8c8520572fde1c9091a7cf3ee6f Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Mon, 30 Sep 2024 11:23:12 +1000 Subject: [PATCH 11/34] Fix certbot tasks and nginx template --- .../roles/galaxy_media_site/tasks/main.yml | 10 +++++++++ .../templates/nginx.vhost.j2 | 22 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/deploy/ansible/roles/galaxy_media_site/tasks/main.yml b/deploy/ansible/roles/galaxy_media_site/tasks/main.yml index 549dd94b..03335ba1 100644 --- a/deploy/ansible/roles/galaxy_media_site/tasks/main.yml +++ b/deploy/ansible/roles/galaxy_media_site/tasks/main.yml @@ -1,5 +1,15 @@ --- +- name: Check whether SSL certificate is already installed + ansible.builtin.shell: "certbot certificates" + register: certbot_certificates + changed_when: false + +- name: Set ssl_cert_exists fact from certbot output + ansible.builtin.set_fact: + ssl_cert_exists: "{{ true if inventory_hostname in certbot_certificates.stdout else false }}" + changed_when: false + - name: create server directory file: path: "{{ server_root }}" diff --git a/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 b/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 index 547a3b1e..2acdde20 100644 --- a/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 +++ b/deploy/ansible/roles/galaxy_media_site/templates/nginx.vhost.j2 @@ -46,4 +46,26 @@ server { client_max_body_size 1000m; } + {% if ssl_cert_exists %} + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/gms.neoformit.com/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/gms.neoformit.com/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + {% endif %} + +} + +{% if ssl_cert_exists %} +server { + + if ($host = gms.neoformit.com) { + return 301 https://$host$request_uri; + } # managed by Certbot + + listen 80; + server_name gms.neoformit.com; + return 404; # managed by Certbot + } +{% endif %} From 357ceb99aa23a9260032e1afc6210b86cad8b919 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Wed, 2 Oct 2024 20:23:09 +1000 Subject: [PATCH 12/34] Revert ansible hosts --- deploy/ansible/hosts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/ansible/hosts b/deploy/ansible/hosts index 621d02e8..c99064c0 100644 --- a/deploy/ansible/hosts +++ b/deploy/ansible/hosts @@ -2,5 +2,4 @@ site.usegalaxy.org.au ansible_connection=ssh ansible_user=ubuntu ansible_ssh_private_key_file=~/.ssh/galaxy [dev_webservers] -# dev-site.gvl.org.au ansible_connection=ssh ansible_user=ubuntu -gms.neoformit.com ansible_connection=ssh ansible_user=ubuntu ansible_ssh_private_key_file=~/.ssh/galaxy +dev-site.gvl.org.au ansible_connection=ssh ansible_user=ubuntu ansible_ssh_private_key_file=~/.ssh/galaxy From 75b64d0e862b3153a75d0a0d6c749b020f501d8f Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Thu, 3 Oct 2024 15:49:49 +1000 Subject: [PATCH 13/34] WIP fix automated target=_blank designation --- webapp/home/templates/home/header.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webapp/home/templates/home/header.html b/webapp/home/templates/home/header.html index eaf720fc..266367f4 100644 --- a/webapp/home/templates/home/header.html +++ b/webapp/home/templates/home/header.html @@ -204,10 +204,10 @@ $('a').each( (i, el) => { if ( el.href - && el.href.includes('https://{{ HOSTNAME }}') + && !el.href.includes('https://{{ HOSTNAME }}') && !isLabPage() ) { - if (el.href.replace(/\/$/, "") === 'https://{{ HOSTNAME }}') { + if (el.href.replace(/\/$/, "") === 'https://usegalaxy.org.au') { console.log(`Remove href from anchor: ${el.href}`); $(el).removeAttr("href"); } else { From 2410e44cdaebd5c185c57833c34eb7ccd218f7f0 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Thu, 3 Oct 2024 16:44:03 +1000 Subject: [PATCH 14/34] Remove hrefs matching window.top hostname --- webapp/home/templates/home/header.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/home/templates/home/header.html b/webapp/home/templates/home/header.html index 266367f4..b44bcf93 100644 --- a/webapp/home/templates/home/header.html +++ b/webapp/home/templates/home/header.html @@ -207,11 +207,11 @@ && !el.href.includes('https://{{ HOSTNAME }}') && !isLabPage() ) { - if (el.href.replace(/\/$/, "") === 'https://usegalaxy.org.au') { - console.log(`Remove href from anchor: ${el.href}`); + if (el.href.startsWith(`https://${window.top.location.host}`)) { + // console.log(`Remove href from anchor: ${el.href}`); $(el).removeAttr("href"); } else { - console.log(`Set target="_blank" on anchor: ${el.href}`); + // console.log(`Set target="_blank" on anchor: ${el.href}`); $(el).attr("target", "_blank"); } } From 3eff63156f96033d6a4629ef6df79919d32c0ab3 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Fri, 4 Oct 2024 08:54:15 +1000 Subject: [PATCH 15/34] Remove header.html window.top.location reference --- webapp/home/templates/home/header.html | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/webapp/home/templates/home/header.html b/webapp/home/templates/home/header.html index b44bcf93..7d27712b 100644 --- a/webapp/home/templates/home/header.html +++ b/webapp/home/templates/home/header.html @@ -200,17 +200,27 @@ // Remove links to usegalaxy.org.au if in iframe // ...otherwise risk having galaxy-in-galaxy which is very weird! + // If GMS (local) URL then do nothing + // If parent root URL then remove href + // Else must be: + // - non-root parent URL + // - OR external URL + // -> set target="_blank" (opens in new tab) + const urlParams = new URLSearchParams(window.location.search); + const parentHostname = urlParams.get('hostname') || 'usegalaxy.org.au'; if ( inIframe() ) { $('a').each( (i, el) => { if ( el.href - && !el.href.includes('https://{{ HOSTNAME }}') + && !el.href.startsWith('https://{{ HOSTNAME }}') && !isLabPage() ) { - if (el.href.startsWith(`https://${window.top.location.host}`)) { + if (el.replace(/\/$/, "") === `https://${parentHostname}`) { + // It's a link to the parent site root URL // console.log(`Remove href from anchor: ${el.href}`); $(el).removeAttr("href"); } else { + // It's a link to the parent site with a path, or an external site // console.log(`Set target="_blank" on anchor: ${el.href}`); $(el).attr("target", "_blank"); } From 61b42619a7037da4e03c817caa032d0f9b494758 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Fri, 4 Oct 2024 08:58:11 +1000 Subject: [PATCH 16/34] Fix ansible update.sh.j2 template --- .../galaxy_media_site/templates/update.sh.j2 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 b/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 index 0d12c7ff..3c2f43ae 100644 --- a/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 +++ b/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 @@ -4,31 +4,31 @@ set -e -# Make sure this isn't being run as root -if [[ $EUID = 0 ]]; then - echo "Don't run this as root - ubuntu is fine!" +# Ensure run as root +if [[ $EUID != 0 ]]; then + echo "Must be run as root - use sudo" exit 0 fi # Source virtual environment and git pull -cd galaxy-content-site +cd {{ project_root }} source '{{ venv_root }}/bin/activate' git pull python -m pip install -r requirements.txt -cd webapp +cd {{ django_root }} # Collect any new static files for Nginx if neccessary python manage.py collectstatic --noinput # Migrate the database (if necessary) -python manage.py migrate +sudo -u www-data python manage.py migrate # Update file ownership to allow www-data to write logs/media -sudo chown -R www-data:www-data webapp/logs -sudo chown -R www-data:www-data webapp/media +chown -R www-data:www-data webapp/logs +chown -R www-data:www-data webapp/media # Restart the GMS web service -sudo service gms restart +service gms restart echo "" echo "~~~ Restarted Galaxy Media Site ~~~" From 2bfb89e274a5e43d7877f96f45ccd7b8f01c8056 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Fri, 4 Oct 2024 09:11:34 +1000 Subject: [PATCH 17/34] Fix typo --- webapp/home/templates/home/header.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/home/templates/home/header.html b/webapp/home/templates/home/header.html index 7d27712b..81def67e 100644 --- a/webapp/home/templates/home/header.html +++ b/webapp/home/templates/home/header.html @@ -215,7 +215,7 @@ && !el.href.startsWith('https://{{ HOSTNAME }}') && !isLabPage() ) { - if (el.replace(/\/$/, "") === `https://${parentHostname}`) { + if (el.href.replace(/\/$/, "") === `https://${parentHostname}`) { // It's a link to the parent site root URL // console.log(`Remove href from anchor: ${el.href}`); $(el).removeAttr("href"); From d8114dd3c01442580a719364dd54207c0c6ca8d0 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Fri, 4 Oct 2024 09:23:14 +1000 Subject: [PATCH 18/34] Deploy workflows use sudo --- .github/workflows/deploy-dev.yml | 2 +- .github/workflows/deploy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 0861a78b..2747f985 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -23,6 +23,6 @@ jobs: echo "${{ secrets.SSH_PRIVATE_KEY_DEV }}" > ~/.ssh/id_rsa ssh-keyscan -H ${{ secrets.SSH_HOST_DEV }} > ~/.ssh/known_hosts - name: connect and pull - run: ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DEV }} "./update.sh && exit" + run: ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST_DEV }} "sudo ./update.sh && exit" - name: cleanup run: rm -rf ~/.ssh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 69283dad..4177f992 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,6 +23,6 @@ jobs: echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa ssh-keyscan -H ${{ secrets.SSH_HOST }} > ~/.ssh/known_hosts - name: connect and pull - run: ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "./update.sh && exit" + run: ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "sudo ./update.sh && exit" - name: cleanup run: rm -rf ~/.ssh From 02d976af82c84ce3ed44886e13f51ef41325761c Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Fri, 4 Oct 2024 09:28:13 +1000 Subject: [PATCH 19/34] update.sh run manage.py migrate as root --- deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 b/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 index 3c2f43ae..b0296552 100644 --- a/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 +++ b/deploy/ansible/roles/galaxy_media_site/templates/update.sh.j2 @@ -21,7 +21,7 @@ cd {{ django_root }} python manage.py collectstatic --noinput # Migrate the database (if necessary) -sudo -u www-data python manage.py migrate +python manage.py migrate # Update file ownership to allow www-data to write logs/media chown -R www-data:www-data webapp/logs From 6b99f03c3a73eac6abeb5588bbc750c2a4366586 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Fri, 4 Oct 2024 09:57:25 +1000 Subject: [PATCH 20/34] Revert prod deploy workflow command --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4177f992..69283dad 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,6 +23,6 @@ jobs: echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa ssh-keyscan -H ${{ secrets.SSH_HOST }} > ~/.ssh/known_hosts - name: connect and pull - run: ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "sudo ./update.sh && exit" + run: ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} "./update.sh && exit" - name: cleanup run: rm -rf ~/.ssh From 098771cb10c63d22145c088b5baa59b2b43dd6be Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Tue, 8 Oct 2024 07:33:36 +1000 Subject: [PATCH 21/34] Add diann form to access request menu --- webapp/home/static/home/css/requests.css | 13 +++++++++++++ .../home/templates/home/requests/access/index.html | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/webapp/home/static/home/css/requests.css b/webapp/home/static/home/css/requests.css index c6a5d2c0..1651aa85 100644 --- a/webapp/home/static/home/css/requests.css +++ b/webapp/home/static/home/css/requests.css @@ -59,4 +59,17 @@ span.material-icons { font-size: 0.8rem; padding: 0; } + .request-menu-cards .ga-btn { + min-width: 100px; + } } +@media (max-width: 575px) { + .request-menu-cards .card { + padding: 2rem 1rem; + margin: .5rem auto; + max-width: 300px; + } + .request-menu-cards .card-body { + font-size: 1rem; + } +} \ No newline at end of file diff --git a/webapp/home/templates/home/requests/access/index.html b/webapp/home/templates/home/requests/access/index.html index 113a7f1c..7fcc2889 100644 --- a/webapp/home/templates/home/requests/access/index.html +++ b/webapp/home/templates/home/requests/access/index.html @@ -81,6 +81,20 @@

Request access to specialist tools

+ +
+
+

+ Analyse data-independent acquisition (DIA) proteomics data with DiaNN +

+
+ +
+ + Apply now + +
+
From 3904d8ad9b3e15a5d2cc8d285a6a8b71d95f3a17 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Tue, 8 Oct 2024 07:56:56 +1000 Subject: [PATCH 22/34] Set SERVER_EMAIL for error email address --- .env.sample | 3 +++ webapp/webapp/settings/base.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.env.sample b/.env.sample index 3079eedb..6c40dddc 100644 --- a/.env.sample +++ b/.env.sample @@ -16,6 +16,9 @@ MAIL_FROM_ADDRESS=noreply@mysite.com MAIL_TO_ADDRESS=helpdesk@mysite.com MAIL_USE_TLS=true +# Optional: Send error emails to a different address than MAIL_FROM_ADDRESS +SERVER_EMAIL=support@mysite.com + MAIL_SMTP_PORT=25 MAIL_HOSTNAME=mail.mysite.com MAIL_SMTP_USERNAME=admin diff --git a/webapp/webapp/settings/base.py b/webapp/webapp/settings/base.py index 5e62519c..eff8ca4b 100644 --- a/webapp/webapp/settings/base.py +++ b/webapp/webapp/settings/base.py @@ -133,7 +133,7 @@ EMAIL_FROM_ADDRESS = os.environ['MAIL_FROM_ADDRESS'] EMAIL_TO_ADDRESS = os.environ['MAIL_TO_ADDRESS'] -SERVER_EMAIL = os.environ['MAIL_FROM_ADDRESS'] +SERVER_EMAIL = os.getenv('SERVER_EMAIL', EMAIL_FROM_ADDRESS) EMAIL_SUBJECT_PREFIX = os.getenv('EMAIL_SUBJECT_PREFIX', 'GMS: ') if os.getenv('MAIL_HOSTNAME'): From 5f8ab3ff9cb3f9043cb89b6c4d877eddd8e71df4 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Tue, 8 Oct 2024 07:57:17 +1000 Subject: [PATCH 23/34] Disable error_mail logger --- webapp/webapp/settings/log/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/webapp/settings/log/config.py b/webapp/webapp/settings/log/config.py index 165d6683..981450bf 100644 --- a/webapp/webapp/settings/log/config.py +++ b/webapp/webapp/settings/log/config.py @@ -114,7 +114,7 @@ def configure_logging(LOG_ROOT): 'debug_file', 'main_file', 'error_file', - 'error_mail', + # 'error_mail', 'error_slack', 'console' ], From 367dc7b4bc0f717763b4c9472d0a60d9be743557 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Tue, 8 Oct 2024 08:32:51 +1000 Subject: [PATCH 24/34] Add sentry log handler --- requirements.txt | 1 + webapp/webapp/settings/prod.py | 12 +++++++++++- webapp/webapp/urls.py | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 712ae716..0cd61d78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,4 @@ lxml Pillow requests==2.* requests_mock==1.* +sentry-sdk==2.* diff --git a/webapp/webapp/settings/prod.py b/webapp/webapp/settings/prod.py index a0eecac8..bee1d9f4 100644 --- a/webapp/webapp/settings/prod.py +++ b/webapp/webapp/settings/prod.py @@ -1,6 +1,7 @@ """Settings for production.""" import os +import sentry_sdk from .base import * from . import validate @@ -60,4 +61,13 @@ LOGGING = config.configure_logging(LOG_ROOT) # Use manifest to manage static file versions for cache busting: -STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage' +STATICFILES_STORAGE = ( + 'django.contrib.staticfiles.storage' + '.ManifestStaticFilesStorage') + +sentry_sdk.init( + dsn="https://426e64399bbafe4210c4fa647c7a2f5b@sentry.galaxyproject.org/20", + # Set traces_sample_rate to 1.0 to capture 100% + # of transactions for tracing. + traces_sample_rate=1.0, +) diff --git a/webapp/webapp/urls.py b/webapp/webapp/urls.py index 3407f3c2..9a206609 100644 --- a/webapp/webapp/urls.py +++ b/webapp/webapp/urls.py @@ -29,6 +29,7 @@ path('news/', include('news.urls')), path('events/', include('events.urls')), path('people/', include('people.urls')), + path('test-error-handlers/', lambda x: 1 / 0), ] if settings.DEBUG: From 5307b6d8c529d827e7593abf2fef08e69b5b5fc9 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Tue, 8 Oct 2024 08:39:08 +1000 Subject: [PATCH 25/34] Remove test error url --- webapp/webapp/urls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/webapp/webapp/urls.py b/webapp/webapp/urls.py index 9a206609..3407f3c2 100644 --- a/webapp/webapp/urls.py +++ b/webapp/webapp/urls.py @@ -29,7 +29,6 @@ path('news/', include('news.urls')), path('events/', include('events.urls')), path('people/', include('people.urls')), - path('test-error-handlers/', lambda x: 1 / 0), ] if settings.DEBUG: From 5e77129916c5cc30f94360e889da60dc3f358c3d Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Wed, 9 Oct 2024 06:00:18 +1000 Subject: [PATCH 26/34] Set sentry logging to ERROR level --- webapp/webapp/settings/prod.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/webapp/webapp/settings/prod.py b/webapp/webapp/settings/prod.py index bee1d9f4..a888310d 100644 --- a/webapp/webapp/settings/prod.py +++ b/webapp/webapp/settings/prod.py @@ -1,6 +1,9 @@ +# flake8: noqa + """Settings for production.""" import os +import logging import sentry_sdk from .base import * @@ -71,3 +74,4 @@ # of transactions for tracing. traces_sample_rate=1.0, ) +logging.getLogger('sentry_sdk').setLevel(logging.ERROR) From 5017a399fd7d2efda4d81ca8040535ebc9b102db Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Tue, 15 Oct 2024 07:46:39 +1000 Subject: [PATCH 27/34] Post footer height on all embedded snippets --- webapp/home/templates/embed-snippet.html | 9 +++++++++ webapp/home/templates/home/snippets/footer.html | 8 -------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/webapp/home/templates/embed-snippet.html b/webapp/home/templates/embed-snippet.html index 78707c18..e4c6e653 100644 --- a/webapp/home/templates/embed-snippet.html +++ b/webapp/home/templates/embed-snippet.html @@ -10,3 +10,12 @@ {% block content %} {% include snippet_path %} {% endblock %} + +{% block script %} + +{% endblock %} diff --git a/webapp/home/templates/home/snippets/footer.html b/webapp/home/templates/home/snippets/footer.html index 6cfbc0b1..208e02f8 100644 --- a/webapp/home/templates/home/snippets/footer.html +++ b/webapp/home/templates/home/snippets/footer.html @@ -95,12 +95,4 @@
CONNECT
- - From d1026252cbc785a76eebf4f5c933c264dd04e14a Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Tue, 15 Oct 2024 08:41:46 +1000 Subject: [PATCH 28/34] Read style from GET params in embed_snippet view --- webapp/home/templates/embed-snippet.html | 4 ++-- webapp/home/templates/home/header-export.html | 2 +- webapp/home/views.py | 17 ++++++++++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/webapp/home/templates/embed-snippet.html b/webapp/home/templates/embed-snippet.html index e4c6e653..4f719b2b 100644 --- a/webapp/home/templates/embed-snippet.html +++ b/webapp/home/templates/embed-snippet.html @@ -14,8 +14,8 @@ {% block script %} {% endblock %} diff --git a/webapp/home/templates/home/header-export.html b/webapp/home/templates/home/header-export.html index cc49fd95..f4730bd4 100644 --- a/webapp/home/templates/home/header-export.html +++ b/webapp/home/templates/home/header-export.html @@ -22,7 +22,7 @@ - + {% block content %} {% endblock %} diff --git a/webapp/home/views.py b/webapp/home/views.py index d8589394..5d94f673 100644 --- a/webapp/home/views.py +++ b/webapp/home/views.py @@ -313,13 +313,28 @@ def custom_400(request, exception, template_name="400.html"): def embed_snippet(request, snippet_path): """Serve an embeddable snippet.""" + ALLOW_STYLE_PARAMS = ['overflow'] + body_style_data = { + k: request.GET.get(k) + for k in ALLOW_STYLE_PARAMS + if k in request.GET + } + body_style = ' '.join( + f'{k}: {v};' + for k, v in body_style_data.items() + ) try: if 'snippets' not in snippet_path: raise Http404 return render(request, 'embed-snippet.html', { 'title': 'Galaxy Media - embedded snippet', 'snippet_path': snippet_path, - 'crop_margin': True, # could make this configurable in future + # Can be referenced in snippet templates: + 'crop_margin': request.GET.get( + 'crop', + 'true', + ).lower() in ('true', '1', 'yes'), + 'body_style': body_style, }) except TemplateDoesNotExist: raise Http404 From 0355d346470ffd5f2f78e3a6ed1d03d100edaa20 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Fri, 18 Oct 2024 12:48:24 +1000 Subject: [PATCH 29/34] Don't disable "agree" button on mac --- .../home/requests/access/snippets/agree-terms-modal.html | 8 +++++++- webapp/home/templates/home/requests/menu.html | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/webapp/home/templates/home/requests/access/snippets/agree-terms-modal.html b/webapp/home/templates/home/requests/access/snippets/agree-terms-modal.html index 9439aad1..7435d57b 100644 --- a/webapp/home/templates/home/requests/access/snippets/agree-terms-modal.html +++ b/webapp/home/templates/home/requests/access/snippets/agree-terms-modal.html @@ -76,6 +76,8 @@
.css('padding', '1rem')); {% endif %} + const isMac = () => navigator.platform.toUpperCase().indexOf('MAC') >= 0; + function agreeTermsAction(el) { enableTermsInput(); $('#agreeTermsInput').attr('checked', true); @@ -92,16 +94,20 @@
tooltip.disable(); } + function setEventHandler() { const termsContainer = $('#termsModal iframe').contents().find('body'); if (!termsContainer.length) { return setTimeout(setEventHandler, 100); } + if (isMac()) { + return enableAgreeTermsButton(); + } $('#termsModal iframe').contents().on('scroll', (event) => { const scrollFromEnd = $(termsContainer)[0].scrollHeight - $('#termsModal iframe').contents().scrollTop() - $('#termsModal iframe').height(); - if (scrollFromEnd < 50) enableAgreeTermsButton(); + scrollFromEnd < 50 && enableAgreeTermsButton(); }); termsContainer.css('margin', 'auto'); } diff --git a/webapp/home/templates/home/requests/menu.html b/webapp/home/templates/home/requests/menu.html index 708ed181..6a65700a 100644 --- a/webapp/home/templates/home/requests/menu.html +++ b/webapp/home/templates/home/requests/menu.html @@ -70,7 +70,8 @@

save

key

- Request access to specialised tools AlphaFold 2, FGENESH++ and Cell Ranger + Request access to specialised tools AlphaFold 2, FGENESH++, + Cell Ranger and more...

From 39b9518df5724e365c4887fd1ee732119062a57b Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Fri, 18 Oct 2024 12:53:58 +1000 Subject: [PATCH 30/34] Fix css bug --- webapp/home/static/home/css/request-form.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/home/static/home/css/request-form.css b/webapp/home/static/home/css/request-form.css index 459b9014..ace9d5ae 100644 --- a/webapp/home/static/home/css/request-form.css +++ b/webapp/home/static/home/css/request-form.css @@ -27,7 +27,7 @@ ul.errorlist { border: 1px solid var(--error); border-radius: .5rem; } -[class^="col"] { +.row [class^="col"] { padding-bottom: 1rem; } .blockquote { From 7373ae544090cee0bcf501f2b30ac3353dcbcfed Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Tue, 22 Oct 2024 18:33:08 +1000 Subject: [PATCH 31/34] Update quota form --- webapp/home/forms.py | 4 +- webapp/home/static/home/css/main.css | 3 + .../home/templates/home/requests/quota.html | 61 ++++++++++++------- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/webapp/home/forms.py b/webapp/home/forms.py index d4c1985e..898231f9 100644 --- a/webapp/home/forms.py +++ b/webapp/home/forms.py @@ -174,8 +174,8 @@ class QuotaRequestForm(OtherFieldFormMixin, forms.Form): email = forms.EmailField() start_date = forms.DateField() duration_months = forms.IntegerField() - disk_tb = forms.IntegerField() - disk_tb_other = forms.IntegerField(required=False) + disk_tb = forms.FloatField() + disk_tb_other = forms.FloatField(required=False) description = forms.CharField() captcha = fields.ReCaptchaField() accepted_terms = forms.BooleanField() diff --git a/webapp/home/static/home/css/main.css b/webapp/home/static/home/css/main.css index fb6955e6..bfbf8d1e 100644 --- a/webapp/home/static/home/css/main.css +++ b/webapp/home/static/home/css/main.css @@ -137,6 +137,9 @@ section { margin-left: 0; margin-right: 0; } +.narrow { + max-width: 600px; +} .container-centered { /* For content centered on full diff --git a/webapp/home/templates/home/requests/quota.html b/webapp/home/templates/home/requests/quota.html index 78ccb13a..40967095 100644 --- a/webapp/home/templates/home/requests/quota.html +++ b/webapp/home/templates/home/requests/quota.html @@ -40,15 +40,15 @@

Galaxy Australia Quota Increase Application

{% csrf_token %} -
-
+
+
{{ form.name.errors }}
-
+
Galaxy Australia Quota Increase Application
-
-
+
+
@@ -87,28 +87,36 @@

Galaxy Australia Quota Increase Application

{{ form.start_date.errors }}
-
+
-
- + -
- - months - -
+ +
+
+ +
+
+ + +
+ {{ form.duration_months.errors }}
-
-
+
+

What is the total storage space you require?

@@ -116,14 +124,21 @@

Galaxy Australia Quota Increase Application

{{ form.disk_tb.errors }}
- + +
+ +
+
- @@ -169,12 +184,12 @@

Galaxy Australia Quota Increase Application

>{{form.description.value|default_if_none:''}}
- Please briefly justify your need for additional data storage. - Describe your data, file types and why they are required for your research. + Please briefly justify your need for additional data storage by + describing your data and intended analysis.
-
+

Agreement on the use of the Galaxy Australia Server

From 31c30dd28c753aae2bc66d8b8f62125bc496500e Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Wed, 23 Oct 2024 05:30:14 +1000 Subject: [PATCH 32/34] Reword --- webapp/home/templates/home/requests/quota.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/home/templates/home/requests/quota.html b/webapp/home/templates/home/requests/quota.html index 40967095..9315f696 100644 --- a/webapp/home/templates/home/requests/quota.html +++ b/webapp/home/templates/home/requests/quota.html @@ -176,7 +176,7 @@

Galaxy Australia Quota Increase Application

{{ form.description.errors }}