Merge pull request #201 from NationalGenomicsInfrastructure/monthly

Monthly merge October 2023
NationalGenomicsInfrastructure · Oct 30, 2023 · 42004a3 · 42004a3
2 parents bfb1583 + d171ea6
commit 42004a3
Show file tree

Hide file tree

Showing 18 changed files with 150 additions and 16 deletions.
diff --git a/env_vars/site_upps_env_all.yml b/env_vars/site_upps_env_all.yml
@@ -38,14 +38,6 @@ tarzan_port: 4444
 arteria_service_log_dir: "{{ static_ngi_log_path }}/arteria/{{ arteria_service_name }}/{{ deployment_environment }}"
 pdc_archive_servername: ngiu
 
-# arteria archive upload
-archive_upload_path_to_archive_root: "{{ static_ngi_pipeline_nobackup }}/arteria/pdc_archive_links/{{ deployment_environment }}"
-archive_upload_port: 10480
-
-# arteria archive verify
-archive_verify_port: 10490
-archive_verify_path_to_verify_root: "{{ static_ngi_pipeline_nobackup }}/arteria/pdc_archive_dump/{{ deployment_environment }}"
-
 # arteria-checksum
 arteria_checksum_monitored_path: "{{ static_runfolder_path }}"
 arteria_checksum_port: 10420
@@ -56,3 +48,14 @@ arteria_delivery_port: 10440
 
 # sequencing-report-service
 seqreport_service_port: 10460
+
+# arteria checkQC
+checkqc_service_port: 10470
+
+# arteria archive upload
+archive_upload_path_to_archive_root: "{{ static_ngi_pipeline_nobackup }}/arteria/pdc_archive_links/{{ deployment_environment }}"
+archive_upload_port: 10480
+
+# arteria archive verify
+archive_verify_port: 10490
+archive_verify_path_to_verify_root: "{{ static_ngi_pipeline_nobackup }}/arteria/pdc_archive_dump/{{ deployment_environment }}"
diff --git a/...sequencing-report-ws/files/checkqc.config → files/checkqc.config b/...sequencing-report-ws/files/checkqc.config → files/checkqc.config
diff --git a/install.yml b/install.yml
@@ -90,6 +90,7 @@
     - { role: ngi_reports, tags: ngi_reports }
     - { role: multiqc, tags: multiqc }
     - { role: arteria-checksum-ws, tags: arteria-checksum }
+    - { role: arteria-checkqc-ws, tags: arteria-checkqc }
     - { role: arteria-delivery-ws, tags: arteria-delivery }
     - { role: arteria-sequencing-report-ws, tags: arteria-sequencing-report }
     - { role: standalone_scripts, tags: standalone_scripts }

diff --git a/roles/archive-upload-ws/defaults/main.yml b/roles/archive-upload-ws/defaults/main.yml
@@ -1,7 +1,7 @@
 ---
 
 archive_upload_repo: https://github.com/Molmed/snpseq-archive-upload.git
-archive_upload_version: v1.0.4
+archive_upload_version: v1.1.0
 
 arteria_service_name: archive-upload-ws
 archive_upload_cores_to_use: 2

diff --git a/roles/archive-upload-ws/templates/archive_upload_app.config.j2 b/roles/archive-upload-ws/templates/archive_upload_app.config.j2
@@ -18,6 +18,9 @@ port: "{{ archive_upload_port }}"
 # Path to the dsmc logs
 log_directory: "{{ arteria_service_log_dir }}" 
 
+dsmc_extra_args:
+  servername: {{ pdc_archive_servername }}
+
 
 # Whitelisted DSMC warnings
 # ANS1809W = a session with the TSM server has been disconnected: will retry again

diff --git a/roles/archive-verify-ws/defaults/main.yml b/roles/archive-verify-ws/defaults/main.yml
@@ -1,7 +1,7 @@
 ---
 
 archive_verify_repo: https://github.com/Molmed/snpseq-archive-verify.git
-archive_verify_version: v1.1.0-rc1
+archive_verify_version: v1.2.1
 arteria_service_name: archive-verify-ws
 
 archive_verify_redis_server_version: 7.0.4

diff --git a/roles/arteria-checkqc-ws/defaults/main.yml b/roles/arteria-checkqc-ws/defaults/main.yml
@@ -0,0 +1,5 @@
+arteria_service_name: arteria-checkqc-ws
+arteria_checkqc_service_log: "{{ arteria_service_log_dir }}/{{ arteria_service_name }}.log"
+
+checkqc_repo: https://github.com/Molmed/checkQC.git
+checkqc_version: v3.8.2
diff --git a/roles/arteria-checkqc-ws/meta/main.yml b/roles/arteria-checkqc-ws/meta/main.yml
@@ -0,0 +1,5 @@
+---
+
+dependencies:
+  - { role: anaconda, tags: anaconda }
+  - { role: setup_base_config, tags: setup_base_config }
diff --git a/roles/arteria-checkqc-ws/tasks/install.yml b/roles/arteria-checkqc-ws/tasks/install.yml
@@ -0,0 +1,28 @@
+---
+
+- name: create virtual python env for {{ arteria_service_name }}
+  shell: "conda create --name {{ arteria_service_name }} python=3.10"
+  args:
+    creates: "{{ arteria_service_env_root }}"
+
+- name: get CheckQC from git
+  git:
+    repo: "{{ checkqc_repo }}"
+    dest: "{{ arteria_service_src_path }}"
+    version: "{{ checkqc_version }}"
+
+- name: install CheckQC requirements
+  pip:
+    requirements: "{{ arteria_service_src_path }}/requirements/prod"
+    chdir: "{{ arteria_service_src_path }}"
+    executable: "{{ arteria_service_env_root }}/bin/pip"
+    state: present
+    extra_args: "-U"
+
+- name: install CheckQC
+  pip:
+    name: .
+    chdir: "{{ arteria_service_src_path }}"
+    executable: "{{ arteria_service_env_root }}/bin/pip"
+    state: present
+    extra_args: "-U"
diff --git a/roles/arteria-checkqc-ws/tasks/install_config.yml b/roles/arteria-checkqc-ws/tasks/install_config.yml
@@ -0,0 +1,37 @@
+---
+
+- name: ensure {{ arteria_service_name }} conf dir exists
+  file:
+    state: directory
+    path: "{{ arteria_service_config_root }}"
+
+- name: deploying {{ arteria_service_name }} app config
+  copy:
+    src: checkqc.config
+    dest: "{{ arteria_service_config_root }}/app.config"
+
+- name: deploy {{ arteria_service_name }} logger config
+  template:
+    src: checkqc_logger.config.j2
+    dest: "{{ arteria_service_config_root }}/logger.config"
+
+- name: modify uppsala's supervisord conf to start {{ arteria_service_name }}
+  ini_file:
+    dest: "{{ supervisord_conf }}"
+    section: program:{{ arteria_service_name }}-{{ deployment_environment }}
+    option: command
+    value: >
+      {{ arteria_service_env_root }}/bin/checkqc-ws
+      --config {{ arteria_service_config_root }}/app.config
+      --log_config {{ arteria_service_config_root }}/logger.config
+      --port={{ checkqc_service_port }}
+      {{ static_runfolder_path }}
+    backup: no
+
+- name: modify uppsala's supervisord conf to start {{ arteria_service_name }}
+  ini_file:
+    dest: "{{ supervisord_conf }}"
+    section: "program:{{ arteria_service_name }}-{{ deployment_environment }}"
+    option: autorestart
+    value: true
+    backup: no
diff --git a/roles/arteria-checkqc-ws/tasks/main.yml b/roles/arteria-checkqc-ws/tasks/main.yml
@@ -0,0 +1,15 @@
+---
+
+- block:
+  - include: install.yml
+  - include: install_config.yml
+
+  - name: add static folders to be created
+    set_fact:
+      static_folders: "{{ static_folders + [arteria_service_log_dir, arteria_service_log_dir + '/nxf_logs', runfolder_path] }}"
+
+  - name: Store {{ arteria_service_name }} tool version in deployment
+    lineinfile:
+      dest: "{{ deployed_tool_versions }}"
+      line: "{{ arteria_service_name }}: {{ checkqc_version }}"
+  when: site in ["upps"]
diff --git a/roles/arteria-checkqc-ws/templates/checkqc_logger.config.j2 b/roles/arteria-checkqc-ws/templates/checkqc_logger.config.j2
@@ -0,0 +1,28 @@
+---
+version: 1
+
+disable_existing_loggers: False
+
+formatters:
+    simple:
+        format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+handlers:
+    console:
+        class: logging.StreamHandler
+        level: DEBUG
+        formatter: simple
+        stream: ext://sys.stdout
+
+    file_handler:
+        class: logging.handlers.RotatingFileHandler
+        level: INFO
+        formatter: simple
+        filename: "{{ arteria_checkqc_service_log }}"
+        maxBytes: 10485760  # 10MB
+        backupCount: 20
+        encoding: utf8
+
+root:
+    level: DEBUG
+    handlers: [console, file_handler]
diff --git a/roles/arteria-sequencing-report-ws/defaults/main.yml b/roles/arteria-sequencing-report-ws/defaults/main.yml
@@ -1,7 +1,7 @@
 ---
 
 seqreport_service_repo: https://github.com/arteria-project/sequencing-report-service.git
-seqreport_service_version: v1.4.0-rc1
+seqreport_service_version: v1.4.1
 
 arteria_service_name: arteria-sequencing-report-ws
 arteria_sequencing_report_wrapper: "{{ arteria_service_config_root }}/arteria_sequencing_report_wrapper.sh"

diff --git a/roles/arteria-sequencing-report-ws/templates/seqreport_app.config.j2 b/roles/arteria-sequencing-report-ws/templates/seqreport_app.config.j2
@@ -17,8 +17,9 @@ nextflow_config:
     nf_profile: {{ seqreports_nf_profile }}
     environment:
         NXF_TEMP: {{ nextflow_env.NXF_TEMP }}
-        NXF_WORK: {{ nextflow_env.NXF_WORK }}
+        NXF_WORK: ${DEFAULT:runfolder_path}/seqreports/nextflow
         NXF_SINGULARITY_CACHEDIR: "{{ ngi_containers }}/seqreports"
+        NXF_ANSI_LOG: false
     # Note that in the parameters section it is possible to do variable
     # subsitution on the following variables.
     # loading the config.

diff --git a/roles/multiqc/defaults/main.yml b/roles/multiqc/defaults/main.yml
@@ -1,6 +1,6 @@
 multiqc_repo: https://github.com/ewels/MultiQC.git
 multiqc_dest: "{{ sw_path }}/multiqc"
-multiqc_version: "v1.15"
+multiqc_version: "v1.16"
 
 multiqc_ngi_repo: https://github.com/NationalGenomicsInfrastructure/MultiQC_NGI.git
 multiqc_ngi_dest: "{{ sw_path }}/multiqc_ngi"

diff --git a/roles/ngi_reports/defaults/main.yml b/roles/ngi_reports/defaults/main.yml
@@ -1,7 +1,7 @@
 ---
 ngi_reports_repo: https://github.com/NationalGenomicsInfrastructure/ngi_reports.git
 ngi_reports_dest: "{{ sw_path }}/ngi_reports"
-ngi_reports_version: 32e1aec482aaebfafd460a3eda888f0477d6ab68
+ngi_reports_version: 84c35b22d9997e4189e2a5e6ec4a5fc281df51e7
 ngi_reports_log: "{{ ngi_log_path }}/ngi_reports.log"
 
 ngi_visual_repo: https://github.com/NationalGenomicsInfrastructure/ngi_visualizations.git

diff --git a/roles/taca/defaults/main.yml b/roles/taca/defaults/main.yml
@@ -9,7 +9,7 @@ flowcell_parser_version: 322abe3ea3bbd4aca1abf090dd51df0c5eca3db2
 
 taca_repo: https://github.com/SciLifeLab/TACA.git
 taca_dest: "{{ sw_path }}/TACA"
-taca_version: d14323ac72bef8731a6da10bfe056cc08cad3709
+taca_version: a23fbbd640d0b3d755e824c0345625bb5eae4724
 
 ngi_pipeline_analysisdir: "{{ ngi_pipeline_workdir }}/ANALYSIS"
 ngi_pipeline_datadir: "{{ ngi_pipeline_workdir }}/DATA"

diff --git a/roles/tarzan/templates/kong.yml.j2 b/roles/tarzan/templates/kong.yml.j2
@@ -31,7 +31,7 @@ services:
     port: {{ archive_verify_port }}
     routes:
     - paths:
-      - /arteria_archive/verify
+      - /verify
       strip_path: true
 
   - name: arteria_sequencing_reports
@@ -42,6 +42,14 @@ services:
       - /seqreports
       strip_path: true
 
+  - name: arteria_checkqc
+    host: localhost
+    port: {{ checkqc_service_port }}
+    routes:
+    - paths:
+      - /checkqc
+      strip_path: true
+
 plugins:
   - name: key-auth
     config: