Skip to content

Commit

Permalink
Merge pull request #510 from galaxyproject/hon-duck
Browse files Browse the repository at this point in the history 
Configure data managers to run
  • Loading branch information
@nuwang
nuwang authored Feb 20, 2025
2 parents 3ad412e + b4f8766 commit 0c12390
Show file tree
Hide file tree
Showing 4 changed files with 168 additions and 16 deletions.
2 changes: 1 addition & 1 deletion galaxy/Chart.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ apiVersion: v2
name: galaxy
type: application
version: 6.0.0
appVersion: "24.2"
appVersion: "24.2.0"
description: Chart for Galaxy, an open, web-based platform for accessible, reproducible, and transparent computational biomedical research.
icon: https://galaxyproject.org/images/galaxy-logos/galaxy_project_logo_square.png

Expand Down
151 changes: 151 additions & 0 deletions galaxy/files/configs/shed_tool_data_table_conf.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
<?xml version="1.0" ?>
<tables>
<table name="all_fasta" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/all_fasta.loc" />
</table>
<table name="__dbkeys__" comment_char="#">
<columns>value, name, len_path</columns>
<file path="/galaxy/server/config/mutable/dbkeys.loc" />
</table>
<table name="bowtie2_indexes" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/bowtie2_indices.loc"/>
</table>
<table name="bwa_mem_indexes" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/bwa_mem_index.loc"/>
</table>
<table name="twobit" comment_char="#">
<columns>value, path</columns>
<file path="/galaxy/server/config/mutable/twobit.loc"/>
</table>
<table name="lastz_seqs" comment_char="#">
<columns>value, name, path</columns>
<file path="/galaxy/server/config/mutable/lastz_seqs.loc" />
</table>
<table name="alignseq_seq" comment_char="#">
<columns>type, value, path</columns>
<file path="/galaxy/server/config/mutable/alignseq.loc"/>
</table>
<table name="picard_indexes" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/picard_index.loc"/>
</table>
<table name="fasta_indexes" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/fasta_indexes.loc"/>
</table>
<table name="hisat2_indexes" comment_char="#" allow_duplicate_entries="False">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/hisat2_indexes.loc"/>
</table>
<table name="rnastar_index2x_versioned" comment_char="#" allow_duplicate_entries="False">
<columns>value, dbkey, name, path, with_gene_model, version</columns>
<file path="/galaxy/server/config/mutable/rnastar_index2x_versioned.loc"/>
</table>
<table name="bowtie_indexes" comment_char="#" allow_duplicate_entries="False">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/bowtie_indices.loc"/>
</table>
<table name="kallisto_indexes" comment_char="#" allow_duplicate_entries="False">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/kallisto_indexes.loc"/>
</table>
<table name="snpeffv_genomedb" comment_char="#" allow_duplicate_entries="False">
<columns>key, version, value, name, path</columns>
<file path="/galaxy/server/config/mutable/snpeffv_genomedb.loc"/>
</table>
<table name="snpeffv_regulationdb" comment_char="#" allow_duplicate_entries="False">
<columns>key, version, genome, value, name</columns>
<file path="/galaxy/server/config/mutable/snpeffv_regulationdb.loc"/>
</table>
<table name="snpeffv_databases" comment_char="#" allow_duplicate_entries="False">
<columns>key, version, value, name</columns>
<file path="/galaxy/server/config/mutable/snpeffv_databases.loc"/>
</table>
<table name="plant_tribes_scaffolds" comment_char="#">
<columns>value, name, path, description</columns>
<file path="/galaxy/server/config/mutable/plant_tribes_scaffolds.loc"/>
</table>
<table name="ncbi_taxonomy" comment_char="#">
<columns>value, name, path</columns>
<file path="/galaxy/server/config/mutable/ncbi_taxonomy.loc"/>
</table>
<table name="ncbi_accession2taxid" comment_char="#">
<columns>value, name, path</columns>
<file path="/galaxy/server/config/mutable/ncbi_accession2taxid.loc"/>
</table>
<table name="gemini_versioned_databases" comment_char="#" allow_duplicate_entries="False">
<columns>value, dbkey, version, name, path</columns>
<file path="/galaxy/server/config/mutable/gemini_versioned_databases.loc"/>
</table>
<table name="kraken2_databases" comment_char="#">
<columns>value, name, path</columns>
<file path="/galaxy/server/config/mutable/kraken2_databases.loc"/>
</table>
<table name="bracken_databases" comment_char="#">
<columns>value, name, path</columns>
<file path="/galaxy/server/config/mutable/bracken_databases.loc"/>
</table>
<table name="plasmidfinder_database" comment_char="#">
<columns>value, name, date, path</columns>
<file path="/galaxy/server/config/mutable/plasmidfinder_database.loc"/>
</table>
<table name="primer_scheme_bedfiles" comment_char="#">
<columns>value, description, path</columns>
<file path="/galaxy/server/config/mutable/primer_scheme_bedfiles.loc"/>
</table>
<table name="all_gff" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/all_gff.loc"/>
</table>
<table name="adapters" comment_char="#" allow_duplicate_entries="False">
<columns>value, name</columns>
<file path="/galaxy/server/config/mutable/adapters.loc"/>
</table>
<table name="bwameth_indexes" comment_char="#" allow_duplicate_entries="False">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/bwameth_indexes.loc"/>
</table>
<table name="salmon_indexes_versioned" comment_char="#" allow_duplicate_entries="False">
<columns>value, dbkey, name, path, version</columns>
<file path="/galaxy/server/config/mutable/salmon_indexes_versioned.loc"/>
</table>
<table name="sailfish_indexes" comment_char="#">
<columns>value, name, path</columns>
<file path="/galaxy/server/config/mutable/sailfish_indexes.loc"/>
</table>
<table name="gene_sets" comment_char="#" allow_duplicate_entries="False">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/gene_sets.loc"/>
</table>
<table name="sam_fa_indexes" comment_char="#">
<columns>line_type, value, path</columns>
<file path="/galaxy/server/config/mutable/sam_fa_indices.loc"/>
</table>
<table name="snpeff_msigdb_database" comment_char="#" allow_duplicate_entries="False">
<columns>value, name, path</columns>
<file path="/galaxy/server/config/mutable/snpeff_msigdb_database.loc"/>
</table>
<table name="snpsift_dbnsfps" comment_char="#" allow_duplicate_entries="False">
<columns>key, build, name, value, annotations</columns>
<file path="/galaxy/server/config/mutable/snpsift_dbnsfp.loc"/>
</table>
<table name="snpeffv_genomedb" comment_char="#" allow_duplicate_entries="False">
<columns>key, version, value, name, path</columns>
<file path="/galaxy/server/config/mutable/snpeffv_genomedb.loc"/>
</table>
<table name="snpeffv_regulationdb" comment_char="#" allow_duplicate_entries="False">
<columns>key, version, genome, value, name</columns>
<file path="/galaxy/server/config/mutable/snpeffv_regulationdb.loc"/>
</table>
<table name="featurecounts_anno" comment_char="#">
<columns>value, dbkey</columns>
<file path="/galaxy/server/config/mutable/featurecounts_anno.loc"/>
</table>
<table name="gemini_databases" comment_char="#" allow_duplicate_entries="False">
<columns>value, dbkey, name, path</columns>
<file path="/galaxy/server/config/mutable/gemini_databases.loc"/>
</table>
</tables>
12 changes: 6 additions & 6 deletions galaxy/templates/_helpers.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ Creates the bash command for the init container used to place files and change p
cp -anL /galaxy/server/config/integrated_tool_panel.xml /galaxy/server/config/mutable/integrated_tool_panel.xml;
cp -anL /galaxy/server/config/sanitize_allowlist.txt /galaxy/server/config/mutable/sanitize_allowlist.txt;
cp -anL /galaxy/server/config/data_manager_conf.xml.sample /galaxy/server/config/mutable/shed_data_manager_conf.xml;
cp -anL /galaxy/server/config/tool_data_table_conf.xml.sample /galaxy/server/config/mutable/shed_tool_data_table_conf.xml;
cp -anL /galaxy/server/config/shed_tool_data_table_conf.xml /galaxy/server/config/mutable/shed_tool_data_table_conf.xml;
cp -aruL /galaxy/server/tool-data {{.Values.persistence.mountPath}}/;
cp -aruL /galaxy/server/tools {{.Values.persistence.mountPath}}/;
echo "Done" > /galaxy/server/config/mutable/init_mounts_done_{{.Release.Revision}};
Expand All @@ -177,15 +177,15 @@ echo "Done" > /galaxy/server/config/mutable/init_mounts_done_{{.Release.Revision
Creates the bash command for the handlers to wait for init scripts
*/}}
{{- define "galaxy.init-container-wait-command" -}}
until [ -f /galaxy/server/config/mutable/db_init_done_{{$.Release.Revision}} ]; do echo "waiting for DB initialization"; sleep 1; done;
until [ -f /galaxy/server/config/mutable/db_init_done_{{$.Release.Revision}} ]; do echo "[`date`] Waiting for DB initialization..."; sleep 1; done;
{{- if $.Values.rabbitmq.enabled }}
until timeout 1 bash -c "echo > /dev/tcp/{{ template "galaxy-rabbitmq.fullname" $ }}/{{.Values.rabbitmq.port}}"; do echo "waiting for rabbitmq service"; sleep 1; done;
until timeout 1 bash -c "echo > /dev/tcp/{{ template "galaxy-rabbitmq.fullname" $ }}/{{.Values.rabbitmq.port}}"; do echo "[`date`] Waiting for rabbitmq service..."; sleep 1; done;
{{- end }}
until [ -f /galaxy/server/config/mutable/init_mounts_done_{{$.Release.Revision}} ]; do echo "waiting for copying onto NFS"; sleep 1; done;
until [ -f /galaxy/server/config/mutable/init_mounts_done_{{$.Release.Revision}} ]; do echo "[`date`] Waiting for copying onto NFS..."; sleep 1; done;
{{- if .Values.setupJob.downloadToolConfs.enabled }}
until [ -f /galaxy/server/config/mutable/init_clone_done_{{$.Release.Revision}} ]; do echo "waiting for refdata copying"; sleep 1; done;
until [ -f /galaxy/server/config/mutable/init_clone_done_{{$.Release.Revision}} ]; do echo "[`date`] Waiting for refdata copying..."; sleep 1; done;
{{- end }}
echo "Initialization waits complete";
echo "[`date`] Initialization waits complete.";
{{- end -}}

{{/*
Expand Down
19 changes: 10 additions & 9 deletions galaxy/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -535,26 +535,21 @@ configs:
shed_tool_config_file: "/galaxy/server/config/mutable/editable_shed_tool_conf.xml"
enable_tool_document_cache: false
tool_data_table_config_path: |-
{{ if .Values.setupJob.downloadToolConfs.enabled }}
{{- .Values.setupJob.downloadToolConfs.volume.mountPath }}/config/shed_tool_data_table_conf.xml
{{- if .Values.refdata.enabled -}}
/galaxy/server/config/mutable/shed_tool_data_table_conf.xml
{{ if .Values.refdata.enabled -}}
,/cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml
{{- end }}
{{- else }}
{{- if .Values.refdata.enabled -}}
/cvmfs/data.galaxyproject.org//managed/location/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml
{{- end }}
{{- end }}
tool_dependency_dir: "{{.Values.persistence.mountPath}}/deps"
job_config_file: "/galaxy/server/config/job_conf.yml"
outputs_to_working_directory: true
builds_file_path: |-
{{ if .Values.refdata.enabled }}
{{ if .Values.refdata.enabled -}}
/cvmfs/data.galaxyproject.org/managed/location/builds.txt
{{- end }}
containers_resolvers_config_file: "/galaxy/server/config/container_resolvers_conf.xml"
workflow_schedulers_config_file: "/galaxy/server/config/workflow_schedulers_conf.xml"
build_sites_config_file: "/galaxy/server/config/build_sites.yml"
data_manager_config_file: "{{if .Values.setupJob.downloadToolConfs.enabled }}{{ .Values.setupJob.downloadToolConfs.volume.mountPath }}/config/shed_data_manager_conf.xml{{ else }}/galaxy/server/config/data_manager_conf.xml.sample{{ end }}"
shed_data_manager_config_file: "/galaxy/server/config/mutable/shed_data_manager_conf.xml"
shed_tool_data_table_config: "/galaxy/server/config/mutable/shed_tool_data_table_conf.xml"
enable_data_manager_user_view: true
Expand Down Expand Up @@ -618,6 +613,8 @@ configs:
{{- (.Files.Get "files/configs/integrated_tool_panel.xml") }}
tool_conf.xml: |
{{- (.Files.Get "files/configs/tool_conf.xml") }}
shed_tool_data_table_conf.xml: |
{{- (.Files.Get "files/configs/shed_tool_data_table_conf.xml") }}
#- Additional dynamic rules to map into the container.
jobs:
Expand Down Expand Up @@ -653,6 +650,10 @@ jobs:
tool.id in GALAXY_LIB_TOOLS_UNVERSIONED or "CONVERTER_" == tool.id[:10] or tool.is_datatype_converter
params:
docker_container_id_override: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
.*data_manager.*:
params:
k8s_run_as_group_id: "{{ .Values.securityContext.runAsGroup }}"
k8s_run_as_user_id: "{{ .Values.securityContext.runAsUser }}"
Summary_Statistics1:
params:
docker_container_id_override: cloudve/gsummary:latest
Expand Down

0 comments on commit 0c12390

Please sign in to comment.