Merge pull request #28 from apriltuesday/EVA-2382

EVA-2382 - update copy to ftp logic

etc/example_submission_config.yaml

@@ -1,4 +1,5 @@
 ftp_dir: '/path/to/ftp/dir'
+public_ftp_dir: '/path/to/public/ftp/dir'
 eloads_dir: '/path/to/eload/dir'
 projects_dir: '/path/to/project/dir'
 eutils_api_key: 1234556lkxflk

eva_submission/eload_ingestion.py

@@ -248,6 +248,7 @@ def run_accession_workflow(self):
             'project_accession': self.project_accession,
             'instance_id': self.eload_cfg.query(self.config_section, 'accession', 'instance_id'),
             'accession_job_props': job_props,
+            'public_ftp_dir': cfg['public_ftp_dir'],
             'public_dir': os.path.join(self.project_dir, project_dirs['public']),
             'logs_dir': os.path.join(self.project_dir, project_dirs['logs']),
             'executable': cfg['executable'],

nextflow/accession.nf

@@ -9,6 +9,7 @@ def helpMessage() {
             --project_accession     project accession
             --instance_id           instance id to run accessioning
             --accession_job_props   job-specific properties, passed as a map
+            --public_ftp_dir        public FTP directory
             --public_dir            directory for files to be made public
             --logs_dir              logs directory
     """
@@ -18,10 +19,11 @@ params.valid_vcfs = null
 params.project_accession = null
 params.instance_id = null
 params.accession_job_props = null
+params.public_ftp_dir = null
 params.public_dir = null
 params.logs_dir = null
 // executables
-params.executable = ["bcftools": "bcftools", "tabix": "tabix", "copy_to_ftp": "copy_to_ftp"]
+params.executable = ["bcftools": "bcftools", "tabix": "tabix"]
 // java jars
 params.jar = ["accession_pipeline": "accession_pipeline"]
 // help
@@ -31,11 +33,12 @@ params.help = null
 if (params.help) exit 0, helpMessage()
 
 // Test input files
-if (!params.valid_vcfs || !params.project_accession || !params.instance_id || !params.accession_job_props || !params.public_dir || !params.logs_dir) {
+if (!params.valid_vcfs || !params.project_accession || !params.instance_id || !params.accession_job_props || !params.public_ftp_dir || !params.public_dir || !params.logs_dir) {
     if (!params.valid_vcfs) log.warn('Provide validated vcfs using --valid_vcfs')
     if (!params.project_accession) log.warn('Provide a project accession using --project_accession')
     if (!params.instance_id) log.warn('Provide an instance id using --instance_id')
     if (!params.accession_job_props) log.warn('Provide job-specific properties using --accession_job_props')
+    if (!params.public_ftp_dir) log.warn('Provide public FTP directory using --public_ftp_dir')
     if (!params.public_dir) log.warn('Provide public directory using --public_dir')
     if (!params.logs_dir) log.warn('Provide logs directory using --logs_dir')
     exit 1, helpMessage()
@@ -171,6 +174,7 @@ process csi_index_vcf {
 
     """
     cd $params.public_dir
-    $params.executable.copy_to_ftp $params.project_accession
+    rsync -va * ${params.public_ftp_dir}/${params.project_accession}
+    ls -l ${params.public_ftp_dir}/${params.project_accession}/*
     """
  }

tests/nextflow-tests/clean_tests.sh

@@ -7,5 +7,5 @@ cwd=${PWD}
 cd ${SCRIPT_DIR}
 
 rm -rf work .nextflow*
-rm -r project
+rm -r project ftp
 cd ${cwd}

tests/nextflow-tests/test_ingestion_config.yaml

@@ -1,5 +1,6 @@
 project_accession: PRJEB12345
 instance_id: 1
+public_ftp_dir: ../../ftp
 logs_dir: ../../../project/logs
 public_dir: ../../../project/public
 valid_vcfs:
@@ -19,7 +20,6 @@ executable:
   bcftools: ../../../bin/fake_bcftools.sh
   bgzip: ../../../bin/fake_bgzip.sh
   tabix: ../../../bin/fake_tabix.sh
-  copy_to_ftp: ../../bin/fake_copy.sh
 jar:
   accession_pipeline: ../../../java/accession.jar
   eva_pipeline: ../../../java/variant-load.jar

tests/nextflow-tests/test_ingestion_nextflow.sh

@@ -7,7 +7,7 @@ SOURCE_DIR="$(dirname $(dirname $SCRIPT_DIR))/nextflow"
 
 cwd=${PWD}
 cd ${SCRIPT_DIR}
-mkdir project project/logs project/public
+mkdir project project/logs project/public ftp
 
 # run accession and variant load
 # note public_dir needs to be an absolute path, unlike others in config
@@ -26,5 +26,5 @@ cat project/logs/*.log
 
 # clean up
 rm -rf work .nextflow*
-rm -r project
+rm -r project ftp
 cd ${cwd}

tests/resources/submission_config.yml

@@ -1,4 +1,5 @@
 ftp_dir: 'tests/resources/ftpboxes'
+public_ftp_dir: '/path/to/public/ftp'
 eloads_dir: 'tests/resources/eloads'
 projects_dir: 'tests/resources/projects'
 eva_pipeline_props: '/path/to/pipeline/properties'