Merge pull request #57 from nkongenelly/DATAOPS-467

Dataops 467: Removed runfolder directory created in delivery_runfolders_for_project_workflow

.github/workflows/tests.yml

@@ -1,6 +1,6 @@
 name: Run Unit Tests
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:

delivery/services/delivery_service.py

@@ -82,7 +82,9 @@ def _create_links_area_for_project_runfolders(self, project_name, projects, batc
         for project in projects:
             try:
                 link_name = os.path.join(project_dir, project.runfolder_name)
-                self.file_system_service.symlink(project.path, link_name)
+                self.file_system_service.symlink(
+                    os.path.join(project.path, project.runfolder_name), link_name
+                )
             except FileExistsError as e:
                 log.error("Project link: {} already exists".format(project_dir))
                 raise e

delivery/services/organise_service.py

@@ -115,7 +115,7 @@ def organise_project(
                     sample,
                     organised_project_runfolder_path,
                     lanes))
-        # symlink the project files
+        # symlink the project files    
         organised_project_files = []
         if project.project_files:
             for project_file in project.project_files:
@@ -147,7 +147,6 @@ def organise_project_file(self, project_file, organised_project_path):
         before organisation
         :param organised_project_path: path where the project will be organised
         """
-
         # the relative path from the project file base to the project file (e.g. plots/filename.png)
         relpath = self.file_system_service.relpath(
             project_file.file_path,

delivery/services/staging_service.py

@@ -91,7 +91,6 @@ def _copy_dir(staging_order_id, external_program_service, session_factory, stagi
             cmd = ['rsync', '--stats', '-r', '--copy-links', '--times',
                    staging_source_with_trailing_slash, staging_order.staging_target]
             log.debug("Running rsync with command: {}".format(" ".join(cmd)))
-
             execution = external_program_service.run(cmd)
 
             staging_order.pid = execution.pid
@@ -103,11 +102,12 @@ def _copy_dir(staging_order_id, external_program_service, session_factory, stagi
 
                 # Parse the file size from the output of rsync stats:
                 # Total file size: 207,707,566 bytes
-                match = re.search(r'Total file size: ([\d,]+) bytes',
-                                  execution_result.stdout,
-                                  re.MULTILINE)
+
+                match = re.search(r'Total file size: ([\d,.]+) bytes',
+                                execution_result.stdout,
+                                re.MULTILINE)
                 size_of_transfer = match.group(1)
-                size_of_transfer = int(size_of_transfer.replace(",", ""))
+                size_of_transfer = int(size_of_transfer.replace(",", "").replace(".", ""))
                 staging_order.size = size_of_transfer
 
                 staging_order.status = StagingStatus.staging_successful

tests/integration_tests/base.py

@@ -30,8 +30,10 @@ def __init__(self, *args):
         # Default duration of mock delivery
         self.mock_duration = 0.1
 
-    def _create_projects_dir_with_random_data(self, base_dir, proj_name='ABC_123'):
+    def _create_projects_dir_with_random_data(self, base_dir, proj_name='ABC_123', runfolder_name=None):
         tmp_proj_dir = os.path.join(base_dir, 'Projects', proj_name)
+        if runfolder_name:
+            tmp_proj_dir = os.path.join(tmp_proj_dir, runfolder_name)
         os.makedirs(tmp_proj_dir)
         with open(os.path.join(tmp_proj_dir, 'test_file'), 'wb') as f:
             f.write(os.urandom(1024))

tests/integration_tests/test_integration_dds.py

@@ -1,3 +1,4 @@
+import os
 import json
 import time
 import tempfile
@@ -7,7 +8,7 @@
 from delivery.models.db_models import StagingStatus, DeliveryStatus
 
 from tests.integration_tests.base import BaseIntegration
-
+from tests.test_utils import unorganised_runfolder
 
 class TestIntegrationDDS(BaseIntegration):
     @gen_test
@@ -118,8 +119,8 @@ def test_can_stage_and_deliver_clean_flowcells(self):
                                          prefix='160930_ST-E00216_0555_BH37CWALXX_') as tmpdir1,\
              tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/',
                                          prefix='160930_ST-E00216_0556_BH37CWALXX_') as tmpdir2:
-                self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123')
-                self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123')
+                self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123', os.path.basename(tmpdir1))
+                self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123', os.path.basename(tmpdir2))
 
                 url = "/".join([self.API_BASE, "stage", "project", 'runfolders', 'XYZ_123'])
                 payload = {'delivery_mode': 'CLEAN'}
@@ -146,12 +147,13 @@ def test_can_stage_and_deliver_batch_flowcells(self):
                                          prefix='160930_ST-E00216_0555_BH37CWALXX_') as tmpdir1, \
                 tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/',
                                             prefix='160930_ST-E00216_0556_BH37CWALXX_') as tmpdir2:
-            self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123')
-            self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123')
+            self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123', os.path.basename(tmpdir1))
+            self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123', os.path.basename(tmpdir2))
 
             url = "/".join([self.API_BASE, "stage", "project", 'runfolders', 'XYZ_123'])
             payload = {'delivery_mode': 'BATCH'}
             response = yield self.http_client.fetch(self.get_url(url), method='POST', body=json.dumps(payload))
+
             self.assertEqual(response.code, 202)
 
             payload = {'delivery_mode': 'BATCH'}
@@ -171,14 +173,20 @@ def test_can_stage_and_deliver_batch_flowcells(self):
                 status_response = yield self.http_client.fetch(link)
                 self.assertEqual(json.loads(status_response.body)["status"], StagingStatus.staging_successful.name)
 
+    def create_unorganised_test_runfolders(self, tmpdir):
+            return unorganised_runfolder(
+                name=os.path.basename(tmpdir),
+                root_path=os.path.dirname(tmpdir)
+            )
+
     @gen_test
     def test_can_stage_and_deliver_force_flowcells(self):
         with tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/',
                                          prefix='160930_ST-E00216_0555_BH37CWALXX_') as tmpdir1, \
                 tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/',
                                             prefix='160930_ST-E00216_0556_BH37CWALXX_') as tmpdir2:
-            self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123')
-            self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123')
+            self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123', os.path.basename(tmpdir1))
+            self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123', os.path.basename(tmpdir2))
 
             # First just stage it
             url = "/".join([self.API_BASE, "stage", "project", 'runfolders', 'XYZ_123'])
@@ -209,6 +217,52 @@ def test_can_stage_and_deliver_force_flowcells(self):
                 status_response = yield self.http_client.fetch(link)
                 self.assertEqual(json.loads(status_response.body)["status"], StagingStatus.staging_successful.name)
 
+    @gen_test
+    def test_can_organise_stage_and_deliver_force_flowcells(self):
+        with tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/',
+                                         prefix='160930_ST-E00216_0555_BH37CWALXX_') as tmpdir1, \
+                tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/',
+                                            prefix='160930_ST-E00216_0556_BH37CWALXX_') as tmpdir2:
+            # First organise
+            unorganised_runfolder1 = self.create_unorganised_test_runfolders(tmpdir1)
+            unorganised_runfolder2 = self.create_unorganised_test_runfolders(tmpdir2)
+
+            self._create_runfolder_structure_on_disk(unorganised_runfolder1)
+            self._create_runfolder_structure_on_disk(unorganised_runfolder2)
+
+            url = "/".join([self.API_BASE, "organise", "runfolder", unorganised_runfolder1.name])
+            response1 = yield self.http_client.fetch(self.get_url(url), method='POST', body='')
+            self.assertEqual(response1.code, 200)
+
+            url = "/".join([self.API_BASE, "organise", "runfolder", unorganised_runfolder2.name])
+            response2 = yield self.http_client.fetch(self.get_url(url), method='POST', body='')
+            self.assertEqual(response2.code, 200)
+
+            # Then stage it
+            url = "/".join([self.API_BASE, "stage", "project", 'runfolders', 'JKL_123'])
+            payload = {'delivery_mode': 'FORCE'}
+            response_forced = yield self.http_client.fetch(self.get_url(url), method='POST', body=json.dumps(payload))
+            self.assertEqual(response_forced.code, 202)
+
+            response_json = json.loads(response_forced.body)
+
+            staging_status_links = response_json.get("staging_order_links")
+            staging_order_ids = response_json.get("staging_order_ids")
+
+            # Insert a pause to allow staging to complete
+            time.sleep(1)
+
+            for project, link in staging_status_links.items():
+                self.assertEqual(project, 'JKL_123')
+
+                status_response = yield self.http_client.fetch(link)
+                self.assertEqual(json.loads(status_response.body)["status"], StagingStatus.staging_successful.name)
+
+            # Assert the staged folder structure has only one runfolder folder
+            temp_staging_dir = f"/tmp/{staging_order_ids.get('JKL_123')}/JKL_123"
+            for runfolder in os.listdir(temp_staging_dir):
+                self.assertFalse(runfolder in os.listdir(f"{temp_staging_dir}/{runfolder}"))
+
     @gen_test
     def test_can_create_project(self):
         project_name = "CD-1234"

tests/resources/readme/README.md → tests/resources/README.md

@@ -22,4 +22,4 @@ This delivery includes sequencing data in FASTQ format, a summary report created
     └── <SampleN id>
         ├── <sampleN name>_<sample number>_R1_001.fastq.gz
         └── <sampleN name>_<sample number>_R2_001.fastq.gz
-```
+```