diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4bb7172..8fceb34 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,6 @@ name: Run Unit Tests -on: [push] +on: [push, pull_request] jobs: build: diff --git a/delivery/services/delivery_service.py b/delivery/services/delivery_service.py index d09c968..d6d2991 100644 --- a/delivery/services/delivery_service.py +++ b/delivery/services/delivery_service.py @@ -82,7 +82,9 @@ def _create_links_area_for_project_runfolders(self, project_name, projects, batc for project in projects: try: link_name = os.path.join(project_dir, project.runfolder_name) - self.file_system_service.symlink(project.path, link_name) + self.file_system_service.symlink( + os.path.join(project.path, project.runfolder_name), link_name + ) except FileExistsError as e: log.error("Project link: {} already exists".format(project_dir)) raise e diff --git a/delivery/services/organise_service.py b/delivery/services/organise_service.py index 49f927b..25b70af 100644 --- a/delivery/services/organise_service.py +++ b/delivery/services/organise_service.py @@ -115,7 +115,7 @@ def organise_project( sample, organised_project_runfolder_path, lanes)) - # symlink the project files + # symlink the project files organised_project_files = [] if project.project_files: for project_file in project.project_files: @@ -147,7 +147,6 @@ def organise_project_file(self, project_file, organised_project_path): before organisation :param organised_project_path: path where the project will be organised """ - # the relative path from the project file base to the project file (e.g. plots/filename.png) relpath = self.file_system_service.relpath( project_file.file_path, diff --git a/delivery/services/staging_service.py b/delivery/services/staging_service.py index 0052487..fb23639 100644 --- a/delivery/services/staging_service.py +++ b/delivery/services/staging_service.py @@ -91,7 +91,6 @@ def _copy_dir(staging_order_id, external_program_service, session_factory, stagi cmd = ['rsync', '--stats', '-r', '--copy-links', '--times', staging_source_with_trailing_slash, staging_order.staging_target] log.debug("Running rsync with command: {}".format(" ".join(cmd))) - execution = external_program_service.run(cmd) staging_order.pid = execution.pid @@ -103,11 +102,12 @@ def _copy_dir(staging_order_id, external_program_service, session_factory, stagi # Parse the file size from the output of rsync stats: # Total file size: 207,707,566 bytes - match = re.search(r'Total file size: ([\d,]+) bytes', - execution_result.stdout, - re.MULTILINE) + + match = re.search(r'Total file size: ([\d,.]+) bytes', + execution_result.stdout, + re.MULTILINE) size_of_transfer = match.group(1) - size_of_transfer = int(size_of_transfer.replace(",", "")) + size_of_transfer = int(size_of_transfer.replace(",", "").replace(".", "")) staging_order.size = size_of_transfer staging_order.status = StagingStatus.staging_successful diff --git a/tests/integration_tests/base.py b/tests/integration_tests/base.py index 13cbbcf..f2942f2 100644 --- a/tests/integration_tests/base.py +++ b/tests/integration_tests/base.py @@ -30,8 +30,10 @@ def __init__(self, *args): # Default duration of mock delivery self.mock_duration = 0.1 - def _create_projects_dir_with_random_data(self, base_dir, proj_name='ABC_123'): + def _create_projects_dir_with_random_data(self, base_dir, proj_name='ABC_123', runfolder_name=None): tmp_proj_dir = os.path.join(base_dir, 'Projects', proj_name) + if runfolder_name: + tmp_proj_dir = os.path.join(tmp_proj_dir, runfolder_name) os.makedirs(tmp_proj_dir) with open(os.path.join(tmp_proj_dir, 'test_file'), 'wb') as f: f.write(os.urandom(1024)) diff --git a/tests/integration_tests/test_integration_dds.py b/tests/integration_tests/test_integration_dds.py index 7234440..8fa7947 100644 --- a/tests/integration_tests/test_integration_dds.py +++ b/tests/integration_tests/test_integration_dds.py @@ -1,3 +1,4 @@ +import os import json import time import tempfile @@ -7,7 +8,7 @@ from delivery.models.db_models import StagingStatus, DeliveryStatus from tests.integration_tests.base import BaseIntegration - +from tests.test_utils import unorganised_runfolder class TestIntegrationDDS(BaseIntegration): @gen_test @@ -118,8 +119,8 @@ def test_can_stage_and_deliver_clean_flowcells(self): prefix='160930_ST-E00216_0555_BH37CWALXX_') as tmpdir1,\ tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/', prefix='160930_ST-E00216_0556_BH37CWALXX_') as tmpdir2: - self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123') - self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123') + self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123', os.path.basename(tmpdir1)) + self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123', os.path.basename(tmpdir2)) url = "/".join([self.API_BASE, "stage", "project", 'runfolders', 'XYZ_123']) payload = {'delivery_mode': 'CLEAN'} @@ -146,12 +147,13 @@ def test_can_stage_and_deliver_batch_flowcells(self): prefix='160930_ST-E00216_0555_BH37CWALXX_') as tmpdir1, \ tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/', prefix='160930_ST-E00216_0556_BH37CWALXX_') as tmpdir2: - self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123') - self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123') + self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123', os.path.basename(tmpdir1)) + self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123', os.path.basename(tmpdir2)) url = "/".join([self.API_BASE, "stage", "project", 'runfolders', 'XYZ_123']) payload = {'delivery_mode': 'BATCH'} response = yield self.http_client.fetch(self.get_url(url), method='POST', body=json.dumps(payload)) + self.assertEqual(response.code, 202) payload = {'delivery_mode': 'BATCH'} @@ -171,14 +173,20 @@ def test_can_stage_and_deliver_batch_flowcells(self): status_response = yield self.http_client.fetch(link) self.assertEqual(json.loads(status_response.body)["status"], StagingStatus.staging_successful.name) + def create_unorganised_test_runfolders(self, tmpdir): + return unorganised_runfolder( + name=os.path.basename(tmpdir), + root_path=os.path.dirname(tmpdir) + ) + @gen_test def test_can_stage_and_deliver_force_flowcells(self): with tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/', prefix='160930_ST-E00216_0555_BH37CWALXX_') as tmpdir1, \ tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/', prefix='160930_ST-E00216_0556_BH37CWALXX_') as tmpdir2: - self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123') - self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123') + self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123', os.path.basename(tmpdir1)) + self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123', os.path.basename(tmpdir2)) # First just stage it url = "/".join([self.API_BASE, "stage", "project", 'runfolders', 'XYZ_123']) @@ -209,6 +217,52 @@ def test_can_stage_and_deliver_force_flowcells(self): status_response = yield self.http_client.fetch(link) self.assertEqual(json.loads(status_response.body)["status"], StagingStatus.staging_successful.name) + @gen_test + def test_can_organise_stage_and_deliver_force_flowcells(self): + with tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/', + prefix='160930_ST-E00216_0555_BH37CWALXX_') as tmpdir1, \ + tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/', + prefix='160930_ST-E00216_0556_BH37CWALXX_') as tmpdir2: + # First organise + unorganised_runfolder1 = self.create_unorganised_test_runfolders(tmpdir1) + unorganised_runfolder2 = self.create_unorganised_test_runfolders(tmpdir2) + + self._create_runfolder_structure_on_disk(unorganised_runfolder1) + self._create_runfolder_structure_on_disk(unorganised_runfolder2) + + url = "/".join([self.API_BASE, "organise", "runfolder", unorganised_runfolder1.name]) + response1 = yield self.http_client.fetch(self.get_url(url), method='POST', body='') + self.assertEqual(response1.code, 200) + + url = "/".join([self.API_BASE, "organise", "runfolder", unorganised_runfolder2.name]) + response2 = yield self.http_client.fetch(self.get_url(url), method='POST', body='') + self.assertEqual(response2.code, 200) + + # Then stage it + url = "/".join([self.API_BASE, "stage", "project", 'runfolders', 'JKL_123']) + payload = {'delivery_mode': 'FORCE'} + response_forced = yield self.http_client.fetch(self.get_url(url), method='POST', body=json.dumps(payload)) + self.assertEqual(response_forced.code, 202) + + response_json = json.loads(response_forced.body) + + staging_status_links = response_json.get("staging_order_links") + staging_order_ids = response_json.get("staging_order_ids") + + # Insert a pause to allow staging to complete + time.sleep(1) + + for project, link in staging_status_links.items(): + self.assertEqual(project, 'JKL_123') + + status_response = yield self.http_client.fetch(link) + self.assertEqual(json.loads(status_response.body)["status"], StagingStatus.staging_successful.name) + + # Assert the staged folder structure has only one runfolder folder + temp_staging_dir = f"/tmp/{staging_order_ids.get('JKL_123')}/JKL_123" + for runfolder in os.listdir(temp_staging_dir): + self.assertFalse(runfolder in os.listdir(f"{temp_staging_dir}/{runfolder}")) + @gen_test def test_can_create_project(self): project_name = "CD-1234" diff --git a/tests/resources/readme/README.md b/tests/resources/README.md similarity index 99% rename from tests/resources/readme/README.md rename to tests/resources/README.md index a31412e..16318a0 100644 --- a/tests/resources/readme/README.md +++ b/tests/resources/README.md @@ -22,4 +22,4 @@ This delivery includes sequencing data in FASTQ format, a summary report created └── ├── __R1_001.fastq.gz └── __R2_001.fastq.gz -``` +``` \ No newline at end of file