Merge pull request #1320 from Sage-Bionetworks/develop-workflow-inves…

…tigate-FDS-84

Reduce execution time for `Test Schematic` workflow

.github/workflows/test.yml

@@ -25,7 +25,7 @@ concurrency:
   cancel-in-progress: true
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04-4core-16GBRAM-150GBSSD
     env:
       POETRY_VERSION:  1.3.0
     strategy:
@@ -128,7 +128,7 @@ jobs:
         run: >
           source .venv/bin/activate;
           pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov=schematic/
-          -m "not (google_credentials_needed or schematic_api or table_operations)"
+          -m "not (google_credentials_needed or schematic_api or table_operations)" --reruns 2 -n auto
 
       - name: Run tests
         env:
@@ -138,7 +138,7 @@ jobs:
         run: >
           source .venv/bin/activate;
           pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov=schematic/
-          -m "not (google_credentials_needed or rule_combos or schematic_api or table_operations)"
+          -m "not (google_credentials_needed or rule_combos or schematic_api or table_operations)" --reruns 2  -n auto
 
       - name: Upload pytest test results
         uses: actions/upload-artifact@v2

pyproject.toml

@@ -78,11 +78,13 @@ typing-extensions = "<4.6.0"
 pytest = "^7.0.0"
 pytest-cov = "^4.0.0"
 pytest-mock = "^3.5.1"
+pytest-rerunfailures = "^12.0"
 flake8 = "^6.0.0"
 python-dotenv = "^0.21.0"
 black = "^23.7.0"
 mypy = "^1.4.1"
 pylint = "^2.16.1"
+pytest-xdist = "^3.5.0"
 
 [tool.poetry.group.aws]
 optional = true

schematic/store/synapse.py

@@ -1896,13 +1896,6 @@ def getDatasetAnnotations(
     def raise_final_error(retry_state):
         return retry_state.outcome.result()
 
-    @retry(stop = stop_after_attempt(5), 
-            wait = wait_chain(*[wait_fixed(10) for i in range (2)] + 
-                    [wait_fixed(15) for i in range(2)] + 
-                    [wait_fixed(20)]),
-            retry=retry_if_exception_type(LookupError),
-            retry_error_callback = raise_final_error)
-
     def checkIfinAssetView(self, syn_id) -> str:
         # get data in administrative fileview for this pipeline
         assetViewTable = self.getStorageFileviewTable()
@@ -1912,6 +1905,12 @@ def checkIfinAssetView(self, syn_id) -> str:
         else: 
             return False
 
+    @retry(stop = stop_after_attempt(5), 
+            wait = wait_chain(*[wait_fixed(10) for i in range (2)] + 
+                    [wait_fixed(15) for i in range(2)] + 
+                    [wait_fixed(20)]),
+            retry=retry_if_exception_type(LookupError),
+            retry_error_callback = raise_final_error)
     def getDatasetProject(self, datasetId: str) -> str:
         """Get parent project for a given dataset ID.
 

tests/conftest.py

@@ -7,10 +7,12 @@
 import pytest
 import pandas as pd
 from dotenv import load_dotenv, find_dotenv
+from time import perf_counter
 
 from schematic.schemas.explorer import SchemaExplorer
 from schematic.configuration.configuration import CONFIG
 from schematic.utils.df_utils import load_df
+from schematic.store.synapse import SynapseStorage
 
 load_dotenv()
 
@@ -90,3 +92,13 @@ def helpers():
 @pytest.fixture(scope="session")
 def config():
     yield CONFIG
+
+@pytest.fixture(scope="session")
+def synapse_store(request):
+    access_token = os.getenv("SYNAPSE_ACCESS_TOKEN")
+    if access_token:
+        synapse_store = SynapseStorage(access_token=access_token)
+    else:
+        synapse_store = SynapseStorage()
+
+    yield synapse_store

tests/test_store.py

@@ -12,29 +12,18 @@
 from synapseclient import EntityViewSchema, Folder
 from synapseclient.core.exceptions import SynapseHTTPError
 from synapseclient.entity import File
-from tenacity import (RetryError, Retrying, stop_after_attempt,
-                      wait_random_exponential)
 
 from schematic.configuration.configuration import Configuration
 from schematic.models.metadata import MetadataModel
 from schematic.schemas.generator import SchemaGenerator
 from schematic.store.base import BaseStorage
-from schematic.store.synapse import (DatasetFileView, ManifestDownload,
-                                     SynapseStorage)
+from schematic.store.synapse import (DatasetFileView, 
+                                    ManifestDownload,)
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
 
-@pytest.fixture
-def synapse_store():
-    access_token = os.getenv("SYNAPSE_ACCESS_TOKEN")
-    if access_token:
-        synapse_store = SynapseStorage(access_token=access_token)
-    else:
-        synapse_store = SynapseStorage()
-    yield synapse_store
-
 @pytest.fixture
 def test_download_manifest_id():
     yield "syn51203973"
@@ -121,13 +110,17 @@ def test_getFileAnnotations(self, synapse_store):
 
     @pytest.mark.parametrize('only_new_files',[True, False])
     def test_get_file_entityIds(self, helpers, synapse_store, only_new_files):
+        #TODO: Automatically reset manifest at path specified below after each test 
+        # so that subsequent runs do not affect each other
         manifest_path = "mock_manifests/test_BulkRNAseq.csv"
         dataset_files = synapse_store.getFilesInStorageDataset('syn39241199')
 
         if only_new_files:
             # Prepare manifest is getting Ids for new files only
             manifest = helpers.get_data_frame(manifest_path)
             entityIds = pd.DataFrame({'entityId': ['syn39242580', 'syn51900502']})
+
+            # If this line errors out then the changes on the manifest file need to be discarded
             manifest = manifest.join(entityIds)
 
             # get entityIds for new files
@@ -153,24 +146,15 @@ def test_annotation_submission(self, synapse_store, helpers, manifest_path, test
         # Upload dataset annotations
         sg = SchemaGenerator(config.model_location)
 
-        try:        
-            for attempt in Retrying(
-                stop = stop_after_attempt(15),
-                wait = wait_random_exponential(multiplier=1,min=10,max=120),
-                retry_error_callback = raise_final_error
-                ):
-                with attempt:         
-                    manifest_id = synapse_store.associateMetadataWithFiles(
-                        schemaGenerator = sg,
-                        metadataManifestPath = helpers.get_data_path(manifest_path),
-                        datasetId = datasetId,
-                        manifest_record_type = manifest_record_type,
-                        useSchemaLabel = True,
-                        hideBlanks = True,
-                        restrict_manifest = False,
-                    )
-        except RetryError:
-            pass
+        manifest_id = synapse_store.associateMetadataWithFiles(
+            schemaGenerator = sg,
+            metadataManifestPath = helpers.get_data_path(manifest_path),
+            datasetId = datasetId,
+            manifest_record_type = manifest_record_type,
+            useSchemaLabel = True,
+            hideBlanks = True,
+            restrict_manifest = False,
+        )
 
         # Retrive annotations
         entity_id = helpers.get_data_frame(manifest_path)["entityId"][0]

tests/test_utils.py

@@ -20,7 +20,6 @@
                                   MissingConfigValueError)
 from schematic.schemas import df_parser
 from schematic.schemas.explorer import SchemaExplorer
-from schematic.store.synapse import SynapseStorage
 from schematic.utils import (cli_utils, df_utils, general, io_utils,
                              validate_utils)
 from schematic.utils.general import (calculate_datetime,
@@ -32,15 +31,6 @@
 
 IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS")
 
-@pytest.fixture
-def synapse_store():
-    access_token = os.getenv("SYNAPSE_ACCESS_TOKEN")
-    if access_token:
-        synapse_store = SynapseStorage(access_token=access_token)
-    else:
-        synapse_store = SynapseStorage()
-    yield synapse_store
-
 class TestGeneral:
     def test_clear_synapse_cache(self, tmp_path):
         # define location of mock synapse cache