Merge branch 'refs/heads/devel'

.gitmodules

@@ -61,3 +61,6 @@
 [submodule "src/ingest-pipeline/airflow/dags/cwl/multiome-rna-atac-pipeline"]
 	path = src/ingest-pipeline/airflow/dags/cwl/multiome-rna-atac-pipeline
 	url = https://github.com/hubmapconsortium/multiome-rna-atac-pipeline
+[submodule "src/ingest-pipeline/airflow/dags/cwl/deepcelltypes"]
+	path = src/ingest-pipeline/airflow/dags/cwl/deepcelltypes
+	url = https://github.com/hubmapconsortium/deepcelltypes

src/ingest-pipeline/airflow/dags/codex_cytokit.py

@@ -61,6 +61,7 @@
         illumination_first_stitching=steps_dir / 'illumination_first_stitching.cwl',
         cytokit=steps_dir / 'run_cytokit.cwl',
         ometiff_second_stitching=steps_dir / 'ometiff_second_stitching.cwl',
+        deepcelltypes=Path('deepcelltypes', 'run_deepcelltypes.cwl'),
         sprm=Path('sprm', 'pipeline.cwl'),
         create_vis_symlink_archive=Path('create-vis-symlink-archive', 'pipeline.cwl'),
         ome_tiff_pyramid=Path('ome-tiff-pyramid', 'pipeline.cwl'),
@@ -215,7 +216,7 @@ def build_cwltool_cwl_ometiff_second_stitching(**kwargs):
         python_callable=utils.pythonop_maybe_keep,
         provide_context=True,
         op_kwargs={
-            'next_op': 'prepare_cwl_sprm',
+            'next_op': 'prepare_cwl_deepcelltypes',
             'bail_op': 'set_dataset_error',
             'test_op': 'pipeline_exec_cwl_ometiff_second_stitching',
         }
@@ -230,6 +231,52 @@ def build_cwltool_cwl_ometiff_second_stitching(**kwargs):
         """
     )
 
+    prepare_cwl_deepcelltypes = DummyOperator(task_id='prepare_cwl_deepcelltypes')
+
+    def build_cwltool_cmd_deepcelltypes(**kwargs):
+        run_id = kwargs['run_id']
+        tmpdir = get_tmp_dir_path(run_id)
+        print('tmpdir: ', tmpdir)
+        parent_data_dir = get_parent_data_dir(**kwargs)
+        print('parent_data_dir: ', parent_data_dir)
+        data_dir = tmpdir / 'cwl_out'
+        print('data_dir: ', data_dir)
+
+        command = [
+            *get_cwltool_base_cmd(tmpdir),
+            cwl_workflows['deepcelltypes'],
+            '--data_dir',
+            data_dir / 'pipeline_output',
+        ]
+
+        return join_quote_command_str(command)
+
+
+    t_build_cmd_deepcelltypes = PythonOperator(
+        task_id='build_cmd_deepcelltypes',
+        python_callable=build_cwltool_cmd_deepcelltypes,
+        provide_context=True,
+        )
+
+    t_pipeline_exec_cwl_deepcelltypes = BashOperator(
+        task_id='pipeline_exec_cwl_deepcelltypes',
+        bash_command=""" \
+        tmp_dir={{tmp_dir_path(run_id)}} ; \
+        cd ${tmp_dir}/cwl_out ; \
+        {{ti.xcom_pull(task_ids='build_cmd_deepcelltypes')}} >> ${tmp_dir}/session.log 2>&1 ; \
+        echo $?
+        """
+    )
+
+    t_maybe_keep_cwl_deepcelltypes = BranchPythonOperator(
+        task_id='maybe_keep_cwl_deepcelltypes',
+        python_callable=utils.pythonop_maybe_keep,
+        provide_context=True,
+        op_kwargs={'next_op': 'prepare_cwl_sprm',
+                   'bail_op': 'set_dataset_error',
+                   'test_op': 'pipeline_exec_cwl_deepcelltypes'}
+        )
+
     prepare_cwl_sprm = DummyOperator(task_id='prepare_cwl_sprm')
 
     def build_cwltool_cmd_sprm(**kwargs):
@@ -249,6 +296,8 @@ def build_cwltool_cmd_sprm(**kwargs):
             data_dir / 'pipeline_output/expr',
             '--mask_dir',
             data_dir / 'pipeline_output/mask',
+            '--cell_types_file',
+            data_dir / 'deepcelltypes_predictions.csv',
         ]
 
         return join_quote_command_str(command)

src/ingest-pipeline/airflow/dags/launch_multi_analysis.py

@@ -64,7 +64,7 @@ def check_one_uuid(
         - data type(s) of the dataset
         - local directory full path of the dataset
         """
-        print(f"Starting uuid {uuid}")
+        print(f"Starting uuid {uuid} with avoid previous version {avoid_previous_version}")
         my_callable = lambda **kwargs: uuid
         ds_rslt = utils.pythonop_get_dataset_state(dataset_uuid_callable=my_callable, **kwargs)
         if not ds_rslt:

src/ingest-pipeline/misc/tools/survey.py

@@ -8,7 +8,6 @@
 import pandas as pd
 import requests
 
-from hubmap_commons.type_client import TypeClient
 from hubmap_commons.hm_auth import AuthHelper
 
 # No trailing slashes in the following URLs!
@@ -612,7 +611,6 @@ def __init__(self, auth_tok=None, instance=None):
         assert (instance is None
                 or instance in ['PROD', 'STAGE', 'TEST', 'DEV']), 'invalid instance'
         self.instance = instance or 'PROD'
-        # self.type_client = TypeClient(ENDPOINTS[self.instance]['assay_info_url'])
 
     def fetch_new_dataset_table(self):
         """