From 869bdef70322c9edf1995d7cc5d35adb77e2abf0 Mon Sep 17 00:00:00 2001 From: sandyr Date: Wed, 12 Apr 2023 15:43:08 +0100 Subject: [PATCH 1/3] fixes serving of ro crates (zips) on nginx-less setup --- emgapi/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/emgapi/utils.py b/emgapi/utils.py index 0f7f2db90..4cd632a3e 100644 --- a/emgapi/utils.py +++ b/emgapi/utils.py @@ -19,7 +19,7 @@ from django.conf import settings from django.db.models import Q -from django.http import HttpResponse +from django.http import HttpResponse, FileResponse logger = logging.getLogger(__name__) @@ -120,6 +120,8 @@ def prepare_results_file_download_response(path_in_results, alias): if settings.DOWNLOADS_BYPASS_NGINX: logger.warning('DOWNLOADS_BYPASS_NGINX is true, so serving download directly as Django response ' '(not via NGINX redirect)') + if str(path_in_results).endswith('zip'): + return FileResponse(open(os.path.join(settings.RESULTS_DIR, path_in_results.lstrip('/')), 'rb')) with open(os.path.join(settings.RESULTS_DIR, path_in_results.lstrip('/')), 'r') as file: response.content = file.read() else: From ad15161d70a47b003c9c43ab6a0bb444a6523dda Mon Sep 17 00:00:00 2001 From: Sandy Rogers Date: Tue, 18 Apr 2023 16:24:23 +0100 Subject: [PATCH 2/3] RO-Crates for adding additional annotations to assemblies (#309) * adds webuploader handling for RO Crates * deps resolution * fixes for django 3.2.18 and associated deps upgrade --- emgapi/views.py | 4 +- emgapi/views_relations.py | 12 +-- emgapi/viewsets.py | 14 +-- .../import_extra_assembly_annotations.py | 91 ++++++++++++++++++- emgapianns/views.py | 2 +- emgapianns/viewsets.py | 2 +- requirements-dev.txt | 4 +- requirements-test.txt | 14 +-- requirements.txt | 40 ++++---- 9 files changed, 127 insertions(+), 56 deletions(-) diff --git a/emgapi/views.py b/emgapi/views.py index 9e40c9564..ee8e1450c 100644 --- a/emgapi/views.py +++ b/emgapi/views.py @@ -1301,7 +1301,7 @@ class GenomeCatalogueViewSet(mixins.RetrieveModelMixin, emg_mixins.ListModelMixin, emg_viewsets.BaseGenomeCatalogueGenericViewSet): - filter_class = emg_filters.GenomeCatalogueFilter + filterset_class = emg_filters.GenomeCatalogueFilter lookup_field = 'catalogue_id' lookup_value_regex = '[^/]+' @@ -1352,7 +1352,7 @@ class GenomeViewSet(mixins.RetrieveModelMixin, emg_mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = emg_serializers.GenomeSerializer - filter_class = emg_filters.GenomeFilter + filterset_class = emg_filters.GenomeFilter lookup_field = 'accession' lookup_value_regex = '[^/]+' diff --git a/emgapi/views_relations.py b/emgapi/views_relations.py index 5a1b3ee36..34cc5f9ba 100644 --- a/emgapi/views_relations.py +++ b/emgapi/views_relations.py @@ -316,7 +316,7 @@ class StudyAnalysisResultViewSet(emg_mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = emg_serializers.AnalysisSerializer - filter_class = emg_filters.AnalysisJobFilter + filterset_class = emg_filters.AnalysisJobFilter filter_backends = ( DjangoFilterBackend, @@ -430,7 +430,7 @@ class RunAnalysisViewSet(emg_mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = emg_serializers.AnalysisSerializer - filter_class = emg_filters.AnalysisJobFilter + filterset_class = emg_filters.AnalysisJobFilter filter_backends = ( DjangoFilterBackend, @@ -861,7 +861,7 @@ class BiomeTreeViewSet(mixins.ListModelMixin, serializer_class = emg_serializers.BiomeSerializer queryset = emg_models.Biome.objects.filter(depth=1) - filter_class = emg_filters.BiomeFilter + filterset_class = emg_filters.BiomeFilter filter_backends = ( DjangoFilterBackend, @@ -983,7 +983,7 @@ class RunAssemblyViewSet(emg_mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = emg_serializers.AssemblySerializer - filter_class = emg_filters.AssemblyFilter + filterset_class = emg_filters.AssemblyFilter filter_backends = ( DjangoFilterBackend, @@ -1026,7 +1026,7 @@ class AssemblyAnalysisViewSet(emg_mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = emg_serializers.AnalysisSerializer - filter_class = emg_filters.AnalysisJobFilter + filterset_class = emg_filters.AnalysisJobFilter filter_backends = ( DjangoFilterBackend, @@ -1073,7 +1073,7 @@ class AssemblyRunsViewSet(emg_mixins.ListModelMixin, serializer_class = emg_serializers.RunSerializer - filter_class = emg_filters.RunFilter + filterset_class = emg_filters.RunFilter filter_backends = ( DjangoFilterBackend, diff --git a/emgapi/viewsets.py b/emgapi/viewsets.py index ee84ca7db..9e9217fa5 100644 --- a/emgapi/viewsets.py +++ b/emgapi/viewsets.py @@ -31,7 +31,7 @@ class BaseSuperStudyViewSet(viewsets.GenericViewSet): serializer_class = emg_serializers.SuperStudySerializer - filter_class = emg_filters.SuperStudyFilter + filterset_class = emg_filters.SuperStudyFilter filter_backends = ( DjangoFilterBackend, @@ -57,7 +57,7 @@ class BaseStudyGenericViewSet(viewsets.GenericViewSet): serializer_class = emg_serializers.StudySerializer - filter_class = emg_filters.StudyFilter + filterset_class = emg_filters.StudyFilter filter_backends = ( DjangoFilterBackend, @@ -88,7 +88,7 @@ class BaseSampleGenericViewSet(viewsets.GenericViewSet): serializer_class = emg_serializers.SampleSerializer - filter_class = emg_filters.SampleFilter + filterset_class = emg_filters.SampleFilter filter_backends = ( DjangoFilterBackend, @@ -123,7 +123,7 @@ class BaseRunGenericViewSet(viewsets.GenericViewSet): serializer_class = emg_serializers.RunSerializer - filter_class = emg_filters.RunFilter + filterset_class = emg_filters.RunFilter filter_backends = ( DjangoFilterBackend, @@ -151,7 +151,7 @@ class BaseAssemblyGenericViewSet(viewsets.GenericViewSet): serializer_class = emg_serializers.AssemblySerializer - filter_class = emg_filters.AssemblyFilter + filterset_class = emg_filters.AssemblyFilter filter_backends = ( DjangoFilterBackend, @@ -177,7 +177,7 @@ class BaseAnalysisGenericViewSet(viewsets.GenericViewSet): serializer_class = emg_serializers.AnalysisSerializer - filter_class = emg_filters.AnalysisJobFilter + filterset_class = emg_filters.AnalysisJobFilter filter_backends = ( DjangoFilterBackend, @@ -210,7 +210,7 @@ class BasePublicationGenericViewSet(viewsets.GenericViewSet): serializer_class = emg_serializers.PublicationSerializer - filter_class = emg_filters.PublicationFilter + filterset_class = emg_filters.PublicationFilter filter_backends = ( DjangoFilterBackend, diff --git a/emgapianns/management/commands/import_extra_assembly_annotations.py b/emgapianns/management/commands/import_extra_assembly_annotations.py index c9424de26..182e833b0 100644 --- a/emgapianns/management/commands/import_extra_assembly_annotations.py +++ b/emgapianns/management/commands/import_extra_assembly_annotations.py @@ -11,7 +11,8 @@ class Command(BaseCommand): help = "Imports a directory of GFFs that as 'extra assembly annotations', " \ - "i.e. annotations from tools that aren't part of the analysis pipelines." + "i.e. annotations from tools that aren't part of the analysis pipelines." \ + "GFFs may (preferably) be wrapped into self-describing RO Crates." obj_list = list() results_directory = None @@ -33,14 +34,20 @@ def add_arguments(self, parser): 'gffs_directory', action='store', type=str, - help='The folder within `results_directory` where the GFF files are, e.g. "sanntis/"' + help='The folder within `results_directory` where the GFF/ROCrate files are, e.g. "crates/"' ) parser.add_argument( 'tool', action='store', type=str, - help='The type of annotation (e.g. sanntis)', - choices=['sanntis'] + help='The type of annotation (e.g. rocrate)', + choices=['sanntis', 'rocrate'] + ) + parser.add_argument( + 'rocrate_description', + action='store', + type=str, + help='A short description if the annotation is a RO Crate', ) def handle(self, *args, **options): @@ -69,6 +76,19 @@ def handle(self, *args, **options): logger.info(f'Will upload sanntis GFF for {erz}') self.upload_sanntis_gff_file(assembly, gffs_directory, file.name) + if options.get('tool') == 'rocrate': + logger.info('Looking for RO Crates (.zips') + for file in Path(self.gffs_dir).glob('**/*.zip'): + logger.info(f'Handling RO Crate Zip file {file}') + erz = 'ERZ' + file.name.split('ERZ')[1].strip('.zip') + try: + assembly = emg_models.Assembly.objects.get(accession=erz) + except emg_models.Assembly.DoesNotExist: + logger.warning(f'No Assembly found for RO Crate apparent ERZ {erz}') + continue + logger.info(f'Will upload RO Crate for {erz}') + self.upload_rocrate(assembly, gffs_directory, file.name) + def upload_sanntis_gff_file( self, assembly, @@ -126,3 +146,66 @@ def upload_sanntis_gff_file( logger.info(f'{"Created" if created else "Updated"} download {dl}') return dl + + def upload_rocrate( + self, + assembly, + subdir, + filename, + ): + description_label = self.desc_label_cache.get('Analysis RO Crate') + if not description_label: + description_label, created = emg_models.DownloadDescriptionLabel \ + .objects \ + .get_or_create(description_label='Analysis RO Crate', defaults={ + "description": "Self-describing analysis workflow product packaged as RO Crate" + }) + if created: + logger.info(f'Added new download description label {description_label}') + self.desc_label_cache[description_label.description_label] = description_label + + fmt = self.fmt_cache.get('RO Crate') + if not fmt: + fmt, created = emg_models.FileFormat \ + .objects \ + .get_or_create(format_name='RO Crate', defaults={ + "format_extension": "zip", + "compression": True + }) + if created: + logger.info(f'Added new file format {fmt}') + self.fmt_cache[fmt.format_name] = fmt + + subdir_obj = self.subdir_cache.get(subdir) + if not subdir_obj: + subdir_obj, created = emg_models.DownloadSubdir.objects.get_or_create(subdir=subdir) + if created: + logger.info(f'Added new downloads subdir {subdir_obj}') + self.subdir_cache[subdir] = subdir_obj + + group = self.group_cache.get('Analysis RO Crate') + if not group: + group, created = emg_models.DownloadGroupType.objects.get_or_create(group_type='Analysis RO Crate') + if created: + logger.info(f'Added new download group type {group}') + self.group_cache[group.group_type] = group + + alias = os.path.basename(filename) + + defaults = { + 'alias': alias, + 'description': description_label, + 'file_format': fmt, + 'group_type': group, + 'realname': os.path.basename(filename), + 'subdir': subdir_obj + } + + dl, created = emg_models.AssemblyExtraAnnotation.objects.update_or_create( + defaults, + assembly=assembly, + alias=alias, + ) + + logger.info(f'{"Created" if created else "Updated"} download {dl}') + return dl diff --git a/emgapianns/views.py b/emgapianns/views.py index 5b32e8c3e..943c0ba33 100644 --- a/emgapianns/views.py +++ b/emgapianns/views.py @@ -871,7 +871,7 @@ class OrganismAnalysisRelationshipViewSet(m_viewsets.ListReadOnlyModelViewSet): serializer_class = emg_serializers.AnalysisSerializer - filter_class = emg_filters.AnalysisJobFilter + filterset_class = emg_filters.AnalysisJobFilter filter_backends = ( DjangoFilterBackend, diff --git a/emgapianns/viewsets.py b/emgapianns/viewsets.py index 45d8787dc..4edd840dc 100644 --- a/emgapianns/viewsets.py +++ b/emgapianns/viewsets.py @@ -64,7 +64,7 @@ class AnalysisRelationshipViewSet(ListReadOnlyModelViewSet): """ serializer_class = emg_serializers.AnalysisSerializer - filter_class = emg_filters.AnalysisJobFilter + filterset_class = emg_filters.AnalysisJobFilter filter_backends = ( DjangoFilterBackend, diff --git a/requirements-dev.txt b/requirements-dev.txt index 039d8b6a0..770f37d37 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ -r requirements.txt # dev tools -django-debug-toolbar==3.2.2 -django-extensions==3.1.3 +django-debug-toolbar==3.8.1 +django-extensions==3.2.1 diff --git a/requirements-test.txt b/requirements-test.txt index 928b52733..e57a91e03 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,9 +1,5 @@ -# py3.4 -multidict==4.5.2; python_version < '3.5' -pytest==4.6.11; python_version < '3.5' - -multidict==5.1.0; python_version > '3.4' -pytest==6.2.5; python_version > '3.4' +multidict==5.1.0 +pytest==6.2.5 pytest-django==4.4.0 pytest-xdist==2.3.0 @@ -13,8 +9,6 @@ mongomock==3.23.0 jsonapi-client==0.9.9 pytest-cov==2.12.1 -pandas==0.25.3; python_version < '3.7' -pandas==1.3.2; python_version > '3.6' +pandas==1.3.2 -responses==0.10.15; python_version < '3.5' -responses==0.13.4; python_version > '3.4' +responses==0.23.1 diff --git a/requirements.txt b/requirements.txt index e37af3f21..54d2d96f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,50 +3,44 @@ # run pip install -r requirements # deployment -gunicorn==19.9.0; python_version < '3.5' -whitenoise==3.3.1; python_version < '3.5' -mysqlclient==1.3.13; python_version < '3.5' -sqlparse==0.2.4; python_version < '3.5' -requests==2.21.0; python_version < '3.5' - -gunicorn==20.0.4; python_version > '3.4' -mysqlclient==1.4.6; python_version > '3.4' + +gunicorn==20.1.0 +mysqlclient==2.1.1 mysql-connector-python~=8.0.23 -sqlparse==0.3.1; python_version > '3.4' -whitenoise==5.0.1; python_version > '3.4' -requests==2.26.0; python_version > '3.4' +sqlparse==0.4.3 +whitenoise==6.4.0 +requests==2.28.2 yamjam==0.1.7 # python 3.4 -PyYAML==5.4.1 +PyYAML==6.0 # log handler -concurrent-log-handler~=0.9.19 +concurrent-log-handler~=0.9.22 -Django==3.2.12 -djangorestframework==3.12.4 -django-filter==2.4.0 +Django==3.2.18 +djangorestframework==3.12 +django-filter==23.1 djangorestframework-jwt~=1.11.0 -django-cors-headers==3.8.0 +django-cors-headers==3.14.0 djangorestframework-jsonapi==4.2.1 cx_Oracle~=6.2.1 djangorestframework-csv==2.1.1 # schema -drf-spectacular==0.21.1 +drf-spectacular==0.26.0 # mongo -mongoengine==0.23.1 -pymongo==3.12.0 +mongoengine==0.27.0 +pymongo==4.3.3 django-rest-framework-mongoengine==3.4.1 # assembly viewer -pysam==0.15.3; python_version < '3.5' -pysam==0.18.0; python_version > '3.4' +pysam==0.21.0 # sourmash search -celery[redis]==5.1.0 +celery[redis]==5.2.7 # my-sql utils django-mysql==4.3.0 From 610c2036fd7a88e2c8cee721f5de08320df020bc Mon Sep 17 00:00:00 2001 From: Sandy Rogers Date: Tue, 18 Apr 2023 16:25:25 +0100 Subject: [PATCH 3/3] v2.4.16 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9aa577927..a904a854e 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ _requirements = os.path.join(_base, 'requirements.txt') _requirements_test = os.path.join(_base, 'requirements-test.txt') -version = "2.4.15" +version = "2.4.16" install_requirements = [] with open(_requirements) as f: