Refactor DRF JSON:API (#248)

This PR:
- removes EMG fork of DRFJA, to use latest version of upstream instead.
- adds various workarounds with EMG API, especially in our JSON and CSV renderers, to cater for using custom fields (accessions and lineages etc) instead of pk fields.
- fixes many cases of broken relationships and ?include=. Things should be (more) JSON:API compliant now.
- update Django templates to EMBL VF2 look-and-feel
- tidies up a few things with API Docs generation.

.github/workflows/test.yml

@@ -39,7 +39,6 @@ jobs:
         pip install -U -r requirements.txt
         pip install -U -r requirements-test.txt
         pip install "flake8==3.4" "pycodestyle==2.3.1" pep8-naming
-        pip install "git+git://github.com/EBI-Metagenomics/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi"
         python setup.py sdist
         pip install -U .
         pip freeze

.gitignore

@@ -14,6 +14,7 @@ media/
 venv/
 database/
 results/
+fixtures/*.sig
 
 .vscode/
 

README.md

@@ -103,7 +103,6 @@ Install application::
 
     pip install -U git+git://github.com/EBI-Metagenomics/emg-backlog-schema.git;
     pip install -U git+git://github.com/EBI-Metagenomics/ena-api-handler.git
-    pip install "git+git://github.com/EBI-Metagenomics/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi"
     pip install https://github.com/EBI-Metagenomics/emgapi/archive/$latestTag.tar.gz
 
 Create database::
@@ -136,7 +135,6 @@ How to install the webuploader (one off)?
 
     pip install -U git+git://github.com/EBI-Metagenomics/emg-backlog-schema.git
     pip install -U git+git://github.com/EBI-Metagenomics/ena-api-handler.git
-    pip install "git+git://github.com/EBI-Metagenomics/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi"
 
     pip install -U -r https://raw.githubusercontent.com/EBI-Metagenomics/emgapi/webuploader/requirements-webuploader.txt
     pip install "git+git://github.com/EBI-Metagenomics/emgapi.git@webuploader"

docker/Dockerfile

@@ -12,7 +12,6 @@ RUN mkdir /opt/emgapi && mkdir -p /opt/staticfiles && mkdir -p /opt/results
 
 COPY requirements* /opt/emgapi/
 
-RUN pip3 install "git+git://github.com/EBI-Metagenomics/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi"
 RUN pip3 install -r /opt/emgapi/requirements-dev.txt
 RUN pip3 install -r /opt/emgapi/requirements-admin.txt
 

docker/run.sh

@@ -14,7 +14,6 @@ virtualenv -p /usr/bin/python3 $venvDir --system-site-packages
 # $HOME/venv/bin/pip install -U "django-redis>=4.4"
 
 echo "Installing EMG API..."
-$venvDir/bin/pip install -U  "git+git://github.com/EBI-Metagenomics/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi"
 $venvDir/bin/pip install -U $srcDir
 
 echo "DB startup..."

docker/tests.Dockerfile

@@ -12,7 +12,9 @@ RUN mkdir /opt/emgapi && mkdir -p /opt/staticfiles && mkdir -p /opt/results
 
 COPY requirements* /opt/emgapi/
 
-RUN pip3 install "git+git://github.com/EBI-Metagenomics/django-rest-framework-json-api@develop#egg=djangorestframework-jsonapi"
+RUN pip3 install git+git://github.com/EBI-Metagenomics/emg-backlog-schema.git
+RUN pip3 install git+git://github.com/EBI-Metagenomics/ena-api-handler.git
+RUN pip3 install -r /opt/emgapi/requirements.txt
 RUN pip3 install -r /opt/emgapi/requirements-dev.txt
 RUN pip3 install -r /opt/emgapi/requirements-test.txt
 RUN pip3 install -r /opt/emgapi/requirements-admin.txt

emgapi/filters.py

@@ -13,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import re
 from decimal import Decimal
 
 from django import forms
@@ -32,6 +32,7 @@
 
 from rest_framework import filters as drf_filters
 
+from rest_framework_json_api import filters as drfja_filters
 
 WORD_MATCH_REGEX = r"{0}"
 FLOAT_MATCH_REGEX = r"^[0-9 \.\,]+$"
@@ -101,6 +102,15 @@ class Meta:
         )
 
 
+class JsonApiPlusSearchQueryParameterValidationFilter(drfja_filters.QueryParameterValidationFilter):
+    """
+    Validate query parameters align to JSON:API standard, but also allow the non-standard ?search= param.
+    """
+    query_regex = re.compile(
+        r"^(sort|include|search|ordering)$|^(?P<type>filter|fields|page|page_size)(\[[\w\.\-]+\])?$"
+    )
+
+
 class BiomeFilter(django_filters.FilterSet):
 
     depth_gte = filters.NumberFilter(

emgapi/migrations/0039_alter_biome_unique_together.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.7 on 2022-02-24 16:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('emgapi', '0038_legacyassembly'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='biome',
+            unique_together={('lineage', 'biome_name')},
+        ),
+    ]

emgapi/mixins.py

@@ -5,11 +5,10 @@
 from django.shortcuts import get_object_or_404
 from django.http.response import StreamingHttpResponse
 from rest_framework.exceptions import APIException
-from rest_framework.renderers import BrowsableAPIRenderer
 
 from rest_framework.response import Response
 
-from emgapi.renderers import CSVStreamingRenderer
+from emgapi.renderers import CSVStreamingRenderer, EMGBrowsableAPIRenderer
 from rest_framework_json_api.renderers import JSONRenderer
 
 
@@ -53,7 +52,7 @@ def list(self, request, *args, **kwargs):
                 # (for a complicated endpoint like Studies).
                 # Return custom exception detailing use of paginated API.
                 if request.accepts('text/html'):
-                    request.accepted_renderer = BrowsableAPIRenderer()
+                    request.accepted_renderer = EMGBrowsableAPIRenderer()
                 else:
                     request.accepted_renderer = JSONRenderer()
                 raise ExcessiveCSVException

emgapi/models.py

@@ -307,7 +307,7 @@ class Meta:
         db_table = 'BIOME_HIERARCHY_TREE'
         ordering = ('biome_id',)
         unique_together = (
-            ('biome_id', 'biome_name'),
+            ('lineage', 'biome_name'),
         )
 
     def __str__(self):

emgapi/relations.py

@@ -18,7 +18,6 @@
 
 from rest_framework_json_api.relations import (
     SerializerMethodResourceRelatedField,
-    SerializerMethodHyperlinkedRelatedField,
     ManySerializerMethodHyperlinkedRelatedField,
     SkipDataMixin,
 )
@@ -30,8 +29,6 @@
 LINKS_PARAMS.append('related_link_self_lookup_fields')
 
 
-
-
 class HyperlinkedSerializerMethodResourceRelatedField(SerializerMethodResourceRelatedField):  # noqa
 
     related_link_self_view_name = None

emgapi/renderers.py

@@ -15,15 +15,116 @@
 # limitations under the License.
 import csv
 
+from django.utils import encoding
 from rest_framework import renderers
-from rest_framework_json_api.renderers import JSONRenderer
 from rest_framework_csv.renderers import CSVRenderer, CSVStreamingRenderer as BaseCSVStreamingRenderer
+from rest_framework.relations import HyperlinkedRelatedField
+from rest_framework_json_api import utils
+from rest_framework_json_api.renderers import JSONRenderer, BrowsableAPIRenderer
+
+
+class DictAsDummyInstance(dict):
+    """
+    Add dot-notation getter and setters to a dict, e.g. when for a fake instance of a proxy model generated with
+    queryset.values(...). Additionally look for a likely fieldname to dummy as a `pk`,
+    e.g. if no 'pk' is set, but `lot_lan_pk` exists, set pk=lon_lat_pk.
+
+    E.g. my_data = {'my_fake_pk': 1, 'some_proxy_field': 'some data'}
+    my_fake_instance = DictAsDummyInstance(my_data)
+    assert my_fake_instance.pk == 1
+    """
+    __getattr__ = dict.get
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if getattr(self, 'pk', None) is None:
+            for field in self.keys():
+                if 'pk' in field:
+                    self.pk = self[field]
+                    break
+
+
+def ensure_pk(instance):
+    if hasattr(instance, 'pk'):
+        return
+    if hasattr(instance, 'EMGMeta') and hasattr(instance.EMGMeta, 'pk_field'):
+        instance.pk = getattr(instance, instance.EMGMeta.pk_field)
+    elif hasattr(instance, 'accession'):
+        instance.pk = instance.accession
 
 
 class DefaultJSONRenderer(JSONRenderer):
     media_type = 'application/json'
     format = 'json'
 
+    @classmethod
+    def build_json_resource_obj(
+            cls,
+            fields,
+            resource,
+            resource_instance,
+            resource_name,
+            serializer,
+            force_type_resolution=False,
+            **kwargs
+    ):
+        if type(resource_instance) is dict:
+            resource_instance = DictAsDummyInstance(resource_instance)
+        ensure_pk(resource_instance)
+        resource_data = super().build_json_resource_obj(fields, resource, resource_instance, resource_name, serializer,
+                                                        force_type_resolution=force_type_resolution)
+        relationships = resource_data.get('relationships')
+        if relationships:
+            for field_name in relationships.keys():
+                field_resource = resource_data['relationships'][field_name]
+                if isinstance(fields.fields.get(field_name), HyperlinkedRelatedField):
+                    id_field = getattr(fields.fields.get(field_name), 'lookup_field')
+                    related_instance = getattr(resource_instance, field_name)
+                    id_value = getattr(related_instance, id_field, None)
+                    if None not in [id_value, field_resource['data']]:
+                        field_resource['data']['id'] = id_value
+                if 'data' in field_resource and field_resource['data'] is None and 'links' in field_resource:
+                    resource_data['relationships'][field_name].pop('data')
+
+        current_serializer = fields.serializer
+        context = current_serializer.context
+        view = context.get("view", None)
+        if hasattr(view, "relationship_lookup_field"):
+            if view.relationship_lookup_field in resource:
+                resource_data['id'] = resource.get(view.relationship_lookup_field, resource_data['id'])
+        elif hasattr(view, 'lookup_field'):
+            if view.lookup_field in resource:
+                resource_data['id'] = encoding.force_str(resource.get(view.lookup_field, resource_data['id']))
+        if "url" in fields and resource_data.get('id') is not None:
+            custom_id = getattr(resource_instance, fields["url"].lookup_field)
+            resource_data['id'] = encoding.force_str(custom_id)
+
+        return resource_data
+
+
+class EMGBrowsableAPIRenderer(BrowsableAPIRenderer):
+    @classmethod
+    def _get_included_serializers(cls, serializer, prefix="", already_seen=None):
+        """Prevents browsable API showing options to include deeply nested serializers
+        (e.g. ?include=biome.studies.biomes)"""
+        if not already_seen:
+            already_seen = set()
+
+        if serializer in already_seen:
+            return []
+
+        included_serializers = []
+        already_seen.add(serializer)
+
+        for include, included_serializer in utils.get_included_serializers(
+                serializer
+        ).items():
+            included_serializers.append(f"{prefix}{include}")
+
+        return included_serializers
+
 
 class JSONLDRenderer(renderers.JSONRenderer):
     media_type = 'application/ld+json'
@@ -40,7 +141,8 @@ class CSVStreamingRenderer(BaseCSVStreamingRenderer):
 
     def render(self, data, *args, **kwargs):
         if not isinstance(data, list):
-            data = data.get(self.results_field, [])
+            if self.results_field in data:
+                data = data.get(self.results_field, [])
         return super(CSVStreamingRenderer, self).render(data, *args, **kwargs)
 
     def flatten_item(self, item):