Merge branch 'develop' into feature/ebi-search-dump-migration

.github/workflows/test.yml

@@ -4,18 +4,17 @@ on: [push, pull_request]
 env:
   EMG_CONFIG: ${{ github.workspace }}/ci/configuration.yaml
 
-
 jobs:
   build:
 
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: [3.8, 3.9]
-        # TODO: Temporarily removed 3.7.1, waiting for https://github.com/celery/celery/issues/7783
 
     steps:
     - uses: actions/checkout@v3
+
     - name: 🐍 - Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
@@ -25,6 +24,7 @@ jobs:
       uses: supercharge/[email protected]
       with:
         mongodb-version: 4.0.6
+
     - name: ⚙️ - Checking environment
       run: |
         python -V
@@ -35,19 +35,21 @@ jobs:
 
     - name: 🔧 - Install Dependencies
       run: |
-        pip install -U -r requirements.txt
-        pip install -U -r requirements-test.txt
+        pip install install .[tests]
         pip freeze
+
     - name: 🧪 - Testing
       run: |
         cat $EMG_CONFIG
         pytest
+
     # TODO: disabled until black formatting completed and all flake issues fixed
     # - name: Flake
     #   continue-on-error: true
     #   run: |
     #     flake8 --version
     #     flake8 -v .
+
     - name: 📮 - Slack Notification
       uses: rtCamp/action-slack-notify@v2
       continue-on-error: true

.gitignore

@@ -23,3 +23,8 @@ fixtures/*.sig
 
 .coverage
 /build/
+
+loglockdir
+logs
+
+secret.key

MANIFEST.in

@@ -1,11 +1,10 @@
-include README.rst
+include README.md
 include LICENSE
-include requirements.txt
+include pyproject.toml
 recursive-include tests *
 recursive-include emgcli *
 recursive-include emgapi *
 recursive-include emgapianns *
 recursive-include emgui *
 recursive-exclude * __pycache__
 recursive-exclude * *.pyc
-recursive-include genome_fixtures *

README.md

@@ -7,34 +7,18 @@ Metagenomics service is a large-scale platform for analyzing and archiving metag
 
 # Setup
 ## Local env.
-
-For development there are 3 options: 
-
-* Use the parent repo ["MGnify Web"](https://github.com/EBI-Metagenomics/mgnify-web) which includes this API repository, as well as two frontend web repositories needed to develop/run the [MGnify website](https://www.ebi.ac.uk/metagenomics).
-* Or, install the stack locally
-* Or, use Docker for the database and mongo
-
-In any case the webapp will be executed from a local virtual environment.
+For development, use the parent repo ["MGnify Web"](https://github.com/EBI-Metagenomics/mgnify-web) which includes this API repository, as well as the frontend web repository needed to develop/run the [MGnify website](https://www.ebi.ac.uk/metagenomics).
 
 ### MGnify Web parent repo
 The parent repo uses docker-compose to configure a development environment and test data for the entire stack of the MGnify website.
 It is the recommended development setup.
 See: [MGnify Web](https://github.com/EBI-Metagenomics/mgnify-web) on GitHub for instructions.
 
+**The Docker setup is just for local dev. at the moment.**
 
-### Stack locally
-
-The app uses `MySQL` version `5.6` and `Mongo` version `3.4`.
-
-TODO: write the instructions for MacOS and Linux.
-
-### Docker
-
-There are 2 docker containers defined, one for `MySQL` and another one `MongoDB`.
-
-The app will be executed from a python virtual environment.
+This API relies on a relational (SQLite or MySQL) and a document (Mongo) database.
 
-**The Docker setup is just for local dev. at the moment.**
+This docker compose setup in the parent repo handles these.
 
 ### Helper scripts
 
@@ -44,11 +28,10 @@ There are some helper scripts that are meant to make running the project locally
 - `gunicorn.sh` run the app using gunicorn with the `--reload` flag.
 
 ## Setup
-Create configuration file in `~/path/to/config.yaml <docker/config.yaml>`_.
+Create/edit configuration file in `./config/<some config>.yaml` and set the env var `EMG_CONFIG` to point to that file.
 
 ### DB config file
-An environment variable named *EMG_CONFIG* needs to be defined for the database config.
-This should contain the path to yaml config file, which must contain the following fields:
+The config file must specify the databases:
 ```yaml
 emg:
   databases:
@@ -60,39 +43,28 @@ emg:
       NAME: 'schema_name'
       USER: 'user'
       PASSWORD: 'password'
-    dev:
-        ....
-    prod:
-        ....
-    era:
-      ENGINE: 'django.db.backends.oracle'
-      NAME: ?
-      USER: ?
-      PASSWORD: ?
-      HOST: ?
-      PORT: ?
+
+    ...
 ```
+(see the example config yamls for full details).
 
-Install `virtualenv <https://virtualenv.pypa.io/en/latest/installation//>`_
+If **not** using the mgnify-web docker compose setup for some reason: 
 
-Create a virtual environment::
-
-    `virtualenv -p python3 venv`
-
-Activate and install the dependencies `source venv/bin/activate && pip install -r requirements-dev.txt`.
+Install [virtualenv](https://virtualenv.pypa.io/en/latest/installation/).
 
-Start containers using::
+Create a virtual environment or a conda env, e.g.: `virtualenv -p python3 venv`
 
-    docker-compose -f docker/docker-compose.yml up --build -d
+Activate and install the dependencies `source venv/bin/activate && pip install .[dev,admin]`.
 
-Run the migrations::
+Run the migrations: `./manage.sh migrate`
 
-    ./manage.sh migrate
+Run the server: `./manage.sh runserver 8000`
 
-Run the server::
+**If using the mgnify-web setup, follow the instructions in the parents repo README, and use the Taskfile in it.**
 
-   ./manage.sh runserver 8000
+---
 
+**TODO: update the following**
 ## Production env.
 ### Install
 

config/local-lite.yml

@@ -16,17 +16,21 @@ emg:
     - 'django.contrib.auth.backends.ModelBackend'
     - 'emgapi.backends.EMGBackend'
 
-  cors_origin_allow_all: true
   debug: true
   results_dir: '/opt/emgapi/results'
   results_production_dir: '/opt/emgapi/results'
   static_root: '/opt/emgapi/staticfiles'
   emg_backend_auth: 'https://wwwdev.ebi.ac.uk/ena/dev/submit/webin/auth/login'
   secure_cookies: false
 
+  log_dir: '/opt/emgapi/logs'
+  log_lock_dir: '/opt/emgapi/loglockdir'
+
+  secret_key: '/opt/emgapi'
+
   mongodb:
     db: emg
-    host: mongodb-lite
+    host: mongodb
   documentation:
     title: 'EBI Metagenomics API'
     description: 'Is a free resource to visualise and discover metagenomic datasets. For more details go to http://www.ebi.ac.uk/metagenomics/'

config/local-tests.yml

@@ -8,7 +8,7 @@ emg:
   emg_backend_auth: "https://backend"
   mongodb:
       db: emg_tests
-      host: mongodb-lite
+      host: mongodb
   sourmash:
     signatures_path: 'fixtures/'
     results_path: 'fixtures/'

docker/Dockerfile

@@ -10,11 +10,11 @@ RUN yum -y install python3 python3-devel python3-setuptools mysql-devel && \
 
 RUN mkdir /opt/emgapi && mkdir -p /opt/staticfiles && mkdir -p /opt/results 
 
-COPY requirements* /opt/emgapi/
+COPY pyproject.toml /opt/emgapi/
+COPY emgcli/__init__.py /opt/emgapi/emgcli/
+# needed for VERSION
 
-RUN pip3 install -r /opt/emgapi/requirements.txt
-RUN pip3 install -r /opt/emgapi/requirements-dev.txt
-RUN pip3 install -r /opt/emgapi/requirements-admin.txt
+RUN pip3 install /opt/emgapi[dev,admin,tests]
 
 ENV PYTHONPATH="${PYTHONPATH}:/opt/emgapi/emgcli"
 

docker/lite.Dockerfile

@@ -2,11 +2,11 @@ FROM python:3.8-bullseye
 
 RUN mkdir /opt/emgapi && mkdir -p /opt/staticfiles && mkdir -p /opt/results
 
-COPY requirements* /opt/emgapi/
+COPY pyproject.toml /opt/emgapi/
+COPY emgcli/__init__.py /opt/emgapi/emgcli/
+# needed for VERSION
 
-RUN pip3 install -r /opt/emgapi/requirements.txt
-RUN pip3 install -r /opt/emgapi/requirements-dev.txt
-RUN pip3 install -r /opt/emgapi/requirements-admin.txt
+RUN pip3 install /opt/emgapi[dev,admin,tests]
 
 ENV PYTHONPATH="${PYTHONPATH}:/opt/emgapi/emgcli"
 

emgapi/admin.py

@@ -87,7 +87,7 @@ class StudyAdmin(admin.ModelAdmin, NoRemoveMixin):
         'project_id',
         'study_name',
     )
-    list_filter = ('is_private', )
+    list_filter = ('is_private',)
     raw_id_fields = ('biome',)
 
     def save_model(self, request, obj, form, change):
@@ -127,7 +127,6 @@ class Meta:
 
 @admin.register(emg_models.SuperStudy)
 class SuperStudyAdmin(admin.ModelAdmin):
-
     inlines = [SuperStudyStudiesInline, SuperStudyBiomesInline, SuperStudyGenomeCataloguesInline]
     form = SuperStudyAdminForm
 
@@ -182,6 +181,18 @@ def get_search_results(self, request, queryset, search_term):
             return super().get_search_results(request, queryset, search_term)
 
 
+class RunExtraAnnotationDownloads(admin.TabularInline):
+    model = emg_models.RunExtraAnnotation
+    raw_id_fields = [
+        'run',
+        'parent_id',
+        'group_type',
+        'subdir',
+        'description',
+        'file_format'
+    ]
+    extra = 0
+
 @admin.register(emg_models.Run)
 class RunAdmin(admin.ModelAdmin, AccessionSearch):
     change_list_template = "admin/change_list_filter_sidebar.html"
@@ -209,6 +220,9 @@ class RunAdmin(admin.ModelAdmin, AccessionSearch):
         'sample',
         'study',
     ]
+    inlines = [
+        RunExtraAnnotationDownloads,
+    ]
 
     filter_property = 'study'
     prefix = 'MGYS'
@@ -361,13 +375,13 @@ class AnalysisJobAdmin(admin.ModelAdmin, AccessionSearch, NoRemoveMixin):
     def get_queryset(self, request):
         return emg_models.AnalysisJob.objects_admin.all() \
             .select_related(
-                'pipeline',
-                'analysis_status',
-                'experiment_type',
-                'run',
-                'study',
-                'assembly',
-                'sample')
+            'pipeline',
+            'analysis_status',
+            'experiment_type',
+            'run',
+            'study',
+            'assembly',
+            'sample')
 
 
 @admin.register(emg_models.StudyErrorType)

emgapi/fields.py

@@ -49,6 +49,11 @@ def get_url(self, obj, view_name, request, format):
             kwargs = {
                 'accession': obj.assembly.accession
             }
+
+        elif hasattr(obj, 'run'):
+            kwargs = {
+                'accession': obj.run.accession
+            }
         kwargs['alias'] = obj.alias
 
         return reverse(

emgapi/migrations/0010_runextraannotation.py

@@ -0,0 +1,35 @@
+# Generated by Django 3.2.18 on 2023-07-17 13:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('emgapi', '0009_genome_annotations_v2_downloads'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RunExtraAnnotation',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('realname', models.CharField(db_column='REAL_NAME', max_length=255)),
+                ('alias', models.CharField(db_column='ALIAS', max_length=255)),
+                ('file_checksum', models.CharField(blank=True, db_column='CHECKSUM', max_length=255)),
+                ('checksum_algorithm', models.ForeignKey(blank=True, db_column='CHECKSUM_ALGORITHM', null=True, on_delete=django.db.models.deletion.CASCADE, to='emgapi.checksumalgorithm')),
+                ('description', models.ForeignKey(blank=True, db_column='DESCRIPTION_ID', null=True, on_delete=django.db.models.deletion.CASCADE, to='emgapi.downloaddescriptionlabel')),
+                ('file_format', models.ForeignKey(blank=True, db_column='FORMAT_ID', null=True, on_delete=django.db.models.deletion.CASCADE, to='emgapi.fileformat')),
+                ('group_type', models.ForeignKey(blank=True, db_column='GROUP_ID', null=True, on_delete=django.db.models.deletion.CASCADE, to='emgapi.downloadgrouptype')),
+                ('parent_id', models.ForeignKey(blank=True, db_column='PARENT_DOWNLOAD_ID', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='parent', to='emgapi.runextraannotation')),
+                ('run', models.ForeignKey(db_column='RUN_ID', on_delete=django.db.models.deletion.CASCADE, related_name='extra_annotations', to='emgapi.run')),
+                ('subdir', models.ForeignKey(blank=True, db_column='SUBDIR_ID', null=True, on_delete=django.db.models.deletion.CASCADE, to='emgapi.downloadsubdir')),
+            ],
+            options={
+                'db_table': 'RUN_DOWNLOAD',
+                'ordering': ('group_type', 'alias'),
+                'unique_together': {('realname', 'alias', 'run')},
+            },
+        ),
+    ]

emgapi/migrations/0011_analysisjob_analysis_summary_json.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.18 on 2023-09-13 10:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('emgapi', '0010_runextraannotation'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='analysisjob',
+            name='analysis_summary_json',
+            field=models.JSONField(blank=True, db_column='ANALYSIS_SUMMARY_JSON', null=True),
+        ),
+    ]