Merge pull request #276 from RockefellerArchiveCenter/deliver

Delivers transfers to microservice environment

aurora/aurora/config.py.example

@@ -62,15 +62,4 @@ EMAIL_USE_SSL = False
 EMAIL_OVERRIDE = True
 EMAIL_OVERRIDE_USERS = ['[email protected]']
 
-# Microservice integrations
-DELIVERY = {
-    'user': 'root',
-    'host': 'http://ursa_major',
-    'dir': '/storage/'
-}
-CALLBACKS = [
-  {
-    'method': 'POST',
-    'uri': 'http://ursa_major/'
-  }
-]
+DELIVERY_URL = 'http://zodiac-web:8001/api/store-accessions/'

aurora/aurora/settings.py

@@ -84,6 +84,10 @@
 REST_FRAMEWORK = {
     'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
     'PAGE_SIZE': 50,
+    'DEFAULT_AUTHENTICATION_CLASSES': (
+        'rest_framework_jwt.authentication.JSONWebTokenAuthentication',
+        'rest_framework.authentication.SessionAuthentication',
+    ),
 }
 
 # Database
@@ -178,5 +182,4 @@
 
 
 # Post-accession callbacks
-DELIVERY = CF.DELIVERY
-CALLBACKS = CF.CALLBACKS
+DELIVERY_URL = CF.DELIVERY_URL

aurora/bag_transfer/api/serializers.py

@@ -21,64 +21,80 @@ class RightsStatementRightsGrantedSerializer(serializers.ModelSerializer):
     def get_end_date(self, obj):
         end_date = obj.end_date
         if not end_date:
-            if obj.end_date_open:
-                end_date = "OPEN"
-            else:
-                end_date = ''
+            end_date = "OPEN" if obj.end_date_open else None
         return end_date
 
     class Meta:
         model = RightsStatementRightsGranted
         fields = ('act', 'start_date', 'end_date', 'note', 'restriction')
 
 
-class RightsStatementSerializer(serializers.BaseSerializer):
-    def to_representation(self, obj):
-        basis_key = obj.rights_basis.lower()
-        if basis_key == 'other':
-            basis_key = 'other_rights'
-        rights_granted = RightsStatementRightsGrantedSerializer(
-            RightsStatementRightsGranted.objects.filter(rights_statement=obj), many=True)
+class RightsStatementSerializer(serializers.ModelSerializer):
+    rights_granted = RightsStatementRightsGrantedSerializer(source='rightsstatementrightsgranted_set', many=True)
+    rights_basis = serializers.StringRelatedField()
+    start_date = serializers.SerializerMethodField()
+    end_date = serializers.SerializerMethodField()
+    note = serializers.SerializerMethodField()
+    jurisdiction = serializers.SerializerMethodField()
+    determination_date = serializers.SerializerMethodField()
+    status = serializers.SerializerMethodField()
+    terms = serializers.SerializerMethodField()
+    citation = serializers.SerializerMethodField()
+    other_rights_basis = serializers.SerializerMethodField()
 
-        if obj.rights_basis == 'Copyright':
-            basis_obj = RightsStatementCopyright.objects.get(rights_statement=obj)
-            basis_dict = {
-                'jurisdiction': getattr(basis_obj, 'copyright_jurisdiction', ''),
-                'determination_date': getattr(basis_obj, 'copyright_status_determination_date', ''),
-                'status': getattr(basis_obj, 'copyright_status', ''),
-            }
-        elif obj.rights_basis == 'License':
-            basis_obj = RightsStatementLicense.objects.get(rights_statement=obj)
-            basis_dict = {
-                'terms': getattr(basis_obj, 'license_terms', ''),
-            }
-        if obj.rights_basis == 'Statute':
-            basis_obj = RightsStatementStatute.objects.get(rights_statement=obj)
-            basis_dict = {
-                'jurisdiction': getattr(basis_obj, 'statute_jurisdiction', ''),
-                'determination_date': getattr(basis_obj, 'statute_status_determination_date', ''),
-                'citation': getattr(basis_obj, 'statute_citation', ''),
-            }
-        if obj.rights_basis == 'Other':
-            basis_obj = RightsStatementOther.objects.get(rights_statement=obj)
-            basis_dict = {
-                'other_rights_basis': getattr(basis_obj, 'other_rights_basis', ''),
-            }
-        end_date = getattr(basis_obj, '{}_applicable_end_date'.format(basis_key))
+    class Meta:
+        model = RightsStatement
+        fields = ('rights_basis', 'determination_date', 'jurisdiction',
+                  'start_date', 'end_date', 'note', 'status', 'terms',
+                  'citation', 'other_rights_basis', 'rights_granted')
+
+    def get_basis_obj(self, obj):
+        if obj.rights_basis == "Copyright":
+            return RightsStatementCopyright.objects.get(rights_statement=obj)
+        elif obj.rights_basis == "License":
+            return RightsStatementLicense.objects.get(rights_statement=obj)
+        elif obj.rights_basis == "Statute":
+            return RightsStatementStatute.objects.get(rights_statement=obj)
+        elif obj.rights_basis == "Other":
+            return RightsStatementOther.objects.get(rights_statement=obj)
+
+    def get_basis_key(self, obj):
+        return 'other_rights' if (obj.rights_basis.lower() == 'other') else obj.rights_basis.lower()
+
+    def get_start_date(self, obj):
+        return getattr(self.get_basis_obj(obj), '{}_applicable_start_date'.format(self.get_basis_key(obj)))
+
+    def get_end_date(self, obj):
+        end_date = getattr(self.get_basis_obj(obj), '{}_applicable_end_date'.format(self.get_basis_key(obj)))
         if not end_date:
-            if '{}_end_date_open'.format(basis_key):
-                end_date = "OPEN"
-            else:
-                end_date = ''
-        common_dict = {
-            'rights_basis': obj.rights_basis,
-            'start_date': getattr(basis_obj, '{}_applicable_start_date'.format(basis_key), ''),
-            'end_date': end_date,
-            'note': getattr(basis_obj, '{}_note'.format(basis_key), ''),
-            'rights_granted': rights_granted.data,
-        }
-        common_dict.update(basis_dict)
-        return common_dict
+            end_date = "OPEN" if '{}_end_date_open'.format(self.get_basis_key(obj)) else None
+        return end_date
+
+    def get_note(self, obj):
+        return getattr(self.get_basis_obj(obj), '{}_note'.format(self.get_basis_key(obj)))
+
+    def get_jurisdiction(self, obj):
+        return getattr(self.get_basis_obj(obj), '{}_jurisdiction'.format(self.get_basis_key(obj))) if (obj.rights_basis in ['Copyright', 'Statute']) else None
+
+    def get_determination_date(self, obj):
+        if obj.rights_basis == 'Copyright':
+            return getattr(self.get_basis_obj(obj), 'copyright_status_determination_date')
+        elif obj.rights_basis == 'Statute':
+            return getattr(self.get_basis_obj(obj), 'statute_determination_date')
+        else:
+            return None
+
+    def get_status(self, obj):
+        return getattr(self.get_basis_obj(obj), 'copyright_status') if (obj.rights_basis == 'Copyright') else None
+
+    def get_terms(self, obj):
+        return getattr(self.get_basis_obj(obj), 'license_terms') if (obj.rights_basis == 'License') else None
+
+    def get_citation(self, obj):
+        return getattr(self.get_basis_obj(obj), 'statute_citation') if (obj.rights_basis == 'Statute') else None
+
+    def get_other_rights_basis(self, obj):
+        return getattr(self.get_basis_obj(obj), 'other_rights_basis') if (obj.rights_basis == 'Other') else None
 
 
 class BAGLogResultSerializer(serializers.Serializer):
@@ -122,9 +138,9 @@ class Meta:
 
 
 class ArchivesSerializer(serializers.HyperlinkedModelSerializer):
-    metadata = BagInfoMetadataSerializer()
-    events = BAGLogSerializer(many=True)
-    rights_statements = RightsStatementSerializer(many=True)
+    metadata = BagInfoMetadataSerializer(read_only=True)
+    events = BAGLogSerializer(many=True, read_only=True)
+    rights_statements = RightsStatementSerializer(many=True, read_only=True)
     file_size = serializers.StringRelatedField(source='machine_file_size')
     file_type = serializers.StringRelatedField(source='machine_file_type')
     identifier = serializers.StringRelatedField(source='machine_file_identifier')
@@ -134,7 +150,8 @@ class Meta:
         fields = ('url', 'identifier', 'organization', 'bag_it_name',
                   'process_status', 'file_size', 'file_type', 'appraisal_note',
                   'additional_error_info', 'metadata', 'rights_statements',
-                  'events', 'created_time', 'modified_time')
+                  'events', 'archivesspace_identifier', 'archivesspace_parent_identifier',
+                  'created_time', 'modified_time')
 
 
 class ArchivesListSerializer(serializers.HyperlinkedModelSerializer):

aurora/bag_transfer/api/views.py

@@ -5,6 +5,7 @@
 from rest_framework.decorators import detail_route, list_route
 from rest_framework.response import Response
 
+from bag_transfer.lib.cleanup import CleanupRoutine
 from bag_transfer.models import Organization, Archives, BAGLog, BagInfoMetadata, BagItProfile, ManifestsRequired, User
 from bag_transfer.mixins.authmixins import ArchivistMixin, OrgReadViewMixin
 from bag_transfer.accession.models import Accession
@@ -79,6 +80,14 @@ def get_serializer_class(self):
             return ArchivesSerializer
         return ArchivesSerializer
 
+    def update(self, request, pk=None, *args, **kwargs):
+        try:
+            identifier = request.data.get('identifier')
+            CleanupRoutine(identifier).run()
+            return super(ArchivesViewSet, self).update(request, *args, **kwargs)
+        except Exception as e:
+            return Response({"detail": str(e)}, status=500)
+
 
 class BAGLogViewSet(OrgReadViewMixin, viewsets.ReadOnlyModelViewSet):
     """Endpoint for events"""

aurora/bag_transfer/appraise/views.py

@@ -81,8 +81,8 @@ def get_filter_method(self): return self.FILTER_ICONTAINS
     def appraise_buttons(self):
         return '<a type=button class="btn btn-xs btn-primary appraisal-accept" href="#">Accept</a>\
                 <a type="button" class="btn btn-xs btn-danger appraisal-reject" href="#">Reject</a>\
-                <a type="button" class="appraisal-note btn btn-xs btn-info edit-note" data-toggle="modal" data-target="#modal-appraisal-note">Note</a>\
-                <a type="button" class="transfer-detail btn btn-xs btn-warning" data-toggle="modal" data-target="#modal-detail" aria-expanded="false">Details</a>'
+                <a type="button" class="appraisal-note btn btn-xs btn-info edit-note" data-toggle="modal" data-target="#modal-appraisal-note" href="#">Note</a>\
+                <a type="button" class="transfer-detail btn btn-xs btn-warning" data-toggle="modal" data-target="#modal-detail" aria-expanded="false" href="#">Details</a>'
 
     def get_initial_queryset(self):
         return Archives.objects.filter(process_status=40)

aurora/bag_transfer/lib/cleanup.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+from os.path import join, isfile
+from os import remove
+
+from aurora import settings
+
+
+class CleanupError(Exception): pass
+
+
+class CleanupRoutine:
+    def __init__(self, identifier):
+        self.identifier = identifier
+        self.dir = settings.DELIVERY_QUEUE_DIR
+
+    def run(self):
+        try:
+            self.filepath = "{}.tar.gz".format(join(self.dir, self.identifier))
+            if isfile(self.filepath):
+                remove(self.filepath)
+                return "Transfer {} removed.".format(self.identifier)
+            return "Transfer {} was not found.".format(self.identifier)
+        except Exception as e:
+            raise CleanupError(e)

aurora/bag_transfer/lib/cron.py

@@ -1,11 +1,13 @@
 import datetime
-from os import listdir
+import json
+from os import listdir, mkdir
 from os.path import join
 import subprocess
 
 from django_cron import CronJobBase, Schedule
 
 from aurora import settings
+from bag_transfer.api.serializers import ArchivesSerializer
 from bag_transfer.lib import files_helper as FH
 from bag_transfer.lib.transfer_routine import *
 from bag_transfer.lib.bag_checker import bagChecker
@@ -114,24 +116,29 @@ class DeliverTransfers(CronJobBase):
 
     def do(self):
         Pter.cron_open(self.code)
-        for transfer in listdir(settings.DELIVERY_QUEUE_DIR):
-            rsynccmd = "rsync -avh --remove-source-files {} {}@{}:{}".format(join(settings.DELIVERY_QUEUE_DIR, transfer),
-                                                                             settings.DELIVERY['user'],
-                                                                             settings.DELIVERY['host'],
-                                                                             settings.DELIVERY['dir'])
-            print(rsynccmd)
-            rsyncproc = subprocess.Popen(rsynccmd,
-                                         shell=True,
-                                         stdin=subprocess.PIPE,
-                                         stdout=subprocess.PIPE,)
-            while True:
-                next_line = rsyncproc.stdout.readline().decode("utf-8")
-                if not next_line:
-                    break
-                print(next_line)
-
-            ecode = rsyncproc.wait()
-            if ecode != 0:
-                continue
+        for archive in Archives.objects.filter(process_status=75):
+            tar_filename = "{}.tar.gz".format(archive.machine_file_identifier)
+            FH.make_tarfile(
+                join(settings.STORAGE_ROOT_DIR, tar_filename),
+                join(settings.STORAGE_ROOT_DIR, archive.machine_file_identifier))
+
+            mkdir(join(settings.DELIVERY_QUEUE_DIR, archive.machine_file_identifier))
+
+            FH.move_file_or_dir(
+                join(settings.STORAGE_ROOT_DIR, tar_filename),
+                join(settings.DELIVERY_QUEUE_DIR, archive.machine_file_identifier, tar_filename))
+
+            archive_json = ArchivesSerializer(archive, context={'request': None}).data
+            with open(join(settings.DELIVERY_QUEUE_DIR, archive.machine_file_identifier, "{}.json".format(archive.machine_file_identifier)), 'wb') as f:
+                json.dump(archive_json, f, indent=4, sort_keys=True, default=str)
+
+            FH.make_tarfile(
+                join(settings.DELIVERY_QUEUE_DIR, "{}.tar.gz".format(archive.machine_file_identifier)),
+                join(settings.DELIVERY_QUEUE_DIR, archive.machine_file_identifier))
+
+            FH.remove_file_or_dir(join(settings.DELIVERY_QUEUE_DIR, archive.machine_file_identifier))
+
+            archive.process_status = 80
+            archive.save()
 
         Pter.cron_close(self.code)

aurora/bag_transfer/lib/files_helper.py

@@ -260,3 +260,7 @@ def chown_path_to_root(file_path):
     if is_dir_or_file(file_path):
         root_uid = pwd.getpwnam('root').pw_uid
         os.chown(file_path, root_uid, root_uid)
+
+def make_tarfile(output_filename, source_dir):
+    with tarfile.open(output_filename, 'w:gz') as tar:
+        tar.add(source_dir, arcname=os.path.basename(source_dir))

aurora/bag_transfer/migrations/0016_auto_20181120_0902.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.15 on 2018-11-20 14:02
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bag_transfer', '0015_auto_20181025_1342'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='archives',
+            name='archivesspace_identifier',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='archives',
+            name='archivesspace_parent_identifier',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

aurora/bag_transfer/models.py

@@ -341,6 +341,8 @@ class Archives(models.Model):
     manifest = models.TextField(blank=True, null=True)
     additional_error_info = models.CharField(max_length=255, null=True, blank=True)
     process_status = models.PositiveSmallIntegerField(choices=processing_statuses, default=20)
+    archivesspace_identifier = models.CharField(max_length=255, null=True, blank=True)
+    archivesspace_parent_identifier = models.CharField(max_length=255, null=True, blank=True)
     created_time = models.DateTimeField(auto_now_add=True)  # process time
     modified_time = models.DateTimeField(auto_now=True)