Blacklist changes

Seeder/api/__init__.py

@@ -7,3 +7,4 @@
 api_router.register('category', viewsets.CategoryViewSet)
 api_router.register('source', viewsets.SourceViewSet)
 api_router.register('seed', viewsets.SeedViewSet)
+api_router.register('blacklist', viewsets.BlacklistViewSet)

Seeder/api/serializers.py

@@ -1,5 +1,6 @@
 import source
 import publishers
+import blacklists
 
 from rest_framework.serializers import ModelSerializer
 from rest_framework.fields import CharField
@@ -50,3 +51,9 @@ class SourceSerializer(ModelSerializer):
     class Meta:
         model = source.models.Source
         exclude = ['created_by', 'owner']
+
+
+class BlacklistSerializer(ModelSerializer):
+    class Meta:
+        model = blacklists.models.Blacklist
+        fields = '__all__'

Seeder/api/viewsets.py

@@ -1,7 +1,10 @@
 import source
+import blacklists
 
 from rest_framework import viewsets
 from rest_framework import mixins as rf_mixins
+from rest_framework.decorators import action
+from rest_framework.response import Response
 from . import serializers
 
 
@@ -27,3 +30,15 @@ class SeedViewSet(viewsets.GenericViewSet, rf_mixins.RetrieveModelMixin,
     """
     serializer_class = serializers.SeedSerializer
     queryset = source.models.Seed.objects.all()
+
+
+class BlacklistViewSet(viewsets.GenericViewSet, rf_mixins.RetrieveModelMixin,
+                       rf_mixins.ListModelMixin):
+    serializer_class = serializers.BlacklistSerializer
+    queryset = blacklists.models.Blacklist.objects.all()
+
+    @action(methods=['get'], detail=False)
+    def lastchanged(self, request):
+        return Response({
+            'lastChanged': blacklists.models.Blacklist.last_change(),
+        })

Seeder/blacklists/migrations/0002_auto_20190703_1018.py

@@ -0,0 +1,29 @@
+# Generated by Django 2.2 on 2019-07-03 10:18
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('blacklists', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='blacklist',
+            name='active',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name='blacklist',
+            name='created',
+            field=models.DateTimeField(default=django.utils.timezone.now, editable=False),
+        ),
+        migrations.AddField(
+            model_name='blacklist',
+            name='last_changed',
+            field=models.DateTimeField(auto_now=True),
+        ),
+    ]

Seeder/blacklists/models.py

@@ -5,9 +5,11 @@
 from django.utils.translation import ugettext_lazy as _
 from reversion import revisions
 
+from core.models import BaseModel
+
 
 @revisions.register
-class Blacklist(models.Model):
+class Blacklist(BaseModel):
     """
     General model for control lists
 
@@ -46,3 +48,15 @@ def collect_urls_by_type(cls, blacklist_type):
         # blacklist urls is now list of contents
         urls_parsed = map(str.splitlines, blacklist_urls)
         return reduce(operator.add, urls_parsed, [])
+
+    @classmethod
+    def last_change(cls):
+        agg = Blacklist.objects.all().aggregate(models.Max('last_changed'))
+        return agg.get('last_changed__max')
+
+    @classmethod
+    def dump(cls):
+        blacklist_urls = cls.objects.all().values_list('url_list', flat=True)
+        urls_parsed = map(str.splitlines, blacklist_urls)
+        # Remove duplicates
+        return list(set(reduce(operator.add, urls_parsed, [])))

Seeder/blacklists/templates/blacklists.html

@@ -2,20 +2,31 @@
 {% load i18n %}
 
 {% block extra_header %}
-<a href="{% url 'blacklists:add' %}" class="pull-right btn btn-primary btn-success">{% trans 'Add' %}</a>
+<a href="{% url 'blacklists:add' %}" class="pull-right btn btn-success">{% trans 'Add' %}</a>
+<a href="{% url 'blacklists:dump' %}" class="pull-right btn btn-primary" target='_blank'>{% trans 'Dump' %}</a>
+<span class="label label-default pull-right" style="font-size: 0.4em;margin-top: 0.6rem;">
+    Last update: {{last_change}} ago
+</span>
 {% endblock %}
 
 {% block content %}
+    <script>
+        // "Select all" link selects all links in blacklist
+        $(document).on('click', '.select_all', (e) => {
+            e.preventDefault();
+            selectText($(e.currentTarget).closest('.panel').find('.panel-body pre')[0]);
+        })
+    </script>
     {% for blacklist in blacklists %}
         <div class="panel panel-default">
           <div class="panel-heading">
               {{ blacklist.title }} |
               <a href="{% url 'blacklists:edit' blacklist.pk %}">{% trans 'Edit' %}</a> |
-              <a href="{% url 'blacklists:history' blacklist.pk %}">{% trans 'History' %}</a>
+              <a href="{% url 'blacklists:history' blacklist.pk %}">{% trans 'History' %}</a> |
+              <a href="#" class="select_all">{% trans 'Select all' %}</a>
           </div>
           <div class="panel-body">
-              <pre>{{ blacklist.url_list }}</pre>
-
+              <pre style="max-height: 500px; overflow-y: scroll;">{{ blacklist.url_list }}</pre>
           </div>
         </div>
     {% endfor %}

Seeder/blacklists/urls.py

@@ -6,4 +6,5 @@
     path('add', AddView.as_view(), name='add'),
     path('<int:pk>/edit', EditView.as_view(), name='edit'),
     path('history/<int:pk>', History.as_view(), name='history'),
+    path('dump', BlacklistDump.as_view(), name='dump'),
 ]

Seeder/blacklists/views.py

@@ -1,9 +1,11 @@
 from django.urls import reverse
 from django.db import transaction
 from django.http.response import HttpResponseRedirect
+from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from django.views.generic.base import TemplateView
 from django.views.generic.edit import FormView
+from django.core.mail import mail_admins
 
 from reversion import revisions
 
@@ -22,6 +24,10 @@ class ListView(BlacklistView, TemplateView):
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
+        last_change = timezone.now().replace(microsecond=0) -\
+            models.Blacklist.last_change().replace(microsecond=0)
+
+        context['last_change'] = last_change
         context['blacklists'] = models.Blacklist.objects.all()
         return context
 
@@ -32,6 +38,11 @@ class AddView(BlacklistView, FormView):
 
     def form_valid(self, form):
         form.save()
+        mail_admins(subject="New Blacklist",
+                    message="Title: {}\nType: {}".format(
+                        form.cleaned_data.get('title'),
+                        form.cleaned_data.get('blacklist_type'),
+                    ))
         return HttpResponseRedirect(reverse('blacklists:list'))
 
 
@@ -43,9 +54,24 @@ def form_valid(self, form):
         with transaction.atomic(), revisions.create_revision():
             form.save()
             revisions.set_comment(form.cleaned_data['comment'])
+        mail_admins(subject="Blacklist Updated",
+                    message="Title: {}\nType: {}".format(
+                        form.cleaned_data.get('title'),
+                        form.cleaned_data.get('blacklist_type'),
+                    ))
         return HttpResponseRedirect(reverse('blacklists:list'))
 
 
+class BlacklistDump(TemplateView):
+    template_name = 'dump.txt'
+    content_type = 'text/text'
+
+    def get_context_data(self, **kwargs):
+        c = super().get_context_data(**kwargs)
+        c['urls'] = models.Blacklist.dump()
+        return c
+
+
 class History(BlacklistView, generic_views.HistoryView):
     """
         History log

Seeder/core/models.py

@@ -1,6 +1,6 @@
 from django.db import models
 from django.dispatch import receiver
-from django.db.models.signals import post_save, post_delete
+from django.db.models.signals import post_save, post_delete, post_migrate
 from django.utils import timezone
 
 from core import widgets
@@ -39,6 +39,7 @@ class Meta:
 
 @receiver(post_save, sender=OrderedModel)
 @receiver(post_delete, sender=OrderedModel)
+@receiver(post_migrate, sender=OrderedModel)
 def reorder_by_order(sender, instance, **kwargs):
     """
     Re-orders objects 'by order': normalizes the order

Seeder/settings/base.py

@@ -44,7 +44,8 @@
 }
 
 ADMINS = (
-    ('Visgean Skeloru', '[email protected]')
+    ('Visgean Skeloru', '[email protected]'),
+    ('Petr Manas', '[email protected]'),
 )
 
 IGNORABLE_404_URLS = (

Seeder/source/views.py

@@ -304,6 +304,6 @@ class SourceDump(TemplateView):
 
     def get_context_data(self, **kwargs):
         c = super().get_context_data(**kwargs)
-        c['seed_urls'] = models.Seed.archiving.public_seeds().all()\
+        c['urls'] = models.Seed.archiving.public_seeds().all()\
             .values_list('url', flat=True)
         return c

Seeder/static/main.js

@@ -0,0 +1,19 @@
+// ----------------------------------------------------------------------------
+// Text manipulation
+// ----------------------------------------------------------------------------
+// Source: https://stackoverflow.com/questions/985272/selecting-text-in-an-element-akin-to-highlighting-with-your-mouse/987376#987376
+function selectText(node) {
+  if (document.body.createTextRange) {
+      const range = document.body.createTextRange();
+      range.moveToElementText(node);
+      range.select();
+  } else if (window.getSelection) {
+      const selection = window.getSelection();
+      const range = document.createRange();
+      range.selectNodeContents(node);
+      selection.removeAllRanges();
+      selection.addRange(range);
+  } else {
+      console.warn("Could not select text in node: Unsupported browser.");
+  }
+}

Seeder/templates/base.html

@@ -13,8 +13,9 @@
 
     {% bootstrap_css %}
     {% bootstrap_javascript %}
-    <!-- Custom styles for this template -->
+    <!-- Custom styles and scritps for this template -->
     <link href="{{ STATIC_URL }}main.css" rel="stylesheet">
+    <script src="{{ STATIC_URL }}main.js"></script>
     {% block extrahead %}{% endblock %}
 
 </head>
@@ -84,18 +85,13 @@
                 <li id="sources">
                     <a href="{% url 'source:list' %}">{% trans 'Sources' %}</a>
                 </li>
-                <li id="harvests">
-                    <a href="{% url 'harvests:calendar' %}">{% trans 'Harvests' %}</a>
-                </li>
                 <li id="publishers">
                     <a href="{% url 'publishers:list' %}">{% trans 'Publishers' %}</a>
                 </li>
                 <li id="contracts">
                     <a href="{% url 'contracts:list' %}">{% trans 'Contracts' %}</a>
                 </li>
-                <li id="blacklists">
-                    <a href="{% url 'blacklists:list' %}">{% trans 'Blacklists' %}</a>
-                </li>
+
                 <li id="qa">
                     <a href="{% url 'qa:list' %}">{% trans 'Quality assurance' %}</a>
                 </li>
@@ -111,6 +107,16 @@
 
                 <hr>
 
+                <li id="harvests">
+                    <a href="{% url 'harvests:calendar' %}">{% trans 'Harvests' %}</a>
+                </li>
+                <li><a href="{% url 'source:dump' %}" target="_blank">{% trans 'Seed DB dump' %}</a></li>
+                <li id="blacklists">
+                    <a href="{% url 'blacklists:list' %}">{% trans 'Blacklists' %}</a>
+                </li>
+
+                <hr>
+
                 <li id="searchlog">
                     <a href="{% url 'search:list' %}">{% trans 'Search logs' %}</a>
                 </li>
@@ -120,7 +126,6 @@
                 <li><a href="https://github.com/WebArchivCZ/Seeder">Github</a></li>
                 <li><a href="https://github.com/WebArchivCZ/Seeder/issues/new">{% trans 'Bug report' %}</a></li>
                 <li><a href="https://seeder.readthedocs.org/en/latest/">{% trans 'Read the docs' %}</a></li>
-                <li><a href="{% url 'source:dump' %}">{% trans 'Seed DB dump' %}</a></li>
                 <li id="harvest_catalogue"><a href="{% url 'harvests:catalogue' %}">{% trans 'Harvests urls' %}</a></li>
 
 

Seeder/templates/dump.txt

@@ -0,0 +1,2 @@
+{% for url in urls %}{{ url|safe }}
+{% endfor %}