From 0d71bed0eda04c61622966d95c9e928c1b85e7dd Mon Sep 17 00:00:00 2001 From: Fasand Date: Wed, 3 Jul 2019 11:08:46 +0200 Subject: [PATCH 01/11] move blacklist, whitelist, harvests to their own nav section --- Seeder/templates/base.html | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Seeder/templates/base.html b/Seeder/templates/base.html index 8aed98d8..44042f32 100755 --- a/Seeder/templates/base.html +++ b/Seeder/templates/base.html @@ -84,18 +84,13 @@
  • {% trans 'Sources' %}
    • -
  • - {% trans 'Harvests' %} -
  • {% trans 'Publishers' %}
  • {% trans 'Contracts' %}
    • -
  • - {% trans 'Blacklists' %} -
    • +
  • {% trans 'Quality assurance' %}
    • @@ -111,6 +106,16 @@
    +
  • + {% trans 'Harvests' %} +
    • +
  • {% trans 'Seed DB dump' %}
    • +
  • + {% trans 'Blacklists' %} +
    • + +
    +
  • {% trans 'Search logs' %}
    • @@ -120,7 +125,6 @@
  • Github
  • {% trans 'Bug report' %}
  • {% trans 'Read the docs' %}
    • -
  • {% trans 'Seed DB dump' %}
  • {% trans 'Harvests urls' %}
    • From 3bda4405ac2aef071d0db692f3328e3125975b97 Mon Sep 17 00:00:00 2001 From: Fasand Date: Wed, 3 Jul 2019 11:35:51 +0200 Subject: [PATCH 02/11] add main.js to Seeder, blacklist select_all button --- Seeder/blacklists/templates/blacklists.html | 13 ++++++++++--- Seeder/static/main.js | 19 +++++++++++++++++++ Seeder/templates/base.html | 3 ++- 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 Seeder/static/main.js diff --git a/Seeder/blacklists/templates/blacklists.html b/Seeder/blacklists/templates/blacklists.html index 7b931964..78a80ffb 100644 --- a/Seeder/blacklists/templates/blacklists.html +++ b/Seeder/blacklists/templates/blacklists.html @@ -6,16 +6,23 @@ {% endblock %} {% block content %} + {% for blacklist in blacklists %}
    {{ blacklist.title }} | {% trans 'Edit' %} | - {% trans 'History' %} + {% trans 'History' %} | + {% trans 'Select all' %}
    - 
    {{ blacklist.url_list }}
    - + 
    {{ blacklist.url_list }}
    {% endfor %} diff --git a/Seeder/static/main.js b/Seeder/static/main.js new file mode 100644 index 00000000..7389fef9 --- /dev/null +++ b/Seeder/static/main.js @@ -0,0 +1,19 @@ +// ---------------------------------------------------------------------------- +// Text manipulation +// ---------------------------------------------------------------------------- +// Source: https://stackoverflow.com/questions/985272/selecting-text-in-an-element-akin-to-highlighting-with-your-mouse/987376#987376 +function selectText(node) { + if (document.body.createTextRange) { + const range = document.body.createTextRange(); + range.moveToElementText(node); + range.select(); + } else if (window.getSelection) { + const selection = window.getSelection(); + const range = document.createRange(); + range.selectNodeContents(node); + selection.removeAllRanges(); + selection.addRange(range); + } else { + console.warn("Could not select text in node: Unsupported browser."); + } +} \ No newline at end of file diff --git a/Seeder/templates/base.html b/Seeder/templates/base.html index 44042f32..0b935cdf 100755 --- a/Seeder/templates/base.html +++ b/Seeder/templates/base.html @@ -13,8 +13,9 @@ {% bootstrap_css %} {% bootstrap_javascript %} - + + {% block extrahead %}{% endblock %} From ee58ade95667650f46777d893cda84cec6797b7d Mon Sep 17 00:00:00 2001 From: Fasand Date: Wed, 3 Jul 2019 12:20:07 +0200 Subject: [PATCH 03/11] Blacklist now extends BaseModel, black/white lists now can be dumped, same template --- .../migrations/0002_auto_20190703_1018.py | 29 +++++++++++++++++++ Seeder/blacklists/models.py | 11 ++++++- Seeder/blacklists/templates/blacklists.html | 1 + Seeder/blacklists/urls.py | 1 + Seeder/blacklists/views.py | 9 ++++++ Seeder/source/templates/dump.txt | 2 -- Seeder/source/views.py | 2 +- Seeder/templates/dump.txt | 2 ++ 8 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 Seeder/blacklists/migrations/0002_auto_20190703_1018.py delete mode 100644 Seeder/source/templates/dump.txt create mode 100644 Seeder/templates/dump.txt diff --git a/Seeder/blacklists/migrations/0002_auto_20190703_1018.py b/Seeder/blacklists/migrations/0002_auto_20190703_1018.py new file mode 100644 index 00000000..5cb08917 --- /dev/null +++ b/Seeder/blacklists/migrations/0002_auto_20190703_1018.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2 on 2019-07-03 10:18 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('blacklists', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='blacklist', + name='active', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='blacklist', + name='created', + field=models.DateTimeField(default=django.utils.timezone.now, editable=False), + ), + migrations.AddField( + model_name='blacklist', + name='last_changed', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/Seeder/blacklists/models.py b/Seeder/blacklists/models.py index 5e991351..3a4dd547 100644 --- a/Seeder/blacklists/models.py +++ b/Seeder/blacklists/models.py @@ -5,9 +5,11 @@ from django.utils.translation import ugettext_lazy as _ from reversion import revisions +from core.models import BaseModel + @revisions.register -class Blacklist(models.Model): +class Blacklist(BaseModel): """ General model for control lists @@ -46,3 +48,10 @@ def collect_urls_by_type(cls, blacklist_type): # blacklist urls is now list of contents urls_parsed = map(str.splitlines, blacklist_urls) return reduce(operator.add, urls_parsed, []) + + @classmethod + def dump(cls): + blacklist_urls = cls.objects.all().values_list('url_list', flat=True) + urls_parsed = map(str.splitlines, blacklist_urls) + # Remove duplicates + return list(set(reduce(operator.add, urls_parsed, []))) diff --git a/Seeder/blacklists/templates/blacklists.html b/Seeder/blacklists/templates/blacklists.html index 78a80ffb..3499b325 100644 --- a/Seeder/blacklists/templates/blacklists.html +++ b/Seeder/blacklists/templates/blacklists.html @@ -3,6 +3,7 @@ {% block extra_header %} {% trans 'Add' %} +{% trans 'Dump' %} {% endblock %} {% block content %} diff --git a/Seeder/blacklists/urls.py b/Seeder/blacklists/urls.py index a9758b45..24fae045 100644 --- a/Seeder/blacklists/urls.py +++ b/Seeder/blacklists/urls.py @@ -6,4 +6,5 @@ path('add', AddView.as_view(), name='add'), path('/edit', EditView.as_view(), name='edit'), path('history/', History.as_view(), name='history'), + path('dump', BlacklistDump.as_view(), name='dump'), ] diff --git a/Seeder/blacklists/views.py b/Seeder/blacklists/views.py index 2d4aade3..5ab5cee2 100644 --- a/Seeder/blacklists/views.py +++ b/Seeder/blacklists/views.py @@ -45,6 +45,15 @@ def form_valid(self, form): revisions.set_comment(form.cleaned_data['comment']) return HttpResponseRedirect(reverse('blacklists:list')) +class BlacklistDump(TemplateView): + template_name = 'dump.txt' + content_type = 'text/text' + + def get_context_data(self, **kwargs): + c = super().get_context_data(**kwargs) + c['urls'] = models.Blacklist.dump() + return c + class History(BlacklistView, generic_views.HistoryView): """ diff --git a/Seeder/source/templates/dump.txt b/Seeder/source/templates/dump.txt deleted file mode 100644 index 259e920f..00000000 --- a/Seeder/source/templates/dump.txt +++ /dev/null @@ -1,2 +0,0 @@ -{% for url in seed_urls %}{{ url|safe }} -{% endfor %} \ No newline at end of file diff --git a/Seeder/source/views.py b/Seeder/source/views.py index 2d27b833..c0a5004d 100755 --- a/Seeder/source/views.py +++ b/Seeder/source/views.py @@ -304,6 +304,6 @@ class SourceDump(TemplateView): def get_context_data(self, **kwargs): c = super().get_context_data(**kwargs) - c['seed_urls'] = models.Seed.archiving.public_seeds().all()\ + c['urls'] = models.Seed.archiving.public_seeds().all()\ .values_list('url', flat=True) return c diff --git a/Seeder/templates/dump.txt b/Seeder/templates/dump.txt new file mode 100644 index 00000000..9718d775 --- /dev/null +++ b/Seeder/templates/dump.txt @@ -0,0 +1,2 @@ +{% for url in urls %}{{ url|safe }} +{% endfor %} \ No newline at end of file From bb3ba12630246efa2e2cc12574d64f21150319a3 Mon Sep 17 00:00:00 2001 From: Fasand Date: Wed, 3 Jul 2019 12:47:39 +0200 Subject: [PATCH 04/11] show last_change in Blacklist ListView --- Seeder/blacklists/models.py | 4 ++++ Seeder/blacklists/templates/blacklists.html | 7 +++++-- Seeder/blacklists/views.py | 5 +++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Seeder/blacklists/models.py b/Seeder/blacklists/models.py index 3a4dd547..8faf78f3 100644 --- a/Seeder/blacklists/models.py +++ b/Seeder/blacklists/models.py @@ -49,6 +49,10 @@ def collect_urls_by_type(cls, blacklist_type): urls_parsed = map(str.splitlines, blacklist_urls) return reduce(operator.add, urls_parsed, []) + @classmethod + def last_change(cls): + return max(cls.objects.all().values_list('last_changed', flat=True)) + @classmethod def dump(cls): blacklist_urls = cls.objects.all().values_list('url_list', flat=True) diff --git a/Seeder/blacklists/templates/blacklists.html b/Seeder/blacklists/templates/blacklists.html index 3499b325..de004a3f 100644 --- a/Seeder/blacklists/templates/blacklists.html +++ b/Seeder/blacklists/templates/blacklists.html @@ -2,8 +2,11 @@ {% load i18n %} {% block extra_header %} -{% trans 'Add' %} -{% trans 'Dump' %} +{% trans 'Add' %} +{% trans 'Dump' %} + + Last update: {{last_change}} ago + {% endblock %} {% block content %} diff --git a/Seeder/blacklists/views.py b/Seeder/blacklists/views.py index 5ab5cee2..6882d0fc 100644 --- a/Seeder/blacklists/views.py +++ b/Seeder/blacklists/views.py @@ -1,6 +1,7 @@ from django.urls import reverse from django.db import transaction from django.http.response import HttpResponseRedirect +from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from django.views.generic.base import TemplateView from django.views.generic.edit import FormView @@ -22,6 +23,10 @@ class ListView(BlacklistView, TemplateView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) + last_change = timezone.now().replace(microsecond=0) -\ + models.Blacklist.last_change().replace(microsecond=0) + + context['last_change'] = last_change context['blacklists'] = models.Blacklist.objects.all() return context From a24c46782100710275f0a81b4bb7fde00bb0bdf6 Mon Sep 17 00:00:00 2001 From: Fasand Date: Mon, 8 Jul 2019 11:43:28 +0200 Subject: [PATCH 05/11] add peter@petermanas.com to base.py ADMINS --- Seeder/settings/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Seeder/settings/base.py b/Seeder/settings/base.py index 37b896e0..aa0c0ccc 100755 --- a/Seeder/settings/base.py +++ b/Seeder/settings/base.py @@ -44,7 +44,8 @@ } ADMINS = ( - ('Visgean Skeloru', 'visgean@gmail.com') + ('Visgean Skeloru', 'visgean@gmail.com'), + ('Petr Manas', 'peter@petermanas.com'), ) IGNORABLE_404_URLS = ( From c4abb42e8fe24152fbf512f28f3f0c0a197800d7 Mon Sep 17 00:00:00 2001 From: Fasand Date: Mon, 8 Jul 2019 11:47:32 +0200 Subject: [PATCH 06/11] open blacklist dump in a new tab/window --- Seeder/blacklists/templates/blacklists.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Seeder/blacklists/templates/blacklists.html b/Seeder/blacklists/templates/blacklists.html index de004a3f..4319cca0 100644 --- a/Seeder/blacklists/templates/blacklists.html +++ b/Seeder/blacklists/templates/blacklists.html @@ -3,7 +3,7 @@ {% block extra_header %} {% trans 'Add' %} -{% trans 'Dump' %} +{% trans 'Dump' %} Last update: {{last_change}} ago From 170f042dd5f823e8e2d73795ce87a937f1926378 Mon Sep 17 00:00:00 2001 From: Fasand Date: Mon, 8 Jul 2019 11:57:12 +0200 Subject: [PATCH 07/11] blacklists: send email to admins when new/updated blacklist --- Seeder/blacklists/views.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Seeder/blacklists/views.py b/Seeder/blacklists/views.py index 6882d0fc..5fdbcbea 100644 --- a/Seeder/blacklists/views.py +++ b/Seeder/blacklists/views.py @@ -5,6 +5,7 @@ from django.utils.translation import ugettext_lazy as _ from django.views.generic.base import TemplateView from django.views.generic.edit import FormView +from django.core.mail import mail_admins from reversion import revisions @@ -37,6 +38,11 @@ class AddView(BlacklistView, FormView): def form_valid(self, form): form.save() + mail_admins(subject="New Blacklist", + message="Title: {}\nType: {}".format( + form.cleaned_data.get('title'), + form.cleaned_data.get('blacklist_type'), + )) return HttpResponseRedirect(reverse('blacklists:list')) @@ -48,8 +54,14 @@ def form_valid(self, form): with transaction.atomic(), revisions.create_revision(): form.save() revisions.set_comment(form.cleaned_data['comment']) + mail_admins(subject="Blacklist Updated", + message="Title: {}\nType: {}".format( + form.cleaned_data.get('title'), + form.cleaned_data.get('blacklist_type'), + )) return HttpResponseRedirect(reverse('blacklists:list')) + class BlacklistDump(TemplateView): template_name = 'dump.txt' content_type = 'text/text' From 5f15ffbdf38572aaddd56ad738c650a7f0caa263 Mon Sep 17 00:00:00 2001 From: Fasand Date: Mon, 8 Jul 2019 12:00:48 +0200 Subject: [PATCH 08/11] whitelist dump opens in a new tab/wwindow --- Seeder/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Seeder/templates/base.html b/Seeder/templates/base.html index 0b935cdf..07a88c06 100755 --- a/Seeder/templates/base.html +++ b/Seeder/templates/base.html @@ -110,7 +110,7 @@
  • {% trans 'Harvests' %}
    • -
  • {% trans 'Seed DB dump' %}
    • +
  • {% trans 'Seed DB dump' %}
  • {% trans 'Blacklists' %}
    • From 6d5d73683c5396d0422ca9bd487866b32ef1c91a Mon Sep 17 00:00:00 2001 From: Fasand Date: Mon, 8 Jul 2019 12:31:31 +0200 Subject: [PATCH 09/11] add Blacklist API with custom action 'lastchanged' --- Seeder/api/__init__.py | 1 + Seeder/api/serializers.py | 7 +++++++ Seeder/api/viewsets.py | 15 +++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/Seeder/api/__init__.py b/Seeder/api/__init__.py index 730dcc2c..462755d5 100755 --- a/Seeder/api/__init__.py +++ b/Seeder/api/__init__.py @@ -7,3 +7,4 @@ api_router.register('category', viewsets.CategoryViewSet) api_router.register('source', viewsets.SourceViewSet) api_router.register('seed', viewsets.SeedViewSet) +api_router.register('blacklist', viewsets.BlacklistViewSet) diff --git a/Seeder/api/serializers.py b/Seeder/api/serializers.py index 9de2dccf..4cdf52cc 100755 --- a/Seeder/api/serializers.py +++ b/Seeder/api/serializers.py @@ -1,5 +1,6 @@ import source import publishers +import blacklists from rest_framework.serializers import ModelSerializer from rest_framework.fields import CharField @@ -50,3 +51,9 @@ class SourceSerializer(ModelSerializer): class Meta: model = source.models.Source exclude = ['created_by', 'owner'] + + +class BlacklistSerializer(ModelSerializer): + class Meta: + model = blacklists.models.Blacklist + fields = '__all__' diff --git a/Seeder/api/viewsets.py b/Seeder/api/viewsets.py index 04034116..1d202365 100755 --- a/Seeder/api/viewsets.py +++ b/Seeder/api/viewsets.py @@ -1,7 +1,10 @@ import source +import blacklists from rest_framework import viewsets from rest_framework import mixins as rf_mixins +from rest_framework.decorators import action +from rest_framework.response import Response from . import serializers @@ -27,3 +30,15 @@ class SeedViewSet(viewsets.GenericViewSet, rf_mixins.RetrieveModelMixin, """ serializer_class = serializers.SeedSerializer queryset = source.models.Seed.objects.all() + + +class BlacklistViewSet(viewsets.GenericViewSet, rf_mixins.RetrieveModelMixin, + rf_mixins.ListModelMixin): + serializer_class = serializers.BlacklistSerializer + queryset = blacklists.models.Blacklist.objects.all() + + @action(methods=['get'], detail=False) + def lastchanged(self, request): + return Response({ + 'lastChanged': blacklists.models.Blacklist.last_change(), + }) From 19af2f07151a5e84db5697b25b8cadf9aed09368 Mon Sep 17 00:00:00 2001 From: Fasand Date: Mon, 8 Jul 2019 12:43:36 +0200 Subject: [PATCH 10/11] TopicCollections: reorder_by_order on post_migrate --- Seeder/core/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Seeder/core/models.py b/Seeder/core/models.py index 630f7553..06985653 100644 --- a/Seeder/core/models.py +++ b/Seeder/core/models.py @@ -1,6 +1,6 @@ from django.db import models from django.dispatch import receiver -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import post_save, post_delete, post_migrate from django.utils import timezone from core import widgets @@ -39,6 +39,7 @@ class Meta: @receiver(post_save, sender=OrderedModel) @receiver(post_delete, sender=OrderedModel) +@receiver(post_migrate, sender=OrderedModel) def reorder_by_order(sender, instance, **kwargs): """ Re-orders objects 'by order': normalizes the order From ae7cbeccde80cf1074be1d399832d65203d14969 Mon Sep 17 00:00:00 2001 From: Fasand Date: Tue, 9 Jul 2019 14:03:08 +0200 Subject: [PATCH 11/11] use aggregator Max instead of max() --- Seeder/blacklists/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Seeder/blacklists/models.py b/Seeder/blacklists/models.py index 8faf78f3..c1753c84 100644 --- a/Seeder/blacklists/models.py +++ b/Seeder/blacklists/models.py @@ -51,7 +51,8 @@ def collect_urls_by_type(cls, blacklist_type): @classmethod def last_change(cls): - return max(cls.objects.all().values_list('last_changed', flat=True)) + agg = Blacklist.objects.all().aggregate(models.Max('last_changed')) + return agg.get('last_changed__max') @classmethod def dump(cls):