diff --git a/.env b/.env new file mode 100644 index 0000000..eb12766 --- /dev/null +++ b/.env @@ -0,0 +1,39 @@ +# +# Environment variables used for development using a virtualenv +# + +# This assumes you've installed postgresql and rabbitmq on your +# machine and have added the accounts need to connect to them: +# +# sudo -u postgres psql +# postgres=# create database feeds; +# postgres=# create user feeds with encrypted password 'feeds'; +# postgres=# grant all privileges on database feeds to feeds; +# +# sudo rabbitmqctl add_user feeds feeds +# sudo rabbitmqctl set_permissions -p / feeds ".*" ".*" ".*" +# sudo rabbitmqctl add_vhost feeds + +# Set environment variables used by the postgresql commands to connect +# to the database. This is the admin account not the account used by +# Django to connect to the database. PGPASSWORD is particularly useful +# to avoid having to enter it for each command. You will need to configure +# the server to allow username/password (md5) authentication on local +# (socket) connections and so avoid having to su to the postgres user first. +# +# /etc/postgresql//main/pg_hba.conf +# local all all md5 +# +# For more info on environment variables see, +# https://www.postgresql.org/docs/current/libpq-envars.html + +PGHOST=localhost +PGPORT=5432 +PGUSER=postgres +PGPASSWORD=postgres + +# Environment variables used by Django + +CELERY_BROKER_URL=amqp://feeds:feeds@localhost:5672/feeds + +DB_HOST=localhost diff --git a/.env.docker b/.env.docker new file mode 100644 index 0000000..83637ca --- /dev/null +++ b/.env.docker @@ -0,0 +1,31 @@ +# +# Environment variables used for running the demo site using docker +# + +# Service: db +# Rather than set POSTGRES_PASSWORD (required) and leave POSTGRES_USER and +# so POSTGRES_DB to default to "postgres" we set the variables to the name +# of the app so it mirrors the configuration for locally installed services +# which might be shared between sites. However, the value of doing this is +# that you easily initialize the database with dumps from production where +# the ownership of the tables is defined. + +PGHOST=db +POSTGRES_DB=feeds +POSTGRES_USER=feeds +POSTGRES_PASSWORD=feeds + +# Service: broker +# This, like the db service, simply mirrors the configuration for rabbitmq +# being installed locally and possibly shared between sites. + +RABBITMQ_DEFAULT_USER=feeds +RABBITMQ_DEFAULT_PASS=feeds +RABBITMQ_DEFAULT_VHOST=feeds + +# Services: web, celery, celery-beat +# Environment variables used in Django settings + +CELERY_BROKER_URL=amqp://feeds:feeds@broker:5672/feeds + +DB_HOST=db diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..a8b0583 --- /dev/null +++ b/.envrc @@ -0,0 +1,12 @@ +# direnv file +# When you cd to this directory, direnv will activate the virtualenv +# and set all the environment variables in the .env file. +# See https://direnv.net/man/direnv-stdlib.1.html + +# Activate the virtualenv +# This simply replicates what venv/bin/activate does. +export VIRTUAL_ENV=venv +PATH_add "$VIRTUAL_ENV/bin" + +# Set environment variables from .env +dotenv diff --git a/.gitignore b/.gitignore index 59ea6b5..bfe772e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ __pycache__/ # Ignore dot files except those required for the project .* !.editorconfig +!.envrc +!.env +!.env.docker !.flake8 !.gitattributes !.github diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d4dba6f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# +# Dockerfile: configuration for building the images for the web app, +# celery-beat and celery-worker. +# + +# Base python image +FROM python:3.8.16-buster +# Add the site to the python path +ENV PYTHONPATH /code +# Send all output on stdout and stderr straight to the container logs +ENV PYTHONUNBUFFERED 1 +# Set the locale +ENV LC_ALL C.UTF-8 +# Terminals support 256 colours +ENV TERM xterm-256color + +# Mount the site directory + +RUN mkdir /code +WORKDIR /code + +# Install dependencies + +RUN apt-get update \ + && apt-get install -y --no-install-recommends + +RUN apt-get install -y --no-install-recommends \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install requirements + +COPY requirements/dev.txt /tmp/requirements.txt +RUN pip install --upgrade setuptools pip wheel\ + && pip install pip-tools==6.10.0\ + && pip install -r /tmp/requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index af126d1..9d11ebd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -11,11 +11,9 @@ include CHANGELOG.rst exclude setup.cfg exclude pyproject.toml -recursive-include src/app_project/locale * -recursive-include src/app_project/templates * -recursive-include src/app_project/static * +recursive-include src/feeds/ * -recursive-exclude src/app_project/tests * +recursive-exclude src/feeds/tests * recursive-exclude * .cache recursive-exclude * __pycache__ diff --git a/Makefile b/Makefile index 576d25f..fe271d0 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,8 @@ PYTHON = python3.10 site_python := /usr/bin/env $(PYTHON) root_dir := $(realpath .) -app_dir = $(root_dir)/src/app_project -demo_dir = $(root_dir)/project +app_dir = $(root_dir)/src/feeds +demo_dir = $(root_dir)/demo venv_name = venv venv_dir = $(root_dir)/$(venv_name) @@ -133,7 +133,7 @@ clean: clean-venv clean-build clean-tests clean-mypy clean-coverage clean-docs $(venv_dir): $(site_python) -m venv $(venv_dir) $(pip) install --upgrade pip setuptools wheel - $(pip) install pip-tools==6.10.0 + $(pip) install pip-tools requirements/dev.txt: requirements/dev.in requirements/docs.in requirements/tests.in $(pip-compile) requirements/dev.in @@ -189,19 +189,19 @@ translations: .PHONY: flake8 flake8: - $(flake8) $(site_dir) + $(flake8) $(app_dir) .PHONY: isort isort: - $(isort) --check $(site_dir) + $(isort) --check $(app_dir) .PHONY: black black: - $(black) --check $(site_dir) + $(black) --check $(app_dir) .PHONY: mypy mypy: - $(mypy) $(site_dir) + $(mypy) $(app_dir) .PHONY: checks checks: flake8 black isort mypy @@ -212,7 +212,7 @@ checks: flake8 black isort mypy .PHONY: coverage coverage: - $(pytest) --cov=app_project --cov-config=setup.cfg --cov-report html + $(pytest) --cov=feeds --cov-config=setup.cfg --cov-report html .PHONY: test test: diff --git a/README.md b/README.md index 699ef4a..0b91714 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,26 @@ -# Django App Project +# Django Feeds -[![Build Status](https://img.shields.io/github/actions/workflow/status/StuartMacKay/django-app-template/ci.yml?branch=master)](https://github.com/StuartMacKay/django-app-template/actions/workflows/ci.yml?query=branch%3Amaster) +[![Build Status](https://img.shields.io/github/actions/workflow/status/StuartMacKay/django-feeds/ci.yml?branch=master)](https://github.com/StuartMacKay/django-feeds/actions/workflows/ci.yml?query=branch%3Amaster) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) -## Features +Django Feeds is an aggregator for RSS and Atom feeds. -* Development with [black](https://github.com/psf/black) so everybody gets the code formatting rules they deserve -* Development with [flake8](https://flake8.pycqa.org/en/latest/) so people using ed get syntax checking -* Development with [isort](https://pycqa.github.io/isort/) for automatically sorting imports -* Development with [mypy](https://mypy-lang.org/) for type-hinting to catch errors -* Testing with [pytest](https://docs.pytest.org/), [FactoryBoy](https://factoryboy.readthedocs.io/en/stable/) and [tox](https://tox.wiki/en/latest/) -* Manage versions with [bump2version](https://pypi.org/project/bump2version/) - for semantic version numbers -* Manage dependency versions with [pip-tools](https://github.com/jazzband/pip-tools) -* Manage dependency upgrades with [pip-upgrade](https://github.com/simion/pip-upgrader) +## Features -## Quick start +## Quick Start -First, download and unzip the project files in the directory of your choice. -Then rename the project to something more useful: -```shell -mv django-app-template django-myapp -``` +## Demo -Change to the project directory and start setting things up: -```shell -cd django-myapp -``` +If you clone or download the [django-feeds](https://github.com/StuartMacKay/django-feeds) +repository there is a demonstration application that lets you see how it +all works. -First, build the virtualenv and install all the dependencies. This will -also build the library: ```shell +git clone git@github.com:StuartMacKay/django-feeds.git +cd django-feeds make install -``` - -Now run the demo site: -```shell make demo -``` - - -Run the database migrations: -```shell -./manage.py migrate -``` - -Run the tests: -```shell -make tests -``` - -Run the django server: -```shell -./manage.py runserver -``` - -Open a browser and visit http://localhost:8000 and, voila, we have a working -site. Well cover the deployment later. - -Almost all steps used the project Makefile. That's great if you're running -Linux with GNU Make and not much fun if you're not. All is not lost, however. -All the Makefile's targets contain only one or two commands, so even if you -are running Windows you should still be able to get the site running without -too much effort. +``` +It's a standard django project so if you don't have `make` available +then just look at the [Makefile](Makefile) and run the commands from +the various targets. diff --git a/TODO.md b/TODO.md index b6dbda7..484e8f9 100644 --- a/TODO.md +++ b/TODO.md @@ -1,2 +1,21 @@ # TODO +The code runs and runs well. It has been in production for almost two +years, so it is rather polished. It was factored out of the +[Voices for Independence](https://www.voices.scot) web site with s small +number of changes and is now being repackaged to make it reusable in other +projects we have in mind. + +* replace the original bootstrap css classes with semantic names, so it + can be restyled easily. + +* move 'business logic' out of the views and out of the querysets into a + services layer - an internal API. That way you're not forced + to subclass the views, and you can easily write your own. + +* Add more blocks to the templates and create more snippets so again you + can easily replace what is provided in the app. + +* Also load the feeds using cron, so you are not forced to use celery. + +* Add a REST API - at some point. diff --git a/demo/__init__.py b/demo/__init__.py new file mode 100644 index 0000000..cf2e85f --- /dev/null +++ b/demo/__init__.py @@ -0,0 +1 @@ +from .celery import app as celery_app diff --git a/demo/apps.py b/demo/apps.py new file mode 100644 index 0000000..a536b9b --- /dev/null +++ b/demo/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class Config(AppConfig): + name = "demo" diff --git a/demo/conf/asgi.py b/demo/asgi.py similarity index 80% rename from demo/conf/asgi.py rename to demo/asgi.py index c0d7d61..6516f3b 100644 --- a/demo/conf/asgi.py +++ b/demo/asgi.py @@ -11,6 +11,6 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.conf.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings") application = get_asgi_application() diff --git a/demo/celery.py b/demo/celery.py new file mode 100644 index 0000000..eca5503 --- /dev/null +++ b/demo/celery.py @@ -0,0 +1,73 @@ +import logging +import os + +import structlog +from celery import Celery # type: ignore +from celery.schedules import crontab +from celery.signals import setup_logging # type: ignore +from django_structlog.celery.steps import DjangoStructLogInitStep # type: ignore + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings") + +app = Celery() + +# Load the configuration +app.config_from_object("demo.settings") +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() +# Configure workers to use structlog +app.steps["worker"].add(DjangoStructLogInitStep) + +# A celery task runs every hour, on the error and checked the schedule for +# each active feed. If the schedule, also a crontab, matches the current +# time then the feed is loaded. This two-level approach comes in useful for +# feeds that are either updated in frequently, so there's no need to cook +# the planet by checking every hour, or, dealing with throttling by services +# such as Cloudflare, where too many request s + +FEEDS_TASK_SCHEDULE = os.environ.get("FEEDS_TASK_SCHEDULE", "0 * * * *") +minute, hour, day_of_week, day_of_month, month_of_year = FEEDS_TASK_SCHEDULE.split() + +app.conf.beat_schedule = { + "load-feeds": { + "task": "demo.tasks.load_feeds", + "schedule": crontab( + minute=minute, + hour=hour, + day_of_week=day_of_week, + day_of_month=day_of_month, + month_of_year=month_of_year, + ), + }, + "daily-digest": { + "task": "feed.tasks.daily_digest", + "schedule": crontab(minute="0", hour="2"), # 2am, each day + }, +} + + +@setup_logging.connect +def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs): # noqa + from django.conf import settings + + config = settings.LOGGING + logging.config.dictConfig(config) # type: ignore + + # noinspection DuplicatedCode + structlog.configure( + processors=[ + structlog.stdlib.filter_by_level, + structlog.processors.TimeStamper(fmt="iso"), + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.processors.UnicodeDecoder(), + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ], + context_class=structlog.threadlocal.wrap_dict(dict), + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, # type: ignore + cache_logger_on_first_use=True, + ) diff --git a/demo/conf/urls.py b/demo/conf/urls.py deleted file mode 100644 index 921ba72..0000000 --- a/demo/conf/urls.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -URL Configuration for the demonstration site. - -""" -from django.contrib import admin -from django.urls import path, include, reverse_lazy -from django.views.generic import RedirectView - -import debug_toolbar # type: ignore - - -urlpatterns = [ - path("", RedirectView.as_view(url=reverse_lazy("app_index"))), - path("admin/", admin.site.urls), - path("app/", include("app_project.urls")), - path("__debug__/toolbar/", include(debug_toolbar.urls)), # type: ignore -] diff --git a/demo/paginators.py b/demo/paginators.py new file mode 100644 index 0000000..c6c77cf --- /dev/null +++ b/demo/paginators.py @@ -0,0 +1,68 @@ +from datetime import timedelta +from math import ceil + +from django.core.paginator import Paginator +from django.utils import timezone +from django.utils.functional import cached_property + +from text_unidecode import unidecode # type: ignore + + +class DayPaginator(Paginator): + """ + The DayPaginator is used to show a week's worth of Articles on each + page, starting with today. In principle a page can be empty, though + one a number of Sources are active this is unlikely. + """ + + def page(self, number): + number = self.validate_number(number) + top = (number - 1) * self.per_page + bottom = top + self.per_page + tomorrow = timezone.now().replace(hour=0, minute=0, second=0, microsecond=0) + tomorrow += timedelta(days=1) + since = tomorrow - timedelta(days=bottom) + until = tomorrow - timedelta(days=top) + results = self.object_list.filter(date__gte=since, date__lt=until) + return self._get_page(results, number, self) + + @cached_property + def num_pages(self): + if self.count == 0: + return 0 + first = timezone.now() + last = self.object_list.last().date + return ceil((first - last).days / self.per_page) + + +class AlphabetPaginator(Paginator): + def __init__(self, object_list, per_page, orphans=0, allow_empty_first_page=True): + super().__init__(object_list, per_page, orphans, allow_empty_first_page) + + # Build an index of the first page on which each letter + # occurs. That allows a jump table to be displayed which + # takes the reader to the page where items starting with + # a given letter begin. + + self.index = {} + + attr_name = self.object_list.query.order_by[0] + + for idx, object in enumerate(self.object_list): + # Only index objects if the field is not blank + if value := str(getattr(object, attr_name)): + # normalise accented characters + letter = unidecode(value[0].upper()) + + if letter not in self.index: + page_number = int(idx / self.per_page) + 1 + self.index[letter] = page_number + + # Add the normalised letter to the object so a mini-heading + # can be displayed using the ifchanged template tag + object.index_letter = letter + + # Add the first index page where the letter occurs. That + # allows a 'continued' label to display at the top of the + # page + object.index_page = self.index[letter] diff --git a/demo/conf/settings.py b/demo/settings.py similarity index 56% rename from demo/conf/settings.py rename to demo/settings.py index 115b01e..5ec6ce6 100644 --- a/demo/conf/settings.py +++ b/demo/settings.py @@ -1,14 +1,12 @@ """ -MVS (Minimal Viable Settings) for running a basic site. +Demo site settings """ import os - DEBUG = True -CONF_DIR = os.path.dirname(os.path.abspath(__file__)) -DEMO_DIR = os.path.dirname(CONF_DIR) +DEMO_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(DEMO_DIR) SECRET_KEY = "+^%b!5_!ul7prtm_y0w_xluJg32aqna&+)&thhy3)jqr2g*0%s" @@ -21,7 +19,11 @@ "django.contrib.messages", "django.contrib.staticfiles", "debug_toolbar", - "app_project.apps.Config", + "django_celery_beat", + "django_extensions", + "tagulous", + "feeds.apps.Config", + "demo.apps.Config", ) MIDDLEWARE = [ @@ -30,22 +32,23 @@ "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", + "debug_toolbar.middleware.DebugToolbarMiddleware", ] -ROOT_URLCONF = "demo.conf.urls" +ROOT_URLCONF = "demo.urls" STATIC_URL = "/static/" TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [os.path.join(DEMO_DIR, "templates")], + "DIRS": [], "APP_DIRS": True, "OPTIONS": { "context_processors": [ "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", - "django.template.context_processors.request" + "django.template.context_processors.request", ], }, }, @@ -53,22 +56,33 @@ DATABASES = { "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": os.path.join(ROOT_DIR, "db.sqlite3"), + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": "feeds", + "USER": "feeds", + "PASSWORD": "feeds", + "HOST": os.environ["DB_HOST"], + "PORT": 5432, } } DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - "LOCATION": "site", - } -} - INTERNAL_IPS = [ "127.0.0.1", ] USE_TZ = True + +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "") + +CELERY_TASK_ALWAYS_EAGER = CELERY_BROKER_URL == "" + +CELERY_ACCEPT_CONTENT = ["json"] + +CELERY_WORKER_HIJACK_ROOT_LOGGER = False + +CELERY_TASK_ROUTES = { + "demo.tasks.load_feeds": {"delivery_mode": "transient"}, +} + +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" diff --git a/demo/static/images/rss.svg b/demo/static/images/rss.svg new file mode 100644 index 0000000..7bcbdac --- /dev/null +++ b/demo/static/images/rss.svg @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demo/static/js/index.js b/demo/static/js/index.js index e69de29..3319270 100644 --- a/demo/static/js/index.js +++ b/demo/static/js/index.js @@ -0,0 +1,23 @@ +function articleClick() { + 'use strict'; + let links = document.getElementsByClassName('article-link'); + for (let i = 0; i < links.length; i++) { + links[i].addEventListener('click', function(event) { + let xhttp = new XMLHttpRequest(); + let url = this.getAttribute("href"); + event.preventDefault(); + xhttp.open("POST", this.dataset.link, true); + xhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); + xhttp.send("article=" + this.dataset.pk); + setTimeout(function(){ + window.location.href = url; + }, 100); + }); + } +} + + +document.addEventListener('DOMContentLoaded', (event) => { + 'use strict'; + articleClick(); +}); diff --git a/demo/tasks.py b/demo/tasks.py new file mode 100644 index 0000000..3ad8eb2 --- /dev/null +++ b/demo/tasks.py @@ -0,0 +1,8 @@ +from feeds.services import feeds + +from .celery import app + + +@app.task() +def load_feeds() -> None: + feeds.load_feeds() diff --git a/demo/templates/demo/articles.html b/demo/templates/demo/articles.html new file mode 100644 index 0000000..88814e7 --- /dev/null +++ b/demo/templates/demo/articles.html @@ -0,0 +1,72 @@ +{% extends "base.html" %} +{% load feeds i18n %} + +{% block title %}{{ block.super }} - {% translate "Articles" %}{% endblock %} + +{% block sidebar %} + {% if tags %} +
+ {% block tags %} + {% translate "This week..." as title %} + {% include "demo/snippets/tag_cloud.html" %} + {% endblock %} +
+ {% endif %} +{% endblock %} + +{% block main %} +
+ {% for date, articles in object_groups.items %} +
+
+
{{ date|date:"M" }}
+
{{ date|date:"j" }}
+
+
    + {% for article in articles %} +
  • +
    + {{ article.title }} + {% for label in article|categories:"label" %} + {{ label.label|capfirst }} + {% endfor %} +
    + + {% if article.has_authors %} + {% for author in article.authors.all %} + {% if not forloop.first %} ,{% endif %}{{ author.name }} + {% endfor %} + - + {% endif %} + {{ article.source.name }} + {% if article.archive_url %} + - {% translate "Archived" %} + {% endif %} + {% if article.views %} + - {% blocktranslate count views=article.views %} + {{ views }} view + {% plural %} + {{ views }} views + {% endblocktranslate %} + {% endif %} + + {% if article.comment %} +
    {{ article.comment|safe }}
    + {% endif %} +
    • + {% endfor %} +
+
+ {% endfor %} + + {% block pagination %} + {% include "demo/snippets/pagination.html" with previous_page_title="Next Week" next_page_title="Previous Week"%} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/author.html b/demo/templates/demo/author.html new file mode 100644 index 0000000..a455bbf --- /dev/null +++ b/demo/templates/demo/author.html @@ -0,0 +1,68 @@ +{% extends "base.html" %} +{% load feeds i18n static %} + +{% block title %} + {% blocktranslate with name=author.name %} + Articles by {{ name }} + {% endblocktranslate %} +{% endblock %} + +{% block content %} +
+

{{ author.name }}

+ {% if author.description %} +

{{ author.description|safe }}

+ {% endif %} + + {% block tags %} + {% if tags %} + {% include "demo/snippets/tag_author_cloud.html" %} + {% endif %} + {% endblock %} + +
+ +
+ + + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/authors.html b/demo/templates/demo/authors.html new file mode 100644 index 0000000..4388234 --- /dev/null +++ b/demo/templates/demo/authors.html @@ -0,0 +1,55 @@ +{% extends "base.html" %} +{% load i18n %} + +{% block sidebar %} +
+

{% translate "Authors" %}

+
+ {% translate "Browse" %} +
+ {% block description %}{% endblock %} +
+{% endblock %} + +{% block main %} +
+ + + + + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/snippets/pagination.html b/demo/templates/demo/snippets/pagination.html new file mode 100644 index 0000000..e7e8844 --- /dev/null +++ b/demo/templates/demo/snippets/pagination.html @@ -0,0 +1,26 @@ +{% load i18n %} +{% if is_paginated %} +
+ +
+{% endif %} diff --git a/demo/templates/demo/snippets/tag_author_cloud.html b/demo/templates/demo/snippets/tag_author_cloud.html new file mode 100644 index 0000000..bc2099a --- /dev/null +++ b/demo/templates/demo/snippets/tag_author_cloud.html @@ -0,0 +1,13 @@ +{% load i18n %} +
+ {% translate "Topics covered..." as default_title %} + {% if title %}{{ title }}{% else %}{{ default_title }}{% endif %} + +
diff --git a/demo/templates/demo/snippets/tag_cloud.html b/demo/templates/demo/snippets/tag_cloud.html new file mode 100644 index 0000000..af5ad55 --- /dev/null +++ b/demo/templates/demo/snippets/tag_cloud.html @@ -0,0 +1,13 @@ +{% load i18n %} +
+ {% translate "Topics covered..." as default_title %} + {% if title %}{{ title }}{% else %}{{ default_title }}{% endif %} + +
diff --git a/demo/templates/demo/snippets/tag_source_cloud.html b/demo/templates/demo/snippets/tag_source_cloud.html new file mode 100644 index 0000000..e64c1b6 --- /dev/null +++ b/demo/templates/demo/snippets/tag_source_cloud.html @@ -0,0 +1,13 @@ +{% load i18n %} +
+ {% translate "Topics covered..." as default_title %} + {% if title %}{{ title }}{% else %}{{ default_title }}{% endif %} + +
diff --git a/demo/templates/demo/source.html b/demo/templates/demo/source.html new file mode 100644 index 0000000..ead70f2 --- /dev/null +++ b/demo/templates/demo/source.html @@ -0,0 +1,95 @@ +{% extends "base.html" %} +{% load feeds i18n static %} + +{% block title %}{% translate "Source" %}: {{ source.name }}{% endblock %} + +{% block content %} +
+

{{ source.name }}

+ {% if source.description %} +

{{ source.description|safe }}

+ {% endif %} + + {% block tags %} + {% if tags %} + {% include "demo/snippets/tag_source_cloud.html" %} + {% endif %} + {% endblock %} + +
+ {% if source.url or source.feed_set.all %} +

{% translate "Links" %}

+ {% endif %} +
+ {% if source.url %} + + + + + + {% endif %} + + {% for feed in source.feed_set.all %} +
+ + + + + +
+ {% endfor %} + +
+
+ +
+ +
+ + + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/sources.html b/demo/templates/demo/sources.html new file mode 100644 index 0000000..414b042 --- /dev/null +++ b/demo/templates/demo/sources.html @@ -0,0 +1,62 @@ +{% extends "base.html" %} +{% load i18n %} + +{% block title %}{% translate "Sources" %}{% endblock %} + +{% block sidebar %} +
+

{% translate "Sources" %}

+
+ {% translate "Browse" %} +
+
+ {% translate "Popular" %} +
+ {% block description %}{% endblock %} +
+{% endblock %} + +{% block main %} +
+ + + {% block object-list %} + + {% endblock %} + + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/tag.html b/demo/templates/demo/tag.html new file mode 100644 index 0000000..45703f7 --- /dev/null +++ b/demo/templates/demo/tag.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} +{% load feeds i18n %} + +{% block title %}{% translate "Tag" %}: {{ tag.name }}{% endblock %} + +{% block sidebar %} +
+

{{ tag.name }}

+ {% if tag.description %} +
+ {{ tag.description|safe }} +
+ {% endif %} + {% if tag.related.exists %} +

{% translate "Related Tags" %}

+ + {% endif %} +
+{% endblock %} + +{% block main %} +
+ + + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/tags.html b/demo/templates/demo/tags.html new file mode 100644 index 0000000..9458bfe --- /dev/null +++ b/demo/templates/demo/tags.html @@ -0,0 +1,57 @@ +{% extends "base.html" %} +{% load i18n %} + + {% block title %}{% translate "Tags" %}{% endblock %} + +{% block sidebar %} +
+

{% translate "Tags" %}

+
+ {% translate "Browse" %} +
+ {% block description %}{% endblock %} +
+{% endblock %} + +{% block main %} +
+ + + + + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/tests/test_article_view.py b/demo/tests/test_article_view.py new file mode 100644 index 0000000..b8da608 --- /dev/null +++ b/demo/tests/test_article_view.py @@ -0,0 +1,27 @@ +import pytest +from django.urls import reverse + +from feeds.tests.conditions.articles import article_was_clicked +from feeds.tests.conditions.responses import redirects_to +from feeds.tests.factories import ArticleFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def article(): + return ArticleFactory() + + +def test_view(client, article): + response = client.get( + reverse("article", kwargs={"code": article.code}), follow=False + ) + assert redirects_to(response, article.url) + assert article_was_clicked(article) + + +def test_click_view(client, article): + response = client.post(reverse("article-click"), data={"code": article.code}) + assert response.status_code == 204 + assert article_was_clicked(article) diff --git a/demo/tests/test_articles_view.py b/demo/tests/test_articles_view.py new file mode 100644 index 0000000..815f91d --- /dev/null +++ b/demo/tests/test_articles_view.py @@ -0,0 +1,53 @@ +import datetime as dt + +from django.urls import reverse +from django.utils import timezone + +import pytest + +from feeds.tests.conditions.articles import ( + only_articles_between_dates, + only_published_articles, + only_tags_for_articles, +) +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.factories import ( + ArticleFactory, + AuthorFactory, + CategoryFactory, + TagFactory, +) + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def url(): + return reverse("articles") + + +@pytest.fixture +def articles(): + CategoryFactory.create(name="label/repost") + CategoryFactory.create(name="label/updated") + TagFactory.create_batch(20) + AuthorFactory.create_batch(10) + return ArticleFactory.create_batch(100) + + +def test_get(client, url, articles): + response = client.get(url) + articles = response.context["object_list"] + tags = response.context["tags"] + end_date = dt.date.today() + start_date = end_date - dt.timedelta(days=6) + assert is_paginated(response) + assert is_ordered(response) + assert only_published_articles(articles, timezone.now()) + assert only_tags_for_articles(articles, tags) + assert only_articles_between_dates(articles, start_date, end_date) + + +def test_query_count(client, url, articles, django_assert_num_queries): + with django_assert_num_queries(8): + client.get(url) diff --git a/demo/tests/test_author_view.py b/demo/tests/test_author_view.py new file mode 100644 index 0000000..3ab62e9 --- /dev/null +++ b/demo/tests/test_author_view.py @@ -0,0 +1,61 @@ +from django.urls import reverse +from django.utils import timezone + +import pytest + +from feeds.tests.conditions.articles import ( + only_published_articles, + only_tags_for_articles, +) +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.conditions.responses import redirects_to, warning_message_shown +from feeds.tests.factories import ( + ArticleFactory, + AuthorFactory, + CategoryFactory, + SourceFactory, + TagFactory, +) + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def author(): + return AuthorFactory() + + +@pytest.fixture +def url(author): + return reverse("author", args=(author.slug,)) + + +@pytest.fixture +def articles(author): + CategoryFactory.create(name="label/repost") + CategoryFactory.create(name="label/updated") + TagFactory.create_batch(20) + SourceFactory.create_batch(20) + return ArticleFactory.create_batch(100, authors=[author]) + + +def test_get(client, url, author, articles): + response = client.get(url) + articles = response.context["object_list"] + tags = response.context["tags"] + assert is_paginated(response) + assert is_ordered(response) + assert only_published_articles(articles, timezone.now()) + assert only_tags_for_articles(articles, tags) + + +def test_errors(client): + response = client.get(reverse("author", args=("no-such-author",)), follow=True) + assert response.status_code == 200 + assert redirects_to(response, reverse("authors")) + assert warning_message_shown(response) + + +def test_query_count(client, url, articles, django_assert_num_queries): + with django_assert_num_queries(4): + client.get(url) diff --git a/demo/tests/test_authors_view.py b/demo/tests/test_authors_view.py new file mode 100644 index 0000000..9372afb --- /dev/null +++ b/demo/tests/test_authors_view.py @@ -0,0 +1,31 @@ +from django.urls import reverse + +import pytest + +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.factories import ArticleFactory, AuthorFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def url(): + return reverse("authors") + + +@pytest.fixture +def authors(): + authors = AuthorFactory.create_batch(50) + ArticleFactory.create_batch(100) + return authors + + +def test_get(client, url, authors): + response = client.get(url) + assert is_paginated(response) + assert is_ordered(response) + + +def test_query_count(client, url, authors, django_assert_num_queries): + with django_assert_num_queries(2): + client.get(url) diff --git a/demo/tests/test_source_view.py b/demo/tests/test_source_view.py new file mode 100644 index 0000000..b51f7e7 --- /dev/null +++ b/demo/tests/test_source_view.py @@ -0,0 +1,61 @@ +from django.urls import reverse +from django.utils import timezone + +import pytest + +from feeds.tests.conditions.articles import ( + only_published_articles, + only_tags_for_articles, +) +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.conditions.responses import redirects_to, warning_message_shown +from feeds.tests.factories import ( + ArticleFactory, + AuthorFactory, + CategoryFactory, + SourceFactory, + TagFactory, +) + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def source(): + return SourceFactory() + + +@pytest.fixture +def url(source): + return reverse("source", args=(source.slug,)) + + +@pytest.fixture +def articles(source): + CategoryFactory.create(name="label/repost") + CategoryFactory.create(name="label/updated") + TagFactory.create_batch(20) + AuthorFactory.create_batch(10) + return ArticleFactory.create_batch(100, source=source) + + +def test_get(client, url, articles): + response = client.get(url) + articles = response.context["object_list"] + tags = response.context["tags"] + assert is_paginated(response) + assert is_ordered(response) + assert only_published_articles(articles, timezone.now()) + assert only_tags_for_articles(articles, tags) + + +def test_errors(client): + response = client.get(reverse("source", args=("no-such-source",)), follow=True) + assert response.status_code == 200 + assert redirects_to(response, reverse("sources")) + assert warning_message_shown(response) + + +def test_query_count(client, url, articles, django_assert_num_queries): + with django_assert_num_queries(6): + client.get(url) diff --git a/demo/tests/test_sources_view.py b/demo/tests/test_sources_view.py new file mode 100644 index 0000000..7682fb0 --- /dev/null +++ b/demo/tests/test_sources_view.py @@ -0,0 +1,31 @@ +from django.urls import reverse + +import pytest + +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.factories import SourceFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def url(): + return reverse("sources") + + +@pytest.fixture +def sources(): + return SourceFactory.create_batch(50) + + +def test_get(client, url, sources): + response = client.get(url) + assert is_paginated(response) + assert is_ordered(response) + + +def test_query_count(client, url, sources, django_assert_num_queries): + # Includes a query to fetch the current site though this + # query does not appear in the debug toolbar + with django_assert_num_queries(2): + client.get(url) diff --git a/demo/tests/test_tag_view.py b/demo/tests/test_tag_view.py new file mode 100644 index 0000000..6fe554b --- /dev/null +++ b/demo/tests/test_tag_view.py @@ -0,0 +1,75 @@ +from django.urls import reverse +from django.utils import timezone + +import pytest + +from feeds.models import Author, Source +from feeds.tests.conditions.articles import only_published_articles +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.conditions.responses import redirects_to, warning_message_shown +from feeds.tests.factories import ( + ArticleFactory, + AuthorFactory, + CategoryFactory, + SourceFactory, + TagFactory, +) + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def tag(): + return TagFactory() + + +@pytest.fixture +def url(tag): + return reverse("tag", args=(tag.slug,)) + + +@pytest.fixture +def articles(tag): + CategoryFactory.create(name="label/repost") + CategoryFactory.create(name="label/updated") + SourceFactory.create_batch(20) + AuthorFactory.create_batch(10) + return ArticleFactory.create_batch(100, tags=[tag]) + + +def test_get(client, url, articles): + response = client.get(url) + articles = response.context["object_list"] + assert is_paginated(response) + assert is_ordered(response) + assert only_published_articles(articles, timezone.now()) + + +def test_errors(client): + response = client.get(reverse("tag", args=("no-such-tag",)), follow=True) + assert response.status_code == 200 + assert redirects_to(response, reverse("tags")) + assert warning_message_shown(response) + + +def test_tag_for_author(client, tag, articles): + author = Author.objects.first() + url = reverse("tag-author", args=(tag.slug, author.slug)) + response = client.get(url) + expected = [obj.pk for obj in author.articles.published()] + actual = [obj.pk for obj in response.context["object_list"]] + assert sorted(actual) == sorted(expected) + + +def test_tag_for_source(client, tag, articles): + source = Source.objects.first() + url = reverse("tag-source", args=(tag.slug, source.slug)) + response = client.get(url) + expected = [obj.pk for obj in source.articles.published()] + actual = [obj.pk for obj in response.context["object_list"]] + assert sorted(actual) == sorted(expected) + + +def test_query_count(client, url, articles, django_assert_num_queries): + with django_assert_num_queries(2): + client.get(url) diff --git a/demo/tests/test_tags_view.py b/demo/tests/test_tags_view.py new file mode 100644 index 0000000..4143e6f --- /dev/null +++ b/demo/tests/test_tags_view.py @@ -0,0 +1,31 @@ +from django.urls import reverse + +import pytest + +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.factories import TagFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def url(): + return reverse("tags") + + +@pytest.fixture +def tags(): + return TagFactory.create_batch(100) + + +def test_get(client, url, tags): + response = client.get(url) + assert is_paginated(response) + assert is_ordered(response) + + +def test_query_count(client, url, tags, django_assert_num_queries): + # Includes a query to fetch the current site though this + # query does not appear in the debug toolbar + with django_assert_num_queries(2): + client.get(url) diff --git a/demo/urls.py b/demo/urls.py new file mode 100644 index 0000000..0d29c73 --- /dev/null +++ b/demo/urls.py @@ -0,0 +1,72 @@ +from django.urls import path + +""" +URL Configuration for the demonstration site. + +""" +from django.contrib import admin +from django.urls import include, path + +import debug_toolbar # type: ignore + +from demo import views + +urlpatterns = [ + path( + "", + views.ArticlesView.as_view(), + name="articles", + ), + path( + "article/click/", + views.ArticleClickView.as_view(), + name="article-click", + ), + path( + "article//", + views.ArticleView.as_view(), + name="article", + ), + path( + "author/", + views.AuthorsView.as_view(), + name="authors", + ), + path( + "author//", + views.AuthorView.as_view(), + name="author", + ), + path( + "source/", + views.SourcesView.as_view(), + name="sources", + ), + path( + "source//", + views.SourceView.as_view(), + name="source", + ), + path( + "tag/", + views.TagsView.as_view(), + name="tags", + ), + path( + "tag//", + views.TagView.as_view(), + name="tag", + ), + path( + "tag//author//", + views.TagView.as_view(), + name="tag-author", + ), + path( + "tag//source//", + views.TagView.as_view(), + name="tag-source", + ), + path("admin/", admin.site.urls), + path("__debug__/toolbar/", include(debug_toolbar.urls)), # type: ignore +] diff --git a/demo/views/__init__.py b/demo/views/__init__.py new file mode 100644 index 0000000..342de67 --- /dev/null +++ b/demo/views/__init__.py @@ -0,0 +1,20 @@ +from .article import ArticleClickView, ArticleView +from .articles import ArticlesView +from .author import AuthorView +from .authors import AuthorsView +from .source import SourceView +from .sources import SourcesView +from .tag import TagView +from .tags import TagsView + +__all__ = ( + "ArticleView", + "ArticleClickView", + "AuthorView", + "AuthorsView", + "ArticlesView", + "SourceView", + "SourcesView", + "TagView", + "TagsView", +) diff --git a/demo/views/article.py b/demo/views/article.py new file mode 100644 index 0000000..d4d5ded --- /dev/null +++ b/demo/views/article.py @@ -0,0 +1,29 @@ +from django.http import HttpResponse +from django.utils.decorators import method_decorator +from django.views import View, generic +from django.views.decorators.csrf import csrf_exempt + +from feeds.models import Article + + +class ArticleView(generic.RedirectView): + # In the off chance there are Articles with duplicate codes return the + # url for the latest once since that's the one most likely to have been + # clicked. If no such Article exist then return None so the RedirectView + # responds with 410 Gone. + + def get_redirect_url(self, *args, **kwargs): + Article.objects.viewed(kwargs["code"]) + if article := Article.objects.with_code(kwargs["code"]): + return article.url + + +@method_decorator(csrf_exempt, name="dispatch") +class ArticleClickView(View): + # Record outbound link clicks via POST so no trail on who clicked what + # is left in the web server logs. + + def post(self, request): + code = request.POST.get("code") + Article.objects.viewed(code) + return HttpResponse(status=204) diff --git a/demo/views/articles.py b/demo/views/articles.py new file mode 100644 index 0000000..dbaff75 --- /dev/null +++ b/demo/views/articles.py @@ -0,0 +1,37 @@ +from django.views import generic + +from demo.paginators import DayPaginator +from demo.views.utils import shuffle +from feeds.models import Article, Tag + + +class ArticlesView(generic.ListView): + template_name = "demo/articles.html" + paginator_class = DayPaginator + paginate_by = 7 + + def get_queryset(self): + return ( + Article.objects.published() + .select_related("source") + .prefetch_related("authors", "categories") + .order_by("-date") + ) + + def get_groups_by_date(self, object_list): # noqa + groups = {} + for article in object_list: + date = article.date.date() + groups.setdefault(date, []) + groups[date].append(article) + return groups + + def get_tags(self, object_list): # noqa + return shuffle(Tag.objects.for_articles(object_list).weighted()) + + def get_context_data(self, *, object_list=None, **kwargs): + context = super().get_context_data(**kwargs) + objects = context["object_list"] + context["tags"] = self.get_tags(objects) + context["object_groups"] = self.get_groups_by_date(objects) + return context diff --git a/demo/views/author.py b/demo/views/author.py new file mode 100644 index 0000000..251da41 --- /dev/null +++ b/demo/views/author.py @@ -0,0 +1,44 @@ +from typing import List + +from django.contrib import messages +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ +from django.views import generic + +from demo.views.utils import shuffle +from feeds.models import Article, Author, Tag + + +class AuthorView(generic.ListView): + template_name = "demo/author.html" + paginate_by = 20 + + def get_queryset(self): + return ( + Article.objects.published() + .by_author(self.kwargs["slug"]) + .select_related("source") + .order_by("-date") + ) + + def get_author(self) -> Author: + return Author.objects.get(slug=self.kwargs["slug"]) + + def get_tags(self, articles) -> List[Tag]: + return shuffle(Tag.objects.for_articles(articles).weighted()) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["tags"] = self.get_tags(context["object_list"]) + return context + + def get(self, request, *args, **kwargs): + try: + author = self.get_author() + except Author.DoesNotExist: + messages.warning(request, _("An author with this name could not be found")) + return redirect(reverse("authors")) + self.object_list = self.get_queryset() + context = self.get_context_data(author=author) + return self.render_to_response(context) diff --git a/demo/views/authors.py b/demo/views/authors.py new file mode 100644 index 0000000..a72a4f1 --- /dev/null +++ b/demo/views/authors.py @@ -0,0 +1,21 @@ +from django.db.models import Count, Prefetch +from django.views import generic + +from demo.paginators import AlphabetPaginator +from feeds.models import Article, Author + + +class AuthorsView(generic.ListView): + template_name = "demo/authors.html" + paginator_class = AlphabetPaginator + paginate_by = 25 + + def get_queryset(self): + return ( + Author.objects.published() + .order_by("name") + .annotate(article_count=Count("articles")) + .prefetch_related( + Prefetch("articles", queryset=Article.objects.all().order_by("-date")) + ) + ) diff --git a/demo/views/source.py b/demo/views/source.py new file mode 100644 index 0000000..8e35751 --- /dev/null +++ b/demo/views/source.py @@ -0,0 +1,45 @@ +from typing import List + +from django.contrib import messages +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ +from django.views import generic + +from demo.views.utils import shuffle +from feeds.models import Article, Source, Tag + + +class SourceView(generic.ListView): + template_name = "demo/source.html" + paginate_by = 20 + + def get_queryset(self): + return ( + Article.objects.published() + .by_source(self.kwargs["slug"]) + .select_related("source") + .prefetch_related("authors") + .order_by("-date") + ) + + def get_source(self) -> Source: + return Source.objects.get(slug=self.kwargs["slug"]) + + def get_tags(self, articles) -> List[Tag]: + return shuffle(Tag.objects.for_articles(articles).weighted()) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["tags"] = self.get_tags(context["object_list"]) + return context + + def get(self, request, *args, **kwargs): + try: + source = self.get_source() + except Source.DoesNotExist: + messages.warning(request, _("A site with this name could not be found")) + return redirect(reverse("sources")) + self.object_list = self.get_queryset() + context = self.get_context_data(source=source) + return self.render_to_response(context) diff --git a/demo/views/sources.py b/demo/views/sources.py new file mode 100644 index 0000000..2d70dff --- /dev/null +++ b/demo/views/sources.py @@ -0,0 +1,24 @@ +from django.db.models import Count, Prefetch +from django.views import generic + +from demo.paginators import AlphabetPaginator +from feeds.models import Article, Source + + +class SourcesView(generic.ListView): + template_name = "demo/sources.html" + paginator_class = AlphabetPaginator + paginate_by = 25 + + def get_queryset(self): + return ( + Source.objects.all() + .order_by("name") + .annotate(article_count=Count("articles")) + .prefetch_related( + Prefetch( + "articles", + queryset=Article.objects.published().order_by("-date"), + ) + ) + ) diff --git a/demo/views/tag.py b/demo/views/tag.py new file mode 100644 index 0000000..60293cd --- /dev/null +++ b/demo/views/tag.py @@ -0,0 +1,44 @@ +from django.contrib import messages +from django.db.models import Q +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ +from django.views import generic + +from feeds.models import Article, Tag + + +class TagView(generic.ListView): + template_name = "demo/tag.html" + paginate_by = 20 + + def get_queryset(self): + return ( + Article.objects.published() + .by_tag(self.kwargs["slug"]) + .filter(self.get_filters()) + .select_related("source") + .prefetch_related("authors") + .order_by("-date") + ) + + def get_filters(self): + filters = Q() + if author := self.kwargs.get("author", ""): # noqa + filters &= Q(authors__slug=author.strip()) + if source := self.kwargs.get("source", ""): # noqa + filters &= Q(source__slug=source.strip()) + return filters + + def get_tag(self): + return Tag.objects.get(slug=self.kwargs["slug"]) + + def get(self, request, *args, **kwargs): + try: + tag = self.get_tag() + except Tag.DoesNotExist: + messages.warning(request, _("A tag with this title could not be found")) + return redirect(reverse("tags")) + self.object_list = self.get_queryset() + context = self.get_context_data(tag=tag) + return self.render_to_response(context) diff --git a/demo/views/tags.py b/demo/views/tags.py new file mode 100644 index 0000000..a703548 --- /dev/null +++ b/demo/views/tags.py @@ -0,0 +1,23 @@ +from django.db.models import Count, Prefetch +from django.views import generic + +from demo.paginators import AlphabetPaginator +from feeds.models import Article, Tag + + +class TagsView(generic.ListView): + template_name = "demo/tags.html" + paginator_class = AlphabetPaginator + paginate_by = 25 + + def get_queryset(self): + return ( + Tag.objects.all() + .order_by("name") + .annotate(article_count=Count("article")) + .prefetch_related( + Prefetch( + "article_set", queryset=Article.objects.all().order_by("-date") + ) + ) + ) diff --git a/demo/views/utils.py b/demo/views/utils.py new file mode 100644 index 0000000..48b9daa --- /dev/null +++ b/demo/views/utils.py @@ -0,0 +1,7 @@ +import random + + +def shuffle(objects): + random.seed(random.randint(1, 10000)) + random.shuffle(objects) + return objects diff --git a/demo/conf/wsgi.py b/demo/wsgi.py similarity index 80% rename from demo/conf/wsgi.py rename to demo/wsgi.py index 062381a..0a0e4d0 100644 --- a/demo/conf/wsgi.py +++ b/demo/wsgi.py @@ -11,6 +11,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.conf.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings") application = get_wsgi_application() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5721165 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,58 @@ +version: '3.1' + +services: + db: + image: postgres:15 + ports: + - "15432:5432" + volumes: + - ${DB_DATA_DIR-./.data/db}:/var/lib/postgresql/data + env_file: + - .env.docker + + broker: + image: rabbitmq:3.11.9-management + env_file: + - .env.docker + + web: + stdin_open: true + tty: true + build: . + command: /code/docker-init.sh + volumes: &web-volumes + - ./:/code + ports: + - "8000:8000" + depends_on: &web-depends-on + - db + - cache + - broker + env_file: + - .env.docker + + flower: + image: mher/flower + ports: + - "15555:5555" + command: celery flower --broker_api=http://feeds:feeds@broker:15672/api/feeds + depends_on: + - broker + env_file: + - .env.docker + + celery-beat: + build: . + command: celery --app feeds beat --loglevel debug + volumes: *web-volumes + depends_on: *web-depends-on + env_file: + - .env.docker + + celery-worker: + build: . + command: celery --app feeds worker --loglevel debug + volumes: *web-volumes + depends_on: *web-depends-on + env_file: + - .env.docker diff --git a/docker-init.sh b/docker-init.sh new file mode 100755 index 0000000..adc409a --- /dev/null +++ b/docker-init.sh @@ -0,0 +1,3 @@ +#!/bin/bash +python /code/manage.py migrate +python /code/manage.py runserver 0.0.0.0:8000 diff --git a/manage.py b/manage.py index 13e3aa2..86cc0b0 100755 --- a/manage.py +++ b/manage.py @@ -2,9 +2,8 @@ import os import sys - if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.conf.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings") from django.core.management import execute_from_command_line diff --git a/requirements/dev.in b/requirements/dev.in index 09d518d..a62cc84 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -8,6 +8,14 @@ -e . +# Packages required by the demo site + +bs4 +celery +django-celery-beat +requests +text_unidecode + # bumpversion is no longer maintained so we use bump2version until the # point where bump2version emerges as the official bumpversion, see # https://github.com/c4urself/bump2version/issues/86 diff --git a/requirements/dev.txt b/requirements/dev.txt index 7d6131a..c039ab9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,69 +1,130 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile requirements/dev.in # --e file:///home/stuart/Development/django-app-template +-e file:///home/stuart/Development/django-feeds # via -r requirements/dev.in alabaster==0.7.13 # via sphinx -asgiref==3.6.0 - # via django -attrs==22.2.0 - # via pytest +amqp==5.1.1 + # via kombu +asgiref==3.7.2 + # via + # django + # django-structlog babel==2.12.1 # via sphinx -black==23.1.0 +beautifulsoup4==4.12.2 + # via bs4 +billiard==4.1.0 + # via celery +black==23.9.1 # via -r requirements/tests.in -bleach==6.0.0 - # via readme-renderer +bs4==0.0.1 + # via -r requirements/dev.in bump2version==1.0.1 # via -r requirements/dev.in -cachetools==5.3.0 +cachetools==5.3.1 # via tox -certifi==2022.12.7 +celery==5.3.4 + # via + # -r requirements/dev.in + # django-celery-beat + # django-structlog +certifi==2023.7.22 # via requests cffi==1.15.1 # via cryptography -chardet==5.1.0 +chardet==5.2.0 # via tox -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests -click==8.1.3 - # via black +click==8.1.7 + # via + # black + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.0 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery colorama==0.4.6 # via tox -coverage[toml]==7.2.1 +colorclass==2.2.2 + # via pip-upgrader +coverage[toml]==7.3.1 # via pytest-cov -cryptography==39.0.2 +cron-descriptor==1.4.0 + # via django-celery-beat +croniter==1.4.1 + # via django-rss-feeds +cryptography==41.0.4 # via secretstorage -distlib==0.3.6 +distlib==0.3.7 # via virtualenv -django==4.1.7 - # via django-library-project +django==3.2.21 + # via + # django-celery-beat + # django-debug-toolbar + # django-extensions + # django-rss-feeds + # django-structlog + # django-stubs + # django-stubs-ext + # django-tagulous + # django-timezone-field +django-celery-beat==2.5.0 + # via -r requirements/dev.in +django-debug-toolbar==4.2.0 + # via -r requirements/dev.in +django-extensions==3.2.3 + # via django-rss-feeds +django-ipware==5.0.0 + # via django-structlog +django-structlog[celery]==5.3.0 + # via django-rss-feeds +django-stubs==4.2.4 + # via -r requirements/dev.in +django-stubs-ext==4.2.2 + # via django-stubs +django-tagulous==1.3.3 + # via django-rss-feeds +django-timezone-field==6.0.1 + # via django-celery-beat +docopt==0.6.2 + # via pip-upgrader docutils==0.18.1 # via # readme-renderer # sphinx # sphinx-rtd-theme -exceptiongroup==1.1.0 +exceptiongroup==1.1.3 # via pytest -factory-boy==3.2.1 +factory-boy==3.3.0 # via pytest-factoryboy -faker==17.6.0 +faker==19.6.2 # via factory-boy -filelock==3.9.0 +feedparser==6.0.10 + # via django-rss-feeds +filelock==3.12.4 # via # tox # virtualenv -flake8==6.0.0 +flake8==6.1.0 # via -r requirements/tests.in +freezegun==1.2.2 + # via pytest-freezegun idna==3.4 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==6.0.0 +importlib-metadata==6.8.0 # via # keyring # twine @@ -73,7 +134,7 @@ iniconfig==2.0.0 # via pytest isort==5.12.0 # via -r requirements/tests.in -jaraco-classes==3.2.3 +jaraco-classes==3.3.0 # via keyring jeepney==0.8.0 # via @@ -81,143 +142,199 @@ jeepney==0.8.0 # secretstorage jinja2==3.1.2 # via sphinx -keyring==23.13.1 +keyring==24.2.0 # via twine -lack==0.0.0 - # via -r requirements/tests.in -markdown-it-py==2.2.0 +kombu==5.3.2 + # via celery +markdown-it-py==3.0.0 # via rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via jinja2 mccabe==0.7.0 # via flake8 mdurl==0.1.2 # via markdown-it-py -more-itertools==9.1.0 +more-itertools==10.1.0 # via jaraco-classes -mypy==1.1.1 - # via -r requirements/tests.in +mypy==1.5.1 + # via + # -r requirements/tests.in + # django-stubs mypy-extensions==1.0.0 # via # black # mypy -packaging==23.0 +nh3==0.2.14 + # via readme-renderer +packaging==23.1 # via # black + # pip-upgrader # pyproject-api # pytest # sphinx # tox -pathspec==0.11.0 - # via - # black - # twine +pathspec==0.11.2 + # via black +pip-upgrader==1.4.15 + # via -r requirements/dev.in pkginfo==1.9.6 # via twine -platformdirs==3.1.1 +platformdirs==3.10.0 # via # black # tox # virtualenv -pluggy==1.0.0 +pluggy==1.3.0 # via # pytest # tox -pycodestyle==2.10.0 +prompt-toolkit==3.0.39 + # via click-repl +psycopg2-binary==2.9.7 + # via django-rss-feeds +pycodestyle==2.11.0 # via flake8 pycparser==2.21 # via cffi -pyflakes==3.0.1 +pyflakes==3.1.0 # via flake8 -pygments==2.14.0 +pygments==2.16.1 # via # readme-renderer # rich # sphinx -pyproject-api==1.5.0 +pyproject-api==1.6.1 # via tox -pytest==7.2.2 +pytest==7.4.2 # via # -r requirements/tests.in # pytest-cov # pytest-django # pytest-factoryboy -pytest-cov==4.0.0 + # pytest-freezegun +pytest-cov==4.1.0 # via -r requirements/tests.in pytest-django==4.5.2 # via -r requirements/tests.in pytest-factoryboy==2.5.1 # via -r requirements/tests.in +pytest-freezegun==0.4.2 + # via -r requirements/tests.in +python-crontab==3.0.0 + # via django-celery-beat python-dateutil==2.8.2 - # via faker -readme-renderer==37.3 + # via + # celery + # croniter + # django-rss-feeds + # faker + # freezegun + # python-crontab +pytz==2023.3.post1 + # via django +readme-renderer==42.0 # via twine -requests==2.28.2 +requests==2.31.0 # via + # -r requirements/dev.in + # pip-upgrader # requests-toolbelt # sphinx # twine -requests-toolbelt==0.10.1 +requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.3.2 +rich==13.5.3 # via twine secretstorage==3.3.3 # via keyring +sgmllib3k==1.0.0 + # via feedparser six==1.16.0 - # via - # bleach - # python-dateutil + # via python-dateutil snowballstemmer==2.2.0 # via sphinx -sphinx==6.1.3 +soupsieve==2.5 + # via beautifulsoup4 +sphinx==7.2.6 # via # -r requirements/docs.in # sphinx-rtd-theme -sphinx-rtd-theme==1.2.0 + # sphinxcontrib-applehelp + # sphinxcontrib-devhelp + # sphinxcontrib-htmlhelp + # sphinxcontrib-jquery + # sphinxcontrib-qthelp + # sphinxcontrib-serializinghtml +sphinx-rtd-theme==1.3.0 # via -r requirements/docs.in -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==1.0.7 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==1.0.5 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.0.4 # via sphinx -sphinxcontrib-jquery==2.0.0 +sphinxcontrib-jquery==4.1 # via sphinx-rtd-theme sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==1.0.6 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==1.1.9 # via sphinx -sqlparse==0.4.3 - # via django +sqlparse==0.4.4 + # via + # django + # django-debug-toolbar +structlog==23.1.0 + # via django-structlog +terminaltables==3.1.10 + # via pip-upgrader +text-unidecode==1.3 + # via -r requirements/dev.in tomli==2.0.1 # via + # black # coverage + # django-stubs # mypy # pyproject-api # pytest # tox -tox==4.4.6 +tox==4.11.3 # via -r requirements/dev.in twine==4.0.2 # via -r requirements/dev.in -typing-extensions==4.5.0 +types-pytz==2023.3.1.1 + # via django-stubs +types-pyyaml==6.0.12.11 + # via django-stubs +typing-extensions==4.8.0 # via + # asgiref + # black + # django-stubs + # django-stubs-ext # mypy # pytest-factoryboy -urllib3==1.26.14 +tzdata==2023.3 + # via + # celery + # django-celery-beat +urllib3==2.0.5 # via # requests # twine -virtualenv==20.20.0 +vine==5.0.0 + # via + # amqp + # celery + # kombu +virtualenv==20.24.5 # via tox -webencodings==0.5.1 - # via bleach -zipp==3.15.0 +wcwidth==0.2.6 + # via prompt-toolkit +zipp==3.17.0 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/tests.in b/requirements/tests.in index ecd5d57..5c82ef0 100644 --- a/requirements/tests.in +++ b/requirements/tests.in @@ -6,4 +6,5 @@ mypy pytest pytest-django pytest-factoryboy +pytest-freezegun pytest-cov diff --git a/requirements/tests.txt b/requirements/tests.txt index 6fb9879..6e2b428 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,25 +1,25 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile requirements/tests.in # -attrs==22.2.0 - # via pytest -black==23.1.0 +black==23.9.1 # via -r requirements/tests.in -click==8.1.3 +click==8.1.7 # via black -coverage[toml]==7.2.1 +coverage[toml]==7.3.1 # via pytest-cov -exceptiongroup==1.1.0 +exceptiongroup==1.1.3 # via pytest -factory-boy==3.2.1 +factory-boy==3.3.0 # via pytest-factoryboy -faker==17.6.0 +faker==19.6.2 # via factory-boy -flake8==6.0.0 +flake8==6.1.0 # via -r requirements/tests.in +freezegun==1.2.2 + # via pytest-freezegun inflection==0.5.1 # via pytest-factoryboy iniconfig==2.0.0 @@ -28,40 +28,45 @@ isort==5.12.0 # via -r requirements/tests.in mccabe==0.7.0 # via flake8 -mypy==1.1.1 +mypy==1.5.1 # via -r requirements/tests.in mypy-extensions==1.0.0 # via # black # mypy -packaging==23.0 +packaging==23.1 # via # black # pytest -pathspec==0.11.0 +pathspec==0.11.2 # via black -platformdirs==3.1.1 +platformdirs==3.10.0 # via black -pluggy==1.0.0 +pluggy==1.3.0 # via pytest -pycodestyle==2.10.0 +pycodestyle==2.11.0 # via flake8 -pyflakes==3.0.1 +pyflakes==3.1.0 # via flake8 -pytest==7.2.2 +pytest==7.4.2 # via # -r requirements/tests.in # pytest-cov # pytest-django # pytest-factoryboy -pytest-cov==4.0.0 + # pytest-freezegun +pytest-cov==4.1.0 # via -r requirements/tests.in pytest-django==4.5.2 # via -r requirements/tests.in pytest-factoryboy==2.5.1 # via -r requirements/tests.in +pytest-freezegun==0.4.2 + # via -r requirements/tests.in python-dateutil==2.8.2 - # via faker + # via + # faker + # freezegun six==1.16.0 # via python-dateutil tomli==2.0.1 @@ -70,7 +75,8 @@ tomli==2.0.1 # coverage # mypy # pytest -typing-extensions==4.5.0 +typing-extensions==4.8.0 # via + # black # mypy # pytest-factoryboy diff --git a/setup.cfg b/setup.cfg index 2500be9..98379cb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ tag = True tag_name = v{new_version} sign_tags = True -[bumpversion:file:src/app_project/__init__.py] +[bumpversion:file:src/feeds/__init__.py] search = __version__ = "{current_version}" replace = __version__ = "{new_version}" @@ -88,7 +88,7 @@ line_length = 88 default_section = THIRDPARTY known_django = django -known_first_party = app_project +known_first_party = feeds sections = FUTURE, STDLIB, @@ -103,7 +103,7 @@ skip = [tool:pytest] -DJANGO_SETTINGS_MODULE = demo.conf.settings +DJANGO_SETTINGS_MODULE = demo.settings testpaths = src diff --git a/setup.py b/setup.py index 95a045d..15c52fb 100644 --- a/setup.py +++ b/setup.py @@ -17,29 +17,34 @@ def read(filename): setup( - name="django-library-project", + name="django-rss-feeds", version="0.0.0", - description="A deployable Django app.", + description="An aggregator for RSS and Atom feeds.", long_description=read("README.md"), long_description_content_type="text/x-rst", author="Stuart MacKay", author_email="smackay@flagstonesoftware.com", - keywords="django, app, template", - url="https://github.com/StuartMacKay/django-app-template", - packages=[ - "app_project", - "app_project/migrations" - ], + keywords="django, rss, atom", + url="https://github.com/StuartMacKay/django-feeds", + packages=["feeds", "feeds/migrations"], package_dir={"": "src"}, include_package_data=True, zip_safe=False, python_requires=">=3.8", - install_requires="Django>=3.2", + install_requires=[ + "Django>=3.2,<3.3", + "croniter", + "django-extensions", + "django-structlog[celery]", + "django-tagulous", + "feedparser", + "python-dateutil", + "psycopg2-binary", + ], license="License :: OSI Approved :: Apache Software License", classifiers=[ "Development Status :: 3 - Alpha", "Framework :: Django :: 3.2", - "Framework :: Django :: 4.0", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Natural Language :: English", @@ -47,5 +52,5 @@ def read(filename): "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.10", - ] + ], ) diff --git a/src/app_project/admin/__init__.py b/src/app_project/admin/__init__.py deleted file mode 100644 index a816332..0000000 --- a/src/app_project/admin/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .example import ExampleAdmin - -__all__ = ( - "ExampleAdmin", -) diff --git a/src/app_project/admin/example.py b/src/app_project/admin/example.py deleted file mode 100644 index adba364..0000000 --- a/src/app_project/admin/example.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.contrib import admin - -from app_project import models - - -@admin.register(models.Example) -class ExampleAdmin(admin.ModelAdmin): - list_display = ("name",) - list_filter = ("name",) diff --git a/src/app_project/migrations/0001_initial.py b/src/app_project/migrations/0001_initial.py deleted file mode 100644 index b868677..0000000 --- a/src/app_project/migrations/0001_initial.py +++ /dev/null @@ -1,31 +0,0 @@ -# Generated by Django 3.1.1 on 2020-09-06 13:13 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [] - - operations = [ - migrations.CreateModel( - name="Example", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "name", - models.CharField(blank=True, max_length=32, verbose_name="Name"), - ), - ], - ), - ] diff --git a/src/app_project/models/__init__.py b/src/app_project/models/__init__.py deleted file mode 100644 index f04a629..0000000 --- a/src/app_project/models/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .example import Example - -__all__ = ( - "Example", -) diff --git a/src/app_project/models/example.py b/src/app_project/models/example.py deleted file mode 100644 index 0302494..0000000 --- a/src/app_project/models/example.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.db import models -from django.utils.translation import gettext_lazy as _ - - -class Example(models.Model): - name = models.CharField(verbose_name=_("Name"), max_length=32, blank=True) diff --git a/src/app_project/settings.py b/src/app_project/settings.py deleted file mode 100644 index ecfa14d..0000000 --- a/src/app_project/settings.py +++ /dev/null @@ -1,3 +0,0 @@ -# app-specific settings.py -# These will be added to the main settings when the app is loaded. -# See ./apps.py diff --git a/src/app_project/templates/app_project/base.html b/src/app_project/templates/app_project/base.html deleted file mode 100644 index 7219eb5..0000000 --- a/src/app_project/templates/app_project/base.html +++ /dev/null @@ -1,3 +0,0 @@ -{% extends "base.html" %} -{% load static i18n %} - diff --git a/src/app_project/templates/app_project/index.html b/src/app_project/templates/app_project/index.html deleted file mode 100644 index b8fe5d6..0000000 --- a/src/app_project/templates/app_project/index.html +++ /dev/null @@ -1,6 +0,0 @@ -{% extends "app_project/base.html" %} -{% load static i18n %} - -{% block content %} -

App Project

-{% endblock %} diff --git a/src/app_project/tests/test_views.py b/src/app_project/tests/test_views.py deleted file mode 100644 index f7bf597..0000000 --- a/src/app_project/tests/test_views.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.test import TestCase -from django.urls import reverse - - -class IndexViewTests(TestCase): - def test_view(self): - response = self.client.get(reverse("app_index")) - self.assertEqual(200, response.status_code) diff --git a/src/app_project/urls.py b/src/app_project/urls.py deleted file mode 100644 index 0752870..0000000 --- a/src/app_project/urls.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.urls import path - -from .views import IndexView - -urlpatterns = [path("", IndexView.as_view(), name="app_index")] diff --git a/src/app_project/views/__init__.py b/src/app_project/views/__init__.py deleted file mode 100644 index 7789c52..0000000 --- a/src/app_project/views/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .index import IndexView - -__all__ = ( - "IndexView", -) diff --git a/src/app_project/views/index.py b/src/app_project/views/index.py deleted file mode 100644 index 6490fec..0000000 --- a/src/app_project/views/index.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.views.generic import TemplateView - - -class IndexView(TemplateView): - template_name = "app_project/index.html" diff --git a/src/app_project/__init__.py b/src/feeds/__init__.py similarity index 100% rename from src/app_project/__init__.py rename to src/feeds/__init__.py diff --git a/src/feeds/admin/__init__.py b/src/feeds/admin/__init__.py new file mode 100644 index 0000000..0da0a10 --- /dev/null +++ b/src/feeds/admin/__init__.py @@ -0,0 +1,17 @@ +from .alias import AliasAdmin +from .article import ArticleAdmin +from .author import AuthorAdmin +from .category import CategoryAdmin +from .feed import FeedAdmin +from .source import SourceAdmin +from .tag import TagAdmin + +__all__ = ( + "AliasAdmin", + "ArticleAdmin", + "AuthorAdmin", + "CategoryAdmin", + "FeedAdmin", + "SourceAdmin", + "TagAdmin", +) diff --git a/src/feeds/admin/alias.py b/src/feeds/admin/alias.py new file mode 100644 index 0000000..202d005 --- /dev/null +++ b/src/feeds/admin/alias.py @@ -0,0 +1,12 @@ +from django.contrib.admin import ModelAdmin, register + +from feeds.models import Alias + + +@register(Alias) +class AliasAdmin(ModelAdmin): + list_display = ("name", "author", "feed") + search_fields = ("name", "author__name", "feed__name") + autocomplete_fields = ("author", "feed") + readonly_fields = ("created", "modified") + fields = ("name", "author", "feed", "created", "modified") diff --git a/src/feeds/admin/article.py b/src/feeds/admin/article.py new file mode 100644 index 0000000..ce69536 --- /dev/null +++ b/src/feeds/admin/article.py @@ -0,0 +1,162 @@ +from urllib.parse import quote + +from django import forms +from django.contrib import admin +from django.db import models +from django.forms import Textarea +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +import tagulous.admin # type: ignore + +from feeds.admin.prettifiers import prettify +from feeds.models import Article + + +def retitle(title): + class Wrapper(admin.FieldListFilter): + def __new__(cls, *args, **kwargs): + instance = admin.FieldListFilter.create(*args, **kwargs) + instance.title = title + return instance + + return Wrapper + + +class IsArchivedFilter(admin.SimpleListFilter): + title = _("Archived") + parameter_name = "archived" + + def lookups(self, request, model_admin): + return ( + ("yes", _("Yes")), + ("no", _("No")), + ) + + def queryset(self, request, queryset): + if self.value() == "yes": + return queryset.exclude(archive_url__exact="") + if self.value() == "no": + return queryset.filter(archive_url__exact="") + + +class IsTaggedFilter(admin.SimpleListFilter): + title = _("Tagged") + parameter_name = "tagged" + + def lookups(self, request, model_admin): + return ( + ("yes", _("Yes")), + ("no", _("No")), + ) + + def queryset(self, request, queryset): + if self.value() == "yes": + return queryset.exclude(tags=None) + if self.value() == "no": + return queryset.filter(tags=None) + + +class ArticleForm(forms.ModelForm): + class Meta: + model = Article + fields = "__all__" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + search_url = "https://archive.md/" + quote(self.instance.url) + message = _( + "The link to the archived article. " + 'Search for an existing copy' + ) + self.fields["archive_url"].help_text = mark_safe(message % search_url) + + +class ArticleAdmin(admin.ModelAdmin): + list_display = ( + "title", + "date", + "source", + "published", + "archived", + "tagged", + ) + + list_filter = ( + ("publish", retitle("Published")), + IsArchivedFilter, + IsTaggedFilter, + "categories", + ) + + search_fields = ("title",) + + ordering = ("-date",) + + readonly_fields = ( + "code", + "pretty_data", + "created", + "modified", + ) + + autocomplete_fields = ("authors", "source", "tags") + + form = ArticleForm + + fields = ( + "title", + "authors", + "url", + "source", + "date", + "tags", + "categories", + "publish", + "code", + "archive_url", + "summary", + "comment", + "data", + "pretty_data", + "created", + "modified", + ) + + formfield_overrides = { + models.TextField: {"widget": Textarea(attrs={"rows": "5", "cols": "80"})}, + models.JSONField: {"widget": Textarea(attrs={"rows": "10", "cols": "80"})}, + } + + @admin.display(boolean=True, description=_("Published")) + def published(self, obj): + return bool(obj.publish) + + @admin.display(boolean=True, description=_("Archived")) + def archived(self, obj): + return bool(obj.archive_url) + + @admin.display(boolean=True, description=_("Tagged")) + def tagged(self, obj): + return bool(obj.tags.all().count()) + + @admin.display(description=_("Formatted")) + def pretty_data(self, instance): + return mark_safe(prettify(instance.data)) + + def get_queryset(self, request): + return ( + super(ArticleAdmin, self) + .get_queryset(request) + .select_related("source") + .prefetch_related("authors", "categories", "tags") + ) + + def lookup_allowed(self, lookup: str, value: str) -> bool: + if lookup in ("tags__slug__in", "source__slug__in", "authors__slug__in"): + return True + return super().lookup_allowed(lookup, value) + + +tagulous.admin.register(Article, ArticleAdmin) diff --git a/src/feeds/admin/author.py b/src/feeds/admin/author.py new file mode 100644 index 0000000..e2486a5 --- /dev/null +++ b/src/feeds/admin/author.py @@ -0,0 +1,85 @@ +from django.contrib import admin, messages +from django.contrib.admin import ModelAdmin, helpers, register +from django.contrib.admin.utils import model_ngettext +from django.db.models import Count, QuerySet +from django.http import HttpRequest +from django.shortcuts import redirect, render +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ + +from feeds.models import Author + + +@register(Author) +class AuthorAdmin(ModelAdmin): + list_display = ("name", "slug", "articles") + search_fields = ("name",) + + fields = ( + "name", + "slug", + "description", + "created", + "modified", + ) + + readonly_fields = ( + "created", + "modified", + "slug", + ) + + actions = ("merge_authors", "show_articles") + + @admin.display(description=_("Articles")) + def articles(self, obj): + return obj.count + + @admin.action(description=_("Merge selected authors")) + def merge_authors(modeladmin, request, queryset): + if request.POST.get("post"): + selected = Author.objects.get(pk=request.POST["selected"]) + + for obj in queryset: + if obj.pk == selected.pk: + continue + for article in obj.articles.all(): + article.authors.remove(obj) + article.authors.add(selected) + obj.delete() + + if count := queryset.count(): + modeladmin.message_user( + request, + _("Successfully merged %(count)d %(items)s.") + % {"count": count, "items": model_ngettext(modeladmin.opts, count)}, + messages.SUCCESS, + ) + return None + + context = { + **modeladmin.admin_site.each_context(request), + "title": _("Merge authors"), + "queryset": queryset, + "opts": modeladmin.model._meta, + "action_checkbox_name": helpers.ACTION_CHECKBOX_NAME, + "media": modeladmin.media, + } + + return render(request, "admin/merge_authors.html", context=context) + + @admin.action(description=_("Show Articles for selected Authors")) + def show_articles(modeladmin, request, queryset): + url = "%s?authors__slug__in=%s" % ( + reverse("admin:feeds_article_changelist"), + ",".join([obj.slug for obj in queryset]), + ) + return redirect(url) + + def get_queryset(self, request: HttpRequest) -> QuerySet: + return ( + super() + .get_queryset(request) + .prefetch_related("articles") + .annotate(count=Count("articles")) + ) diff --git a/src/feeds/admin/category.py b/src/feeds/admin/category.py new file mode 100644 index 0000000..eee0975 --- /dev/null +++ b/src/feeds/admin/category.py @@ -0,0 +1,14 @@ +from django.contrib import admin + +import tagulous # type: ignore + +from feeds.models import Category + + +class CategoryAdmin(admin.ModelAdmin): + + list_display = ("name", "count", "protected") + search_fields = ("name",) + + +tagulous.admin.register(Category, CategoryAdmin) diff --git a/src/feeds/admin/feed.py b/src/feeds/admin/feed.py new file mode 100644 index 0000000..5ca97c7 --- /dev/null +++ b/src/feeds/admin/feed.py @@ -0,0 +1,184 @@ +import logging + +from django import forms +from django.contrib import admin, messages +from django.contrib.admin import ModelAdmin +from django.db import DataError +from django.utils.http import urlencode +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +import tagulous.admin # type: ignore + +from feeds.models import Feed +from feeds.services import feeds + +log = logging.getLogger(__name__) + + +class StatusFilter(admin.SimpleListFilter): + title = _("Status") + parameter_name = "status" + + ranges = { + "1xx": (100, 200), + "2xx": (200, 300), + "3xx": (300, 400), + "4xx": (400, 500), + "5xx": (500, 600), + } + + def lookups(self, request, model_admin): + return ( + ("1xx", _("1xx")), + ("2xx", _("2xx")), + ("3xx", _("3xx")), + ("4xx", _("4xx")), + ("5xx", _("5xx")), + ("null", _("Unknown")), + ) + + def queryset(self, request, queryset): + value = self.value() + if value == "null": + return queryset.filter(status__isnull=True) + elif value in self.ranges: + begin, end = self.ranges[value] + return queryset.filter(status__gte=begin, status__lt=end) + elif value: + return queryset.none() + + +class FailingFilter(admin.SimpleListFilter): + title = _("Failing") + parameter_name = "failing" + + def lookups(self, request, model_admin): + return ( + ("0", _("No")), + ("1", _("Yes")), + ) + + def queryset(self, request, queryset): + value = self.value() + if value == "0": + return queryset.filter(failures=0) + elif value == "1": + return queryset.filter(failures__gt=0) + elif value: + return queryset.none() + + +class FeedForm(forms.ModelForm): + class Meta: + model = Feed + fields = ( + "name", + "source", + "url", + "categories", + "enabled", + "schedule", + "auto_publish", + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if url := self.data.get("url", self.initial.get("url")): + validate_url = "https://validator.w3.org/feed/check.cgi?" + urlencode( + {"url": url} + ) + message = _('Validate the current url') + self.fields["url"].help_text = mark_safe(message % validate_url) + + def clean_schedule(self): + value = self.cleaned_data["schedule"] + + # Remove any extra spaces to reduce the chances that an invalid + # crontab string will be entered. + + value = value.replace(" ", " ") + value = value.replace(" ,", ",") + value = value.replace(", ", ",") + value = value.replace(" -", "-") + value = value.replace("- ", "-") + value = value.replace(" /", "/") + value = value.replace("/ ", "/") + + return value.strip() + + +class FeedAdmin(ModelAdmin): + list_display = ( + "name", + "enabled", + "auto_publish", + "loaded", + "failures", + "status", + ) + + list_filter = ( + "categories", + StatusFilter, + FailingFilter, + "enabled", + "auto_publish", + ) + + search_fields = ("name",) + + readonly_fields = ( + "created", + "modified", + "loaded", + "failures", + "etag", + "last_modified", + "status", + ) + + actions = [ + "load_feed", + ] + + form = FeedForm + + autocomplete_fields = ("source",) + + @admin.action(description=_("Load selected feeds")) + def load_feed(modeladmin, request, queryset): # noqa + for feed in queryset: + try: + # force the feed to be loaded by resetting the fields + # used to send the Last-Modified or ETag headers. + + feed.last_modified = None + feed.etag = None + + # Call load_feed at the module level so this action + # can be tested by mocking the function. + + result = feeds.load_feed(feed) + except DataError: + log.exception("Could not load feed", extra={"feed": feed.name}) + result = False + + if result: + msg = _('The feed "%s" was loaded successfully.' % feed.name) + messages.info(request, msg) + else: + msg = _('There was an error loading the feed "%s"' % feed.name) + messages.error(request, msg) + + def get_queryset(self, request): + return super(FeedAdmin, self).get_queryset(request) + + def get_readonly_fields(self, request, obj=None): + if obj: + return super().get_readonly_fields(request, obj) + return [] + + +tagulous.admin.register(Feed, FeedAdmin) diff --git a/src/feeds/admin/prettifiers.py b/src/feeds/admin/prettifiers.py new file mode 100644 index 0000000..3597a31 --- /dev/null +++ b/src/feeds/admin/prettifiers.py @@ -0,0 +1,65 @@ +import json + +from pygments import highlight # type: ignore +from pygments.formatters.html import HtmlFormatter # type: ignore +from pygments.lexers.data import JsonLexer # type: ignore + +__all__ = [ + "prettify", +] + + +class PrettyEncoder(json.JSONEncoder): + def __init__(self, *args, **kwargs): + super(PrettyEncoder, self).__init__(*args, **kwargs) + self.current_indent = 0 + self.current_indent_str = "" + + def encode(self, o): + # Special Processing for lists + if isinstance(o, (list, tuple)): + primitives_only = True + for item in o: + if isinstance(item, (list, tuple, dict)): + primitives_only = False + break + output = [] + if primitives_only: + for item in o: + output.append(json.dumps(item)) + return "[ " + ", ".join(output) + " ]" + else: + self.current_indent += self.indent + self.current_indent_str = "".join( + [" " for x in range(self.current_indent)] + ) + for item in o: + output.append(self.current_indent_str + self.encode(item)) + self.current_indent -= self.indent + self.current_indent_str = " " * self.current_indent + return "[\n" + ",\n".join(output) + "\n" + self.current_indent_str + "]" + elif isinstance(o, dict): + output = [] + self.current_indent += self.indent + self.current_indent_str = " " * self.current_indent + items = sorted(o.items()) if self.sort_keys else o.items() + for key, value in items: + output.append( + self.current_indent_str + + json.dumps(key) + + ": " + + self.encode(value) + ) + self.current_indent -= self.indent + self.current_indent_str = " " * self.current_indent + return "{\n" + ",\n".join(output) + "\n" + self.current_indent_str + "}" + else: + return json.dumps(o) + + +def prettify(data): + response = json.dumps(data, sort_keys=True, indent=4, cls=PrettyEncoder) + formatter = HtmlFormatter(style="colorful") + response = highlight(response, JsonLexer(), formatter) + style = "
" + return style + response diff --git a/src/feeds/admin/source.py b/src/feeds/admin/source.py new file mode 100644 index 0000000..87241dc --- /dev/null +++ b/src/feeds/admin/source.py @@ -0,0 +1,67 @@ +from django.contrib import admin +from django.contrib.admin import ModelAdmin, register +from django.db import models +from django.forms import Textarea +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from feeds.admin.prettifiers import prettify +from feeds.models import Source + + +@register(Source) +class SourceAdmin(ModelAdmin): + list_display = ( + "name", + "has_url", + "has_description", + ) + + search_fields = ("name",) + + actions = ["show_articles"] + + readonly_fields = ( + "pretty_data", + "created", + "modified", + "slug", + ) + + fields = ( + "name", + "url", + "description", + "data", + "pretty_data", + "created", + "modified", + "slug", + ) + + formfield_overrides = { + models.TextField: {"widget": Textarea(attrs={"rows": "5", "cols": "80"})}, + models.JSONField: {"widget": Textarea(attrs={"rows": "10", "cols": "80"})}, + } + + @admin.display(boolean=True) + def has_url(self, obj): + return bool(obj.url) + + @admin.display(boolean=True) + def has_description(self, obj): + return bool(obj.description) + + @admin.display(description=_("Formatted")) + def pretty_data(self, instance): + return mark_safe(prettify(instance.data)) + + @admin.action(description=_("Show Articles for selected Sources")) + def show_articles(modeladmin, request, queryset): + url = "%s?source__slug__in=%s" % ( + reverse("admin:feeds_article_changelist"), + ",".join([obj.slug for obj in queryset]), + ) + return redirect(url) diff --git a/src/feeds/admin/tag.py b/src/feeds/admin/tag.py new file mode 100644 index 0000000..bfaa21f --- /dev/null +++ b/src/feeds/admin/tag.py @@ -0,0 +1,119 @@ +from django.contrib import admin, messages +from django.contrib.admin import helpers, register +from django.contrib.admin.utils import model_ngettext +from django.db import models +from django.db.models import Count, QuerySet +from django.forms import Textarea +from django.http import HttpRequest +from django.shortcuts import redirect, render +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ + +from feeds.models import Tag + + +class ArticleCountFilter(admin.SimpleListFilter): + title = _("No. of Articles") + parameter_name = "number_of_articles" + + def lookups(self, request, model_admin): + return ( + (0, _("0")), + (1, _("1")), + (2, _("2")), + (3, _("3")), + ) + + def queryset(self, request, queryset): + if value := self.value(): + try: + return queryset.filter(count=int(value)) + except ValueError: + return queryset.none() + return queryset + + +@register(Tag) +class TagAdmin(admin.ModelAdmin): + list_display = ("name", "slug", "has_summary", "articles") + + list_filter = (ArticleCountFilter,) + + search_fields = ("name",) + + fields = ( + "name", + "slug", + "summary", + "description", + "related", + "created", + "modified", + ) + + readonly_fields = ("created", "modified") + + actions = ["merge_tags", "show_articles"] + + autocomplete_fields = ("related",) + + formfield_overrides = { + models.TextField: {"widget": Textarea(attrs={"rows": "5", "cols": "80"})}, + } + + @admin.display(boolean=True, description=_("Has Summary")) + def has_summary(self, obj): + return bool(obj.summary) + + @admin.display(description=_("Articles")) + def articles(self, obj): + return obj.count + + @admin.action(description=_("Merge selected Tags")) + def merge_tags(modeladmin, request, queryset): + if request.POST.get("post"): + selected = Tag.objects.get(pk=request.POST["selected"]) + + for obj in queryset: + if obj.pk == selected.pk: + continue + for article in obj.article_set.all(): + article.tags.remove(obj) + article.tags.add(selected) + obj.delete() + + if count := queryset.count(): + modeladmin.message_user( + request, + _("Successfully merged %(count)d %(items)s.") + % {"count": count, "items": model_ngettext(modeladmin.opts, count)}, + messages.SUCCESS, + ) + return None + + context = { + **modeladmin.admin_site.each_context(request), + "title": _("Merge Tags"), + "queryset": queryset, + "opts": modeladmin.model._meta, + "action_checkbox_name": helpers.ACTION_CHECKBOX_NAME, + "media": modeladmin.media, + } + + return render(request, "admin/merge_tags.html", context=context) + + @admin.action(description=_("Show Articles for selected Tags")) + def show_articles(modeladmin, request, queryset): + url = "%s?tags__slug__in=%s" % ( + reverse("admin:feeds_article_changelist"), + ",".join([obj.slug for obj in queryset]), + ) + return redirect(url) + + def get_queryset(self, request: HttpRequest) -> QuerySet: + return ( + super() + .get_queryset(request) + .prefetch_related("article_set") + .annotate(count=Count("article")) + ) diff --git a/src/app_project/apps.py b/src/feeds/apps.py similarity index 86% rename from src/app_project/apps.py rename to src/feeds/apps.py index e2e05f8..5222e09 100644 --- a/src/app_project/apps.py +++ b/src/feeds/apps.py @@ -12,8 +12,7 @@ def setup_app_settings(): class Config(AppConfig): - name = "app_project" - verbose_name = "App Project" + name = "feeds" def ready(self): setup_app_settings() diff --git a/src/app_project/locale/en/LC_MESSAGES/django.mo b/src/feeds/locale/en/LC_MESSAGES/django.mo similarity index 100% rename from src/app_project/locale/en/LC_MESSAGES/django.mo rename to src/feeds/locale/en/LC_MESSAGES/django.mo diff --git a/src/app_project/locale/en/LC_MESSAGES/django.po b/src/feeds/locale/en/LC_MESSAGES/django.po similarity index 100% rename from src/app_project/locale/en/LC_MESSAGES/django.po rename to src/feeds/locale/en/LC_MESSAGES/django.po diff --git a/src/feeds/migrations/0001_initial.py b/src/feeds/migrations/0001_initial.py new file mode 100644 index 0000000..8c2777d --- /dev/null +++ b/src/feeds/migrations/0001_initial.py @@ -0,0 +1,655 @@ +# Generated by Django 4.1.7 on 2023-03-13 19:46 + +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields +import feeds.models.article +import feeds.models.feed +import tagulous.models.fields +import tagulous.models.models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Author", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "name", + models.CharField( + help_text="The name of the author", + max_length=100, + verbose_name="Name", + ), + ), + ( + "slug", + django_extensions.db.fields.AutoSlugField( + blank=True, + editable=False, + help_text="The slug uniquely identifying the author", + max_length=100, + populate_from="name", + unique=True, + verbose_name="Slug", + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="A short profile of the author", + verbose_name="Description", + ), + ), + ], + options={ + "verbose_name": "Author", + "verbose_name_plural": "Authors", + }, + ), + migrations.CreateModel( + name="Category", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255, unique=True)), + ("slug", models.SlugField()), + ( + "count", + models.IntegerField( + default=0, + help_text="Internal counter of how many times this tag is in use", + ), + ), + ( + "protected", + models.BooleanField( + default=False, + help_text="Will not be deleted when the count reaches 0", + ), + ), + ("path", models.TextField()), + ( + "label", + models.CharField( + help_text="The name of the tag, without ancestors", + max_length=255, + ), + ), + ( + "level", + models.IntegerField( + default=1, help_text="The level of the tag in the tree" + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="feeds.category", + ), + ), + ], + options={ + "verbose_name": "Category", + "verbose_name_plural": "Categories", + }, + bases=(tagulous.models.models.BaseTagTreeModel, models.Model), + ), + migrations.CreateModel( + name="Source", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "name", + models.CharField( + help_text="The name of the source", + max_length=100, + verbose_name="Name", + ), + ), + ( + "slug", + django_extensions.db.fields.AutoSlugField( + blank=True, + editable=False, + help_text="The slug uniquely identifying the source", + max_length=100, + populate_from="name", + unique=True, + verbose_name="Slug", + ), + ), + ( + "url", + models.URLField( + blank=True, + help_text="The URL for source's web site", + verbose_name="Web site", + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="A description of the source", + verbose_name="Description", + ), + ), + ( + "data", + models.JSONField( + blank=True, + default=dict, + help_text="Data describing a source", + verbose_name="Data", + ), + ), + ], + options={ + "verbose_name": "Source", + "verbose_name_plural": "Sources", + }, + ), + migrations.CreateModel( + name="Tag", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "name", + models.CharField( + default="", + help_text="The name of the tag", + max_length=100, + verbose_name="Name", + ), + ), + ( + "slug", + django_extensions.db.fields.AutoSlugField( + blank=True, + editable=False, + help_text="The slug uniquely identifying the tag", + max_length=100, + populate_from="name", + unique=True, + verbose_name="Slug", + ), + ), + ( + "summary", + models.TextField( + blank=True, + help_text="A short summary of what the tag covers", + verbose_name="Summary", + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="A more detailed description of the tag. May be in HTML", + verbose_name="Description", + ), + ), + ( + "related", + models.ManyToManyField( + blank=True, + help_text="The set of tag(s) that are related to this one", + to="feeds.tag", + verbose_name="Related tags", + ), + ), + ], + options={ + "verbose_name": "Tag", + "verbose_name_plural": "Tags", + }, + ), + migrations.CreateModel( + name="Feed", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "name", + models.CharField( + help_text="The name of the feed", + max_length=100, + verbose_name="Name", + ), + ), + ( + "url", + models.URLField( + help_text="The URL for the RSS feed (RSS or Atom)", + verbose_name="URL", + ), + ), + ( + "enabled", + models.BooleanField( + default=False, + help_text="Enable loading of the RSS feed", + verbose_name="Enabled", + ), + ), + ( + "schedule", + models.CharField( + blank=True, + help_text="Crontab entry which describe the times the feed will be downloaded.Leave blank to use the default schedule defined for all feeds.", + max_length=100, + validators=[feeds.models.feed.validate_crontab], + verbose_name="Schedule", + ), + ), + ( + "auto_publish", + models.BooleanField( + default=False, + help_text="Automatically publish Articles when added from an RSS Feed", + verbose_name="Auto Publish", + ), + ), + ( + "loaded", + models.DateTimeField( + blank=True, + help_text="The date that the RSS feed was last loaded", + null=True, + verbose_name="Loaded", + ), + ), + ( + "failures", + models.IntegerField( + default=0, + help_text="The number of consecutive times a feed has failed to load.", + verbose_name="Failures", + ), + ), + ( + "status", + models.IntegerField( + blank=True, + help_text="The HTTP status code from the last time the feed was fetched", + null=True, + verbose_name="Status", + ), + ), + ( + "etag", + models.CharField( + blank=True, + help_text="The Etag header from the RSS feed (Atom feeds only)", + max_length=100, + null=True, + verbose_name="ETag", + ), + ), + ( + "last_modified", + models.DateTimeField( + blank=True, + help_text="The last-modified header from the RSS feed", + null=True, + verbose_name="Last Modified", + ), + ), + ( + "categories", + tagulous.models.fields.TagField( + _set_tag_meta=True, + blank=True, + force_lowercase=True, + help_text="The categories of articles published by the feed", + to="feeds.category", + tree=True, + verbose_name="Categories", + ), + ), + ( + "source", + models.ForeignKey( + help_text="The web site which hosts the feed", + on_delete=django.db.models.deletion.PROTECT, + to="feeds.source", + verbose_name="Source", + ), + ), + ], + options={ + "verbose_name": "Feed", + "verbose_name_plural": "Feeds", + }, + ), + migrations.CreateModel( + name="Article", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "title", + models.CharField( + help_text="The title of the article", + max_length=1000, + verbose_name="Title", + ), + ), + ( + "url", + models.URLField( + help_text="The link to the article on the source's web site", + max_length=2000, + verbose_name="URL", + ), + ), + ( + "code", + models.CharField( + default=feeds.models.article.article_code, + help_text="The code that used to identify the article", + max_length=6, + verbose_name="Code", + ), + ), + ( + "archive_url", + models.URLField( + blank=True, + help_text="The link to the archived article using a service such as archive.today", + max_length=2000, + verbose_name="Archive URL", + ), + ), + ( + "date", + models.DateTimeField( + help_text="The date the article was published by the source", + verbose_name="Date", + ), + ), + ( + "summary", + models.TextField( + blank=True, + help_text="A summary of the article", + verbose_name="Summary", + ), + ), + ( + "identifier", + models.CharField( + blank=True, + help_text="The unique identifier for the Article from the Feed", + max_length=2000, + verbose_name="Identifier", + ), + ), + ( + "comment", + models.TextField( + blank=True, + help_text="An editorial comment about the article", + verbose_name="Comment", + ), + ), + ( + "publish", + models.BooleanField( + db_index=True, + default=False, + help_text="Publish the article on the site", + verbose_name="Publish", + ), + ), + ( + "views", + models.PositiveIntegerField( + default=0, + help_text="The number of times the article has been viewed (read)", + verbose_name="Views", + ), + ), + ( + "data", + models.JSONField( + blank=True, + default=dict, + help_text="Data describing an article", + verbose_name="Data", + ), + ), + ( + "authors", + models.ManyToManyField( + help_text="The author(s) who wrote the article", + related_name="articles", + to="feeds.author", + verbose_name="Authors", + ), + ), + ( + "categories", + tagulous.models.fields.TagField( + _set_tag_meta=True, + blank=True, + help_text="The categories that describes the article contents", + to="feeds.category", + tree=True, + verbose_name="Categories", + ), + ), + ( + "feed", + models.ForeignKey( + blank=True, + help_text="The feed from the host web site", + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="articles", + to="feeds.feed", + verbose_name="RSS Feed", + ), + ), + ( + "source", + models.ForeignKey( + help_text="The source (blog, news site, etc.) which published the article", + on_delete=django.db.models.deletion.CASCADE, + related_name="articles", + to="feeds.source", + verbose_name="Source", + ), + ), + ( + "tags", + models.ManyToManyField( + blank=True, + help_text="Keywords for subjects covered in the content", + to="feeds.tag", + verbose_name="Tags", + ), + ), + ], + options={ + "verbose_name": "Article", + "verbose_name_plural": "Articles", + "get_latest_by": "date", + }, + ), + migrations.CreateModel( + name="Alias", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "name", + models.CharField( + blank=True, + help_text="A name used to lookup an author from a feed entry", + max_length=100, + verbose_name="Name", + ), + ), + ( + "author", + models.ForeignKey( + help_text="The Author that the feed entry author maps to", + on_delete=django.db.models.deletion.CASCADE, + related_name="aliases", + to="feeds.author", + verbose_name="Author", + ), + ), + ( + "feed", + models.ForeignKey( + help_text="The feed where the alias is used", + on_delete=django.db.models.deletion.CASCADE, + to="feeds.feed", + verbose_name="Feed", + ), + ), + ], + options={ + "verbose_name": "Alias", + "verbose_name_plural": "Aliases", + }, + ), + ] diff --git a/demo/conf/__init__.py b/src/feeds/migrations/__init__.py similarity index 100% rename from demo/conf/__init__.py rename to src/feeds/migrations/__init__.py diff --git a/src/feeds/models/__init__.py b/src/feeds/models/__init__.py new file mode 100644 index 0000000..7ef2698 --- /dev/null +++ b/src/feeds/models/__init__.py @@ -0,0 +1,17 @@ +from .alias import Alias +from .article import Article +from .author import Author +from .category import Category +from .feed import Feed +from .source import Source +from .tag import Tag + +__all__ = ( + "Alias", + "Article", + "Author", + "Category", + "Feed", + "Source", + "Tag", +) diff --git a/src/feeds/models/alias.py b/src/feeds/models/alias.py new file mode 100644 index 0000000..e90412d --- /dev/null +++ b/src/feeds/models/alias.py @@ -0,0 +1,35 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from django_extensions.db.models import TimeStampedModel + + +class Alias(TimeStampedModel): + class Meta: + verbose_name = _("Alias") + verbose_name_plural = _("Aliases") + + name = models.CharField( + verbose_name=_("Name"), + help_text=_("A name used to lookup an author from a feed entry"), + max_length=100, + blank=True, + ) + + author = models.ForeignKey( + to="feeds.Author", + related_name="aliases", + on_delete=models.CASCADE, + verbose_name=_("Author"), + help_text=_("The Author that the feed entry author maps to"), + ) + + feed = models.ForeignKey( + to="feeds.Feed", + on_delete=models.CASCADE, + verbose_name=_("Feed"), + help_text=_("The feed where the alias is used"), + ) + + def __str__(self): + return self.name diff --git a/src/feeds/models/article.py b/src/feeds/models/article.py new file mode 100644 index 0000000..e6f64e1 --- /dev/null +++ b/src/feeds/models/article.py @@ -0,0 +1,176 @@ +import datetime as dt + +from django.db import models +from django.db.models import F +from django.utils.crypto import get_random_string +from django.utils.translation import gettext_lazy as _ + +from django_extensions.db.models import TimeStampedModel +from tagulous.models import TagField + + +def article_code(): + return get_random_string(6) + + +class ArticleQuerySet(models.QuerySet): + def published(self) -> "ArticleQuerySet": + return self.filter(publish=True) + + def for_date(self, date: dt.date) -> "ArticleQuerySet": + return self.filter(date__date=date) + + def since_date(self, date: dt.date) -> "ArticleQuerySet": + return self.filter(date__date__gt=date) + + def by_author(self, slug) -> "ArticleQuerySet": + return self.filter(authors__slug=slug) + + def by_source(self, slug) -> "ArticleQuerySet": + return self.filter(source__slug=slug) + + def by_tag(self, slug) -> "ArticleQuerySet": + return self.filter(tags__slug=slug) + + def with_code(self, code) -> "Article": + return self.filter(code=code).latest("created") + + def with_identifier(self, identifier) -> "ArticleQuerySet": + return self.filter(identifier=identifier) + + def viewed(self, code) -> None: + self.filter(code=code).update(count=F("views") + 1) + + +ArticleManager = models.Manager.from_queryset(ArticleQuerySet) # type: ignore + + +class Article(TimeStampedModel): + class Meta: + verbose_name = _("Article") + verbose_name_plural = _("Articles") + get_latest_by = "date" + + title = models.CharField( + verbose_name=_("Title"), + help_text=_("The title of the article"), + max_length=1000, + ) + + authors = models.ManyToManyField( + to="feeds.Author", + related_name="articles", + verbose_name=_("Authors"), + help_text=_("The author(s) who wrote the article"), + ) + + url = models.URLField( + verbose_name=_("URL"), + help_text=_("The link to the article on the source's web site"), + max_length=2000, + ) + + code = models.CharField( + verbose_name=_("Code"), + help_text=_("The code that used to identify the article"), + max_length=6, + default=article_code, + ) + + archive_url = models.URLField( + verbose_name=_("Archive URL"), + help_text=_( + "The link to the archived article using a service such as archive.today" + ), + max_length=2000, + blank=True, + ) + + date = models.DateTimeField( + verbose_name=_("Date"), + help_text=_("The date the article was published by the source"), + ) + + summary = models.TextField( + verbose_name=_("Summary"), + help_text=_("A summary of the article"), + blank=True, + ) + + source = models.ForeignKey( + to="feeds.Source", + related_name="articles", + on_delete=models.CASCADE, + verbose_name=_("Source"), + help_text=_("The source (blog, news site, etc.) which published the article"), + ) + + feed = models.ForeignKey( + to="feeds.Feed", + related_name="articles", + on_delete=models.PROTECT, + verbose_name=_("RSS Feed"), + help_text=_("The feed from the host web site"), + null=True, + blank=True, + ) + + identifier = models.CharField( + verbose_name=_("Identifier"), + help_text=_("The unique identifier for the Article from the Feed"), + max_length=2000, + blank=True, + ) + + comment = models.TextField( + verbose_name=_("Comment"), + help_text=_("An editorial comment about the article"), + blank=True, + ) + + publish = models.BooleanField( + verbose_name=_("Publish"), + help_text=_("Publish the article on the site"), + default=False, + db_index=True, + ) + + categories = TagField( + verbose_name=_("Categories"), + help_text=_("The categories that describes the article contents"), + to="feeds.Category", + blank=True, + ) + + # Tags are added after the article is added to the database. + # The idea is that articles on the same subjects from different + # sources have the same set of Tags - more or less. + + tags = models.ManyToManyField( + verbose_name=_("Tags"), + help_text=_("Keywords for subjects covered in the content"), + to="feeds.Tag", + blank=True, + ) + + views = models.PositiveIntegerField( + verbose_name=_("Views"), + help_text="The number of times the article has been viewed (read)", + default=0, + ) + + data = models.JSONField( + verbose_name=_("Data"), + help_text=_("Data describing an article"), + default=dict, + blank=True, + ) + + objects = ArticleManager() # type: ignore + + def __str__(self) -> str: + return self.title + + def has_authors(self): + names = [author.name for author in self.authors.all()] + return names and " ".join(names) != self.source.name diff --git a/src/feeds/models/author.py b/src/feeds/models/author.py new file mode 100644 index 0000000..f2c93a2 --- /dev/null +++ b/src/feeds/models/author.py @@ -0,0 +1,47 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from django_extensions.db.fields import AutoSlugField +from django_extensions.db.models import TimeStampedModel + + +class AuthorQuerySet(models.QuerySet): + def published(self) -> "AuthorQuerySet": + return self.filter(articles__publish=True).distinct() + + def with_slug(self, slug) -> "AuthorQuerySet": + return self.filter(slug=slug) + + +AuthorManager = models.Manager.from_queryset(AuthorQuerySet) # type: ignore + + +class Author(TimeStampedModel): # type: ignore + class Meta: + verbose_name = _("Author") + verbose_name_plural = _("Authors") + + name = models.CharField( + verbose_name=_("Name"), + help_text=_("The name of the author"), + max_length=100, + ) + + slug = AutoSlugField( + verbose_name=_("Slug"), + help_text=_("The slug uniquely identifying the author"), + populate_from="name", + max_length=100, + unique=True, + ) + + description = models.TextField( + verbose_name=_("Description"), + help_text=_("A short profile of the author"), + blank=True, + ) + + objects = AuthorManager() # type: ignore + + def __str__(self): + return self.name diff --git a/src/feeds/models/category.py b/src/feeds/models/category.py new file mode 100644 index 0000000..ef96509 --- /dev/null +++ b/src/feeds/models/category.py @@ -0,0 +1,12 @@ +from django.utils.translation import gettext_lazy as _ + +import tagulous # type: ignore + + +class Category(tagulous.models.TagTreeModel): + class Meta: + verbose_name = _("Category") + verbose_name_plural = _("Categories") + + class TagMeta: + force_lowercase = True diff --git a/src/feeds/models/feed.py b/src/feeds/models/feed.py new file mode 100644 index 0000000..db738d5 --- /dev/null +++ b/src/feeds/models/feed.py @@ -0,0 +1,142 @@ +import datetime as dt +from typing import List + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.translation import gettext_lazy as _ + +import tagulous # type: ignore +from croniter import croniter +from django_extensions.db.models import TimeStampedModel + + +def validate_crontab(value): + if not croniter.is_valid(value): + raise ValidationError( + _('"%(value)s" is not a valid crontab entry'), + params={"value": value}, + ) + + minutes = value.split()[0] + + if minutes != "0" and minutes != "*": + raise ValidationError( + _( + "The celery task to load feeds runs every hour on the hour" + "so the minutes field (the first) must be set to '0' or '*'" + ) + ) + + +class FeedQuerySet(models.QuerySet): + def disabled(self) -> "FeedQuerySet": + return self.filter(enabled=False) + + def enabled(self) -> "FeedQuerySet": + return self.filter(enabled=True) + + def scheduled_for(self, timestamp: dt.datetime) -> List["Feed"]: + feeds: List["Feed"] = [] + for feed in Feed.objects.enabled(): + schedule: str = feed.schedule or settings.FEEDS_LOAD_SCHEDULE + if croniter.match(schedule, timestamp): + feeds.append(feed) + return feeds + + +FeedManager = models.Manager.from_queryset(FeedQuerySet) # type: ignore + + +class Feed(TimeStampedModel): # type: ignore + class Meta: + verbose_name = _("Feed") + verbose_name_plural = _("Feeds") + + name = models.CharField( + verbose_name=_("Name"), + help_text=_("The name of the feed"), + max_length=100, + ) + + source = models.ForeignKey( + to="feeds.Source", + on_delete=models.PROTECT, + verbose_name=_("Source"), + help_text=_("The web site which hosts the feed"), + ) + + url = models.URLField( + verbose_name=_("URL"), + help_text=_("The URL for the RSS feed (RSS or Atom)"), + ) + + categories = tagulous.models.TagField( + verbose_name=_("Categories"), + help_text=_("The categories of articles published by the feed"), + to="feeds.Category", + blank=True, + ) + + enabled = models.BooleanField( + verbose_name=_("Enabled"), + help_text=_("Enable loading of the RSS feed"), + default=False, + ) + + schedule = models.CharField( + validators=[validate_crontab], + verbose_name=_("Schedule"), + help_text=_( + "Crontab entry which describe the times the feed will be downloaded." + "Leave blank to use the default schedule defined for all feeds." + ), + max_length=100, + blank=True, + ) + + auto_publish = models.BooleanField( + verbose_name=_("Auto Publish"), + help_text=_("Automatically publish Articles when added from an RSS Feed"), + default=False, + ) + + loaded = models.DateTimeField( + verbose_name=_("Loaded"), + help_text=_("The date that the RSS feed was last loaded"), + null=True, + blank=True, + ) + + failures = models.IntegerField( + verbose_name=_("Failures"), + help_text=_("The number of consecutive times a feed has failed to load."), + default=0, + ) + + status = models.IntegerField( + verbose_name=_("Status"), + help_text=_("The HTTP status code from the last time the feed was fetched"), + null=True, + blank=True, + ) + + etag = models.CharField( + verbose_name=_("ETag"), + help_text=_("The Etag header from the RSS feed (Atom feeds only)"), + max_length=100, + null=True, + blank=True, + ) + + last_modified = models.DateTimeField( + verbose_name=_("Last Modified"), + help_text=_("The last-modified header from the RSS feed"), + null=True, + blank=True, + ) + + objects = FeedManager() # type: ignore + + def __str__(self): + return self.name diff --git a/src/feeds/models/source.py b/src/feeds/models/source.py new file mode 100644 index 0000000..ab82c99 --- /dev/null +++ b/src/feeds/models/source.py @@ -0,0 +1,47 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from django_extensions.db.fields import AutoSlugField +from django_extensions.db.models import TimeStampedModel + + +class Source(TimeStampedModel): # type: ignore + class Meta: + verbose_name = _("Source") + verbose_name_plural = _("Sources") + + name = models.CharField( + verbose_name=_("Name"), + help_text=_("The name of the source"), + max_length=100, + ) + + slug = AutoSlugField( + verbose_name=_("Slug"), + help_text=_("The slug uniquely identifying the source"), + populate_from="name", + max_length=100, + unique=True, + ) + + url = models.URLField( + verbose_name=_("Web site"), + help_text=_("The URL for source's web site"), + blank=True, + ) + + description = models.TextField( + verbose_name=_("Description"), + help_text=_("A description of the source"), + blank=True, + ) + + data = models.JSONField( + verbose_name=_("Data"), + help_text=_("Data describing a source"), + default=dict, + blank=True, + ) + + def __str__(self): + return self.name diff --git a/src/feeds/models/tag.py b/src/feeds/models/tag.py new file mode 100644 index 0000000..5d70e6e --- /dev/null +++ b/src/feeds/models/tag.py @@ -0,0 +1,81 @@ +from typing import Dict, List + +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from django_extensions.db.fields import AutoSlugField +from django_extensions.db.models import TimeStampedModel + + +class TagQuerySet(models.QuerySet): + def for_articles(self, articles) -> "TagQuerySet": + return self.filter(article__in=articles) + + def weighted(self) -> "List[Tag]": + results: Dict[Tag, int] = {} + + for tag in self: + results.setdefault(tag, 0) + results[tag] += 1 + + counts = results.values() + + if counts: + highest = max(counts) + lowest = min(counts) + delta = highest - lowest + 1 + categories = 6 + interval = delta / categories + + for tag, count in results.items(): + tag.weight = int((count - lowest) / interval) + 1 + + return list(results.keys()) + + +TagManager = models.Manager.from_queryset(TagQuerySet) # type: ignore + + +class Tag(TimeStampedModel): # type: ignore + class Meta: + verbose_name = _("Tag") + verbose_name_plural = _("Tags") + + name = models.CharField( + verbose_name=_("Name"), + help_text=_("The name of the tag"), + max_length=100, + default="", + ) + + slug = AutoSlugField( + verbose_name=_("Slug"), + help_text=_("The slug uniquely identifying the tag"), + populate_from="name", + max_length=100, + unique=True, + ) + + summary = models.TextField( + verbose_name=_("Summary"), + help_text=_("A short summary of what the tag covers"), + blank=True, + ) + + description = models.TextField( + verbose_name=_("Description"), + help_text=_("A more detailed description of the tag. May be in HTML"), + blank=True, + ) + + related = models.ManyToManyField( + to="self", + verbose_name=_("Related tags"), + help_text=_("The set of tag(s) that are related to this one"), + blank=True, + ) + + objects = TagManager() # type: ignore + + def __str__(self): + return self.name diff --git a/src/app_project/migrations/__init__.py b/src/feeds/services/__init__.py similarity index 100% rename from src/app_project/migrations/__init__.py rename to src/feeds/services/__init__.py diff --git a/src/feeds/services/feeds.py b/src/feeds/services/feeds.py new file mode 100644 index 0000000..02f14de --- /dev/null +++ b/src/feeds/services/feeds.py @@ -0,0 +1,327 @@ +import logging +import re +from datetime import datetime +from html import unescape +from typing import List, Optional +from urllib.error import HTTPError, URLError + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.core.validators import URLValidator +from django.utils import timezone +from django.utils.text import slugify + +import feedparser # type: ignore +from dateutil.parser import parse as parse_date +from feedparser import FeedParserDict + +from feeds.models import Alias, Article, Author, Feed + +__all__ = ("load_feed", "get_user_agent") + + +validate_url = URLValidator() + + +def load_feeds() -> None: + now = timezone.now() + feeds = Feed.objects.scheduled_for(now) + count = 0 + + log = logging.getLogger(f"{__name__}.load_feeds") + log.debug("Feeds scheduled", extra={"num_feeds": len(feeds)}) + + for feed in feeds: + if load_feed(feed): + count += 1 + + log.debug("Feeds loaded", extra={"num_feeds": count}) + + +def load_feed(feed: Feed) -> bool: + # The Last-Modified and ETag headers, from the previous fetch, are + # sent when fetching a feed, so we only load entries if the feed + # has been updated. + + # Since we do conditional fetches, the feeds can be updated frequently. + # That lets avoid worrying about retries for network and remote + # server errors since it will probably have been resolved by the + # next fetch. + + log = logging.getLogger(f"{__name__}.load_feed") + + url = feed.url + etag = feed.etag + modified = feed.last_modified + + log.debug( + "Feed request", + extra={"feed": feed.name, "url": url, "last_modified": modified, "etag": etag}, + ) + + feed.loaded = timezone.now() + + try: + response = feedparser.parse( + get_source(feed), agent=get_user_agent(), modified=modified, etag=etag + ) + except (HTTPError, URLError): + log.exception("Feed not fetched", extra={"feed": feed.name}) + feed.failures += 1 + feed.save() + return False + + # The status field is not present in the response when a feed + # is loaded from a string or a file. + + feed.status = response.get("status") + + value = response.headers.get("last-modified") + modified = parse_date(value) if value else None + etag = response.headers.get("etag") + content_length = response.headers.get("content-length") + num_entries = len(response.entries) + + log.debug( + "Feed response", + extra={ + "feed": feed.name, + "status": feed.status, + "last_modified": modified, + "etag": etag, + "content_length": content_length, + }, + ) + + # Strictly speaking, if the feed is unchanged then no XML was returned, + # so we don't know if the feed is valid since nothing was parsed. However, + # setting the valid flag to None implies some form of error occurred when + # in fact the feed is operating normally. + + if feed.status == 304: + log.debug("Feed is unchanged", extra={"feed": feed.name}) + feed.failures = 0 + feed.save() + return False + + # Assume anything other than a 200 OK status at this point means that the + # feed XML was not returned. This avoids confusingly reporting a parse + # error if an HTML error page was returned, for example, if the feed url + # changed. Note that the status will be None when running tests. + + if feed.status and feed.status != 200: + log.error( + "Feed not loaded", + extra={ + "feed": feed.name, + "status": feed.status, + "num_entries": num_entries, + }, + ) + feed.failures += 1 + feed.save() + return False + + # One source of feed parsing failures is non-printing characters, which + # presumably were the copy and pasted into the post when it was created. + # We don't want abandon the load at this point as it might be something + # that the feed author can correct and since we're having trouble other + # people are as well, so it's worth contacting them over this. + + if response.bozo: + exc = response.get("bozo_exception", None) + log.error("Feed not parsed", exc_info=exc) + feed.failures += 1 + feed.save() + return False + + feed.etag = etag + feed.last_modified = modified + feed.failures = 0 + feed.save() + + num_created = 0 + + for item in response.entries: + values = { + "identifier": get_identifier(item), + "title": get_title(item), + "url": get_url(item), + "date": get_published(item), + "summary": get_summary(item), + "tags": get_tags(item), + } + + # Simply skip over any items that are "badly formed". This is + # preferable to simply letting exceptions be raised when saving + # an Article as "badly formed" feed are not that unusual, so it's + # better to ignore any missing data as it's likely to be present + # at a later time, for example, an author forgets to add a title + # to the article and publishes the post. + + required = ["identifier", "title", "url", "date"] + + if not all([v for k, v in values.items() if k in required]): + continue + + article: Article = get_article(str(values.pop("identifier"))) + + article.title = values["title"] # type: ignore + article.url = values["url"] # type: ignore + article.date = values["date"] # type: ignore + article.summary = values["summary"] # type: ignore + article.data["feed:tags"] = values["tags"] + + if article.pk is None: + num_created += 1 + article.feed = feed + article.source = feed.source + article.publish = feed.auto_publish + + article.save() + + for category in feed.categories.all(): + article.categories.add(category) + + for author in get_authors(item, feed): + article.authors.add(author) + + # Note the slight change in wording compared to when the feed status + # was 304, see above. That makes it easier to tell what happened simply + # by reading the message and not looking at the other values logged. + + message = "Feed was unchanged" if num_created == 0 else "Feed was loaded" + + log.debug( + message, + extra={ + "feed": feed.name, + "num_entries": num_entries, + "num_created": num_created, + }, + ) + + return True + + +def get_source(feed: Feed) -> str: + # Feedparse can parse a file, url or string so this method exists to + # allow an XML string to be injected for testing. + return feed.url + + +def get_user_agent() -> str: + return settings.FEEDS_USER_AGENT + + +def get_article(identifier: str) -> Article: + if Article.objects.with_identifier(identifier).exists(): + article = Article.objects.with_identifier(identifier).get() + else: + article = Article(identifier=identifier) + return article + + +def get_identifier(item: FeedParserDict) -> str: + # Use the entry link as a last resort as the feed would likely fail + # validation. This happens surprisingly often. + return item.get("id", item.get("link", "")) + + +def get_url(item: FeedParserDict) -> str: + try: + if link := item.get("link", ""): + validate_url(link) + except ValidationError: + link = "" + + return link + + +def get_title(item: FeedParserDict) -> str: + # Some feeds, e.g. cartoons, might just contain an image so there is no title + return normalize_title(unescape(item.get("title", ""))) + + +def get_published(item: FeedParserDict) -> datetime: + if date := item.get("published", None): + date = parse_date(date) + return date + + +def get_updated(item: FeedParserDict) -> datetime: + if date := item.get("updated", None): + date = parse_date(date) + return date + + +def get_authors(item: FeedParserDict, feed: Feed) -> List[Author]: + authors: List[Author] = [] + + names = [ + author.get("name") # noqa + for author in item.get("authors", []) + if author.get("name") # noqa + ] + + for name in names: + author: Optional[Author] + alias: Optional[Alias] = Alias.objects.filter(name=name, feed=feed).first() + + if alias: + author = alias.author + else: + slug: str = slugify(name) + + try: + # Check if the Author exists using the slug. That way case changes + # in the Author's name do not result in multiple authors. + author, created = Author.objects.get_or_create( + slug=slug, defaults={"name": name} + ) + except Author.MultipleObjectsReturned: + log = logging.getLogger(f"{__name__}.load_feed") + log.exception("Multiple Authors found", extra={"author": name}) + author = Author.objects.with_slug(slug).first() + + if author: + authors.append(author) + + return authors + + +def get_summary(item: FeedParserDict) -> str: + return unescape(item.get("summary", "")) + + +def get_tags(item: FeedParserDict) -> List[str]: + return sorted([tag.get("term") for tag in item.get("tags", []) if tag.get("term")]) + + +def normalize_title(value: str) -> str: + # Skip changes if the title is empty + if not value: + return value + # Strip double-quotes around a title + if value[0] == value[-1] == '"': + value = value[1:-1] + # Strip single-quotes around a title + if value[0] == value[-1] == "'": + value = value[1:-1] + # Remove any trailing periods but leave an ellipsis alone + if value[-1] == "." and value[-2] != ".": + value = value[:-1] + # Remove extra whitespace + value = " ".join(value.split()) + # Put a space before and after a hyphen + value = re.sub(r"(\w(\")?)-", r"\1 -", value) + value = re.sub(r"-((\")?(\w))", r"- \1", value) + # Put a space after a comma + value = re.sub(r"(\w),(\w)", r"\1, \2", value) + # Remove a space before a comma + value = re.sub(r"(\w) ,", r"\1,", value) + # Put a space before an opening bracket + value = re.sub(r"(\w)\(", r"\1 (", value) + # Put a space after a closing bracket + value = re.sub(r"\)(\w)", r") \1", value) + return value diff --git a/src/feeds/settings.py b/src/feeds/settings.py new file mode 100644 index 0000000..957ceba --- /dev/null +++ b/src/feeds/settings.py @@ -0,0 +1,55 @@ +""" +Django settings for django-feeds. + +""" +import os + +from django.core.exceptions import ImproperlyConfigured + +from croniter import croniter + +# ######### +# FEEDS +# ######### + +# Feeds are loaded using a celery task. You can schedule how often this task +# runs using the FEEDS_TASK_SCHEDULE setting which defaults to run every hour +# on the hour. Rather than fetch every feed each time you can set a schedule +# on individual feeds to reduce server load and bandwidth. If no schedule is +# set then the FEEDS_LOAD_SCHEDULE setting is used. When the celery task runs +# it compares the feed's schedule with the current time and if they match, +# the feed is loaded. +# +# IMPORTANT: the times represented by the FEEDS_TASK_SCHEDULE setting and the +# FEEDS_LOAD_SCHEDULE setting must coincide otherwise the feed will never be +# loaded. Since the celery task runs on the hour, you can add a little margin +# for error by setting the minute field for the feed schedule to "*" rather +# than "0". That way, if the task gets blocked temporarily as long as it gets +# executed in the next 59 minutes, the schedules will match and the feed will +# still be loaded. + +FEEDS_TASK_SCHEDULE = os.environ.get("FEEDS_TASK_SCHEDULE", "0 * * * *") + +if not croniter.is_valid(FEEDS_TASK_SCHEDULE): + raise ImproperlyConfigured("FEEDS_TASK_SCHEDULE setting is not a valid cron entry") + +FEEDS_LOAD_SCHEDULE = os.environ.get("FEEDS_LOAD_SCHEDULE", FEEDS_TASK_SCHEDULE) + +if not croniter.is_valid(FEEDS_LOAD_SCHEDULE): + raise ImproperlyConfigured("FEEDS_LOAD_SCHEDULE setting is not a valid cron entry") + +# A default user-agent string that is used when loading RSS feeds. Some sites +# will return an error is the user-agent is not given. + +FEEDS_USER_AGENT = os.environ.get("FEEDS_USER_AGENT", "Django Feeds") + +# ############ +# Tagulous +# ############ + +SERIALIZATION_MODULES = { + "xml": "tagulous.serializers.xml_serializer", + "json": "tagulous.serializers.json", + "python": "tagulous.serializers.python", + "yaml": "tagulous.serializers.pyyaml", +} diff --git a/src/feeds/templates/admin/merge_authors.html b/src/feeds/templates/admin/merge_authors.html new file mode 100644 index 0000000..81e1334 --- /dev/null +++ b/src/feeds/templates/admin/merge_authors.html @@ -0,0 +1,52 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n admin_urls static %} + +{% block extrahead %} + {{ block.super }} + {{ media }} + +{% endblock %} + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %} + +{% block breadcrumbs %} +
+{% translate 'Home' %} +› {{ opts.app_config.verbose_name }} +› {{ opts.verbose_name_plural|capfirst }} +› {% translate 'Merge' %} +
+{% endblock %} + +{% block content %} +

+ {% translate "Selected authors" %} +

+ +
{% csrf_token %} +
+
    + {% for obj in queryset %} +
  • + {{ obj.name }} + +
    • + {% endfor %} +
+ +
+ + +
+ + + + + {% translate "No, take me back" %} +
+
+{% endblock %} diff --git a/src/feeds/templates/admin/merge_tags.html b/src/feeds/templates/admin/merge_tags.html new file mode 100644 index 0000000..a4da2b6 --- /dev/null +++ b/src/feeds/templates/admin/merge_tags.html @@ -0,0 +1,52 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n admin_urls static %} + +{% block extrahead %} + {{ block.super }} + {{ media }} + +{% endblock %} + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %} + +{% block breadcrumbs %} +
+{% translate 'Home' %} +› {{ opts.app_config.verbose_name }} +› {{ opts.verbose_name_plural|capfirst }} +› {% translate 'Merge' %} +
+{% endblock %} + +{% block content %} +

+ {% translate "Selected tags" %} +

+ +
{% csrf_token %} +
+
    + {% for obj in queryset %} +
  • + {{ obj.name }} + +
    • + {% endfor %} +
+ +
+ + +
+ + + + + {% translate "No, take me back" %} +
+
+{% endblock %} diff --git a/src/app_project/tests/__init__.py b/src/feeds/templatetags/__init__.py similarity index 100% rename from src/app_project/tests/__init__.py rename to src/feeds/templatetags/__init__.py diff --git a/src/feeds/templatetags/feeds.py b/src/feeds/templatetags/feeds.py new file mode 100644 index 0000000..d95c9b9 --- /dev/null +++ b/src/feeds/templatetags/feeds.py @@ -0,0 +1,19 @@ +from django import template + +register = template.Library() + + +@register.filter +def categories(value, arg): + """Returns all the Category objects on an Article or Feed with a given prefix. + + The set of matching Categories objects is created using a list comprehension + as it assumes the categories were prefetched whereas filtering the QuerySet + would trigger another fetch from the database. + + """ + return [ + category + for category in value.categories.all() + if category.name.startswith(arg) and category.level > 1 + ] diff --git a/src/feeds/tests/__init__.py b/src/feeds/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/admin/__init__.py b/src/feeds/tests/admin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/admin/test_alias_admin.py b/src/feeds/tests/admin/test_alias_admin.py new file mode 100644 index 0000000..a43a0b7 --- /dev/null +++ b/src/feeds/tests/admin/test_alias_admin.py @@ -0,0 +1,12 @@ +from django.test import TestCase + +import pytest + +from feeds.tests.factories import AliasFactory +from feeds.tests.mixins import AdminTests + + +@pytest.mark.django_db +class AliasAdminTests(AdminTests, TestCase): + factory_class = AliasFactory + query_count = 5 diff --git a/src/feeds/tests/admin/test_article_admin.py b/src/feeds/tests/admin/test_article_admin.py new file mode 100644 index 0000000..6521030 --- /dev/null +++ b/src/feeds/tests/admin/test_article_admin.py @@ -0,0 +1,12 @@ +from django.test import TestCase + +import pytest + +from feeds.tests.factories import ArticleFactory +from feeds.tests.mixins import AdminTests + + +# @pytest.mark.django_db +# class ArticleAdminTests(AdminTests, TestCase): +# factory_class = ArticleFactory +# query_count = 9 diff --git a/src/feeds/tests/admin/test_author_admin.py b/src/feeds/tests/admin/test_author_admin.py new file mode 100644 index 0000000..4de6c63 --- /dev/null +++ b/src/feeds/tests/admin/test_author_admin.py @@ -0,0 +1,62 @@ +from random import choice + +from django.test import TestCase + +import pytest +from django.urls import reverse + +from feeds.models import Author +from feeds.tests.factories import ArticleFactory, AuthorFactory +from feeds.tests.mixins import AdminTests + +pytestmark = pytest.mark.django_db + + +class AuthorAdminTests(AdminTests, TestCase): + factory_class = AuthorFactory + query_count = 6 + + +def test_merge_authors_action(admin_client): + authors = AuthorFactory.create_batch(5) + article = ArticleFactory(authors=authors) + selected = choice(authors) + + url = reverse("admin:feeds_author_changelist") + data = { + "post": "yes", + "_selected_action": [author.pk for author in authors], + "action": "merge_authors", + "selected": selected.pk, + } + + admin_client.post(url, data, follow=True) + + # Merged authors are deleted + actual = [author.pk for author in Author.objects.all()] + assert actual == [selected.pk] + + # Merged author are replaced + actual = [author.pk for author in article.authors.all()] + assert actual == [selected.pk] + + +def test_show_articles_for_authors_action(admin_client): + authors = AuthorFactory.create_batch(5) + selected = choice(authors) + ArticleFactory.create_batch(10) + + url = reverse("admin:feeds_author_changelist") + data = { + "post": "yes", + "_selected_action": [selected.pk], + "action": "show_articles", + } + + response = admin_client.post(url, data, follow=True) + + request = response.request + slugs = selected.slug + + assert request["PATH_INFO"] == reverse("admin:feeds_article_changelist") + assert request["QUERY_STRING"] == f"authors__slug__in={slugs}" diff --git a/src/feeds/tests/admin/test_feed_admin.py b/src/feeds/tests/admin/test_feed_admin.py new file mode 100644 index 0000000..54e9ff4 --- /dev/null +++ b/src/feeds/tests/admin/test_feed_admin.py @@ -0,0 +1,34 @@ +from unittest import mock + +from django.test import TestCase + +import pytest +from django.urls import reverse + +from feeds.tests.factories import FeedFactory +from feeds.tests.mixins import AdminTests + + +pytestmark = pytest.mark.django_db + + +class FeedAdminTests(AdminTests, TestCase): + factory_class = FeedFactory + query_count = 6 + + +def mock_load_feed(feed): + pass + + +@mock.patch("feeds.services.feeds.load_feed") +def test_feed_load_feed_action(mocked, admin_client): + mocked.side_effect = mock_load_feed + feed = FeedFactory.create() + url = reverse("admin:feeds_feed_changelist") + data = {"action": "load_feed", "_selected_action": [str(feed.pk)]} + + response = admin_client.post(url, data, follow=True) + + assert response.status_code == 200 + assert mocked.call_count == 1 diff --git a/src/feeds/tests/admin/test_source_admin.py b/src/feeds/tests/admin/test_source_admin.py new file mode 100644 index 0000000..cbd66d3 --- /dev/null +++ b/src/feeds/tests/admin/test_source_admin.py @@ -0,0 +1,37 @@ +from random import choice + +from django.test import TestCase + +import pytest +from django.urls import reverse + +from feeds.tests.factories import ArticleFactory, SourceFactory +from feeds.tests.mixins import AdminTests + +pytestmark = pytest.mark.django_db + + +class SourceAdminTests(AdminTests, TestCase): + factory_class = SourceFactory + query_count = 5 + + +def test_show_articles_for_sources_action(admin_client): + sources = SourceFactory.create_batch(5) + selected = choice(sources) + ArticleFactory.create_batch(10) + + url = reverse("admin:feeds_source_changelist") + data = { + "post": "yes", + "_selected_action": [selected.pk], + "action": "show_articles", + } + + response = admin_client.post(url, data, follow=True) + + request = response.request + slugs = selected.slug + + assert request["PATH_INFO"] == reverse("admin:feeds_article_changelist") + assert request["QUERY_STRING"] == f"source__slug__in={slugs}" diff --git a/src/feeds/tests/admin/test_tag_admin.py b/src/feeds/tests/admin/test_tag_admin.py new file mode 100644 index 0000000..01ab956 --- /dev/null +++ b/src/feeds/tests/admin/test_tag_admin.py @@ -0,0 +1,63 @@ +from random import choice + +from django.test import TestCase + +import pytest +from django.urls import reverse + +from feeds.models import Tag +from feeds.tests.factories import ArticleFactory, TagFactory +from feeds.tests.mixins import AdminTests + + +pytestmark = pytest.mark.django_db + + +class TagAdminTests(AdminTests, TestCase): + factory_class = TagFactory + query_count = 6 + + +def test_merge_tags_action(admin_client): + tags = TagFactory.create_batch(5) + article = ArticleFactory(tags=tags) + selected = choice(tags) + + url = reverse("admin:feeds_tag_changelist") + data = { + "post": "yes", + "_selected_action": [tag.pk for tag in tags], + "action": "merge_tags", + "selected": selected.pk, + } + + admin_client.post(url, data, follow=True) + + # Merged tags are deleted + actual = [tag.pk for tag in Tag.objects.all()] + assert actual == [selected.pk] + + # Merged tags are replaced + actual = [tag.pk for tag in article.tags.all()] + assert actual == [selected.pk] + + +def test_show_articles_for_tag_action(admin_client): + tags = TagFactory.create_batch(5) + selected = choice(tags) + ArticleFactory.create_batch(10) + + url = reverse("admin:feeds_tag_changelist") + data = { + "post": "yes", + "_selected_action": [selected.pk], + "action": "show_articles", + } + + response = admin_client.post(url, data, follow=True) + + request = response.request + slugs = selected.slug + + assert request["PATH_INFO"] == reverse("admin:feeds_article_changelist") + assert request["QUERY_STRING"] == f"tags__slug__in={slugs}" diff --git a/src/feeds/tests/conditions/__init__.py b/src/feeds/tests/conditions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/conditions/articles.py b/src/feeds/tests/conditions/articles.py new file mode 100644 index 0000000..f4028de --- /dev/null +++ b/src/feeds/tests/conditions/articles.py @@ -0,0 +1,21 @@ +from feeds.models import Article + + +def only_published_articles(articles, timestamp) -> bool: + return all(article.date and article.date < timestamp for article in articles) + + +def article_was_clicked(obj: Article) -> bool: + views = obj.views + obj.refresh_from_db() + return views + 1 == obj.views + + +def only_tags_for_articles(articles, tags) -> bool: + expected = set([tag.pk for article in articles for tag in article.tags.all()]) + actual = [tag.pk for tag in tags] + return sorted(actual) == sorted(expected) + + +def only_articles_between_dates(articles, start_date, end_date) -> bool: + return all(start_date <= article.date.date() <= end_date for article in articles) diff --git a/src/feeds/tests/conditions/emails.py b/src/feeds/tests/conditions/emails.py new file mode 100644 index 0000000..7456e76 --- /dev/null +++ b/src/feeds/tests/conditions/emails.py @@ -0,0 +1,28 @@ +from django.core import mail + + +def matches(obj, **kwargs): + results = [] + for k, v in kwargs.items(): + if hasattr(obj, k) and getattr(obj, k): + results.append(True) + else: + results.append(False) + return all(results) + + +def emails_equal(e1, e2) -> bool: + return ( + e1.to == e2.to and e1.from_email == e2.from_email and e1.subject == e2.subject + ) + + +def email_was_sent(sender, receiver) -> bool: + def _matches(item): + return matches(item, to=[receiver], from_email=sender) + + return any(map(_matches, mail.outbox)) + + +def email_not_sent() -> bool: + return len(mail.outbox) == 0 diff --git a/src/feeds/tests/conditions/forms.py b/src/feeds/tests/conditions/forms.py new file mode 100644 index 0000000..eb22640 --- /dev/null +++ b/src/feeds/tests/conditions/forms.py @@ -0,0 +1,21 @@ +def data_is_empty(form) -> bool: + return form.data == {} + + +def cleaned_data_is_empty(form) -> bool: + return form.cleaned_data == {} + + +def form_is_bound(response, key="form") -> bool: + form = response.context.get(key) + return form.is_bound + + +def form_is_valid(response, key="form") -> bool: + form = response.context.get(key) + return form.is_valid() + + +def form_is_reset(response, key="form") -> bool: + form = response.context.get(key) + return form and data_is_empty(form) and cleaned_data_is_empty(form) diff --git a/src/feeds/tests/conditions/models.py b/src/feeds/tests/conditions/models.py new file mode 100644 index 0000000..5aecd46 --- /dev/null +++ b/src/feeds/tests/conditions/models.py @@ -0,0 +1,16 @@ +from django.forms import model_to_dict + + +def get_fields(obj): + return model_to_dict(obj, fields=[field.name for field in obj._meta.fields]) # noqa + + +def get_updated(obj): + return obj._meta.model.objects.get(pk=obj.id) # noqa + + +def changed_fields(obj, *names) -> bool: + d1 = get_fields(obj) + d2 = get_fields(get_updated(obj)) + changed = [k for k, v in d1.items() if v != d2[k]] + return sorted(names) == sorted(changed) diff --git a/src/feeds/tests/conditions/querysets.py b/src/feeds/tests/conditions/querysets.py new file mode 100644 index 0000000..79f537f --- /dev/null +++ b/src/feeds/tests/conditions/querysets.py @@ -0,0 +1,17 @@ +from typing import Union + +from django.db.models import QuerySet +from django.template.response import TemplateResponse + + +def is_paginated(response: TemplateResponse, key: str = "paginator") -> bool: + paginator = response.context[key] + return paginator.num_pages > 1 + + +def is_ordered(obj: Union[QuerySet, TemplateResponse], key: str = "paginator") -> bool: + if isinstance(obj, QuerySet): # type: ignore + return obj.ordered + elif isinstance(obj, TemplateResponse): + return obj.context[key].object_list.ordered + return False diff --git a/src/feeds/tests/conditions/responses.py b/src/feeds/tests/conditions/responses.py new file mode 100644 index 0000000..68ee514 --- /dev/null +++ b/src/feeds/tests/conditions/responses.py @@ -0,0 +1,37 @@ +from django.contrib import messages + + +def redirects_to(response, url) -> bool: + if hasattr(response, "redirect_chain"): + redirect_url, status_code = response.redirect_chain[-1] + else: + status_code = response.status_code + redirect_url = response.url + return status_code == 302 and redirect_url == url + + +def is_error_message(message) -> bool: + return message.level == messages.ERROR + + +def is_warning_message(message) -> bool: + return message.level == messages.WARNING + + +def is_success_message(message) -> bool: + return message.level == messages.SUCCESS + + +def success_message_shown(response) -> bool: + objs = list(response.context.get("messages", [])) + return bool(objs and len(objs) == 1) and is_success_message(objs[0]) + + +def warning_message_shown(response) -> bool: + objs = list(response.context.get("messages", [])) + return bool(objs and len(objs) == 1) and is_warning_message(objs[0]) + + +def error_message_shown(response) -> bool: + objs = list(response.context.get("messages", [])) + return bool(objs and len(objs) == 1) and is_error_message(objs[0]) diff --git a/src/feeds/tests/conftest.py b/src/feeds/tests/conftest.py new file mode 100644 index 0000000..729a6f8 --- /dev/null +++ b/src/feeds/tests/conftest.py @@ -0,0 +1,23 @@ +import pytest + + +# Override the local setting used for development to ensure that feeds +# are always loaded by default, regardless of the time that tests are +# run. This reduces the amount of setup needed for feed-related tests. +@pytest.fixture(autouse=True) +def use_flexible_load_schedule(settings): + settings.FEEDS_LOAD_SCHEDULE = "* * * * *" + + +@pytest.fixture(autouse=True) +def use_dummy_cache_backend(settings): + settings.CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + } + } + + +@pytest.fixture(autouse=True) +def disable_open_graph_loads(settings): + settings.LOAD_OPEN_GRAPH_DATA = False diff --git a/src/feeds/tests/factories/__init__.py b/src/feeds/tests/factories/__init__.py new file mode 100644 index 0000000..2ff09ef --- /dev/null +++ b/src/feeds/tests/factories/__init__.py @@ -0,0 +1,30 @@ +import factory # type: ignore + +from feeds.tests.providers import html + +from .alias import AliasFactory +from .article import ArticleFactory +from .author import AuthorFactory +from .category import CategoryFactory +from .feed import FeedFactory +from .source import SourceFactory +from .tag import TagFactory +from .user import UserFactory + +__all__ = [ + "AliasFactory", + "ArticleFactory", + "AuthorFactory", + "CategoryFactory", + "FeedFactory", + "SourceFactory", + "TagFactory", + "UserFactory", +] + +# Register the custom providers here, so they are available anywhere +# the factories are used. This could be added to the root conftest, +# but we occasionally want to be able to create instances of factories +# in the django shell, particularly for ad-hoc testing. + +factory.Faker.add_provider(html.Provider) diff --git a/src/feeds/tests/factories/alias.py b/src/feeds/tests/factories/alias.py new file mode 100644 index 0000000..c003743 --- /dev/null +++ b/src/feeds/tests/factories/alias.py @@ -0,0 +1,15 @@ +import factory # type: ignore +from faker import Faker + +from feeds.models import Alias + +fake = Faker() + + +class AliasFactory(factory.django.DjangoModelFactory): + class Meta: + model = Alias + + name = factory.Faker("user_name") + author = factory.SubFactory("feeds.tests.factories.AuthorFactory") + feed = factory.SubFactory("feeds.tests.factories.FeedFactory", enabled=True) diff --git a/src/feeds/tests/factories/article.py b/src/feeds/tests/factories/article.py new file mode 100644 index 0000000..a534dd3 --- /dev/null +++ b/src/feeds/tests/factories/article.py @@ -0,0 +1,132 @@ +import random + +from django.utils import timezone + +import factory # type: ignore +from faker import Faker + +from feeds.models import Article + +fake = Faker() + +tzinfo = timezone.get_default_timezone() + +# The distributions are used to set the probability that an Article will have +# a given number of tags, authors or clicks. + +tag_distribution = [0] * 10 + [1] * 70 + [2] * 10 + [3] * 10 +category_distribution = [0] * 70 + [1] * 20 + [2] * 10 +author_distribution = [0] * 5 + [1] * 80 + [2] * 10 + [3] * 5 +click_distribution = [0] * 5 + [1] * 80 + [2] * 10 + [3] * 5 + + +class ArticleFactory(factory.django.DjangoModelFactory): + class Meta: + model = Article + + # When articles are generated from the entries are loaded from a feed + # any trailing period is removed so all the titles across all the feeds + # are displayed consistently. Here we leave the trailing period in place + # since we don't specifically test for its presence, and it keeps the + # code simple. + # + # http_status is a custom provider which returns a status of 200 OK 95% + # of the time and 404 Not Found for the remaining 5%, which is much + # larger than the observed rate in production. The status code is not + # used to filter Articles (just yet) but we want to create a data set + # that is close to reality. + # + # The publication date is set for a random time in the past two weeks + # since the home page is paginated by week and this will generate a + # set of Articles across multiple pages. + # + # The 'publish' flag is set to True 80% of the time so the list of + # articles in any given week contains a mix of published and unpublished + # items. + # + # The post_generation hooks for tags and authors use in order + # of precedence: + # + # 1. Any objects passed as arguments to the factory. + # 2. Any existing objects from the database. + # 3. New instances. + # + # The number of objects depends on the probability distributions defined + # above. + + title = factory.Faker("sentence") + url = factory.Faker("url") + date = factory.Faker("date_time_between", start_date="-2w", tzinfo=tzinfo) + summary = factory.Faker("paragraph") + publish = factory.Faker("boolean", chance_of_getting_true=95) + source = factory.SubFactory("feeds.tests.factories.SourceFactory") + identifier = factory.Faker("url") + feed = factory.SubFactory("feeds.tests.factories.FeedFactory") + data = {"default": "value"} # type: ignore + views = factory.Faker('pyint', min_value=0, max_value=100) + + @factory.post_generation + def categories(self, create, extracted, **kwargs): + if create: + if extracted: + self.categories.set(extracted) + else: + from .category import Category, CategoryFactory + + count = random.choice(category_distribution) + + if Category.objects.exists(): + categories = list(Category.objects.all()) + for n in range(count): + self.categories.add(random.choice(categories)) + else: + for n in range(count): + self.categories.add(CategoryFactory()) + + @factory.post_generation + def tags(self, create, extracted, **kwargs): + if create: + if extracted: + self.tags.set(extracted) + else: + from .tag import Tag, TagFactory + + count = random.choice(tag_distribution) + + if Tag.objects.exists(): + tags = list(Tag.objects.all()) + for n in range(count): + self.tags.add(random.choice(tags)) + else: + for n in range(count): + self.tags.add(TagFactory()) + + @factory.post_generation + def authors(self, create, extracted, **kwargs): + if create: + if extracted: + self.authors.set(extracted) + else: + from .author import Author, AuthorFactory + + count = random.choice(author_distribution) + + if Author.objects.exists(): + authors = list(Author.objects.all()) + for n in range(count): + self.authors.add(random.choice(authors)) + else: + for n in range(count): + self.authors.add(AuthorFactory()) + + @factory.post_generation # type: ignore + def data(self, create, extracted, **kwargs): + if extracted: + self.data = extracted + else: + self.data = { + "og:title": self.title, + "og:url": self.url, + } + if create: + self.save() diff --git a/src/feeds/tests/factories/author.py b/src/feeds/tests/factories/author.py new file mode 100644 index 0000000..fd4900c --- /dev/null +++ b/src/feeds/tests/factories/author.py @@ -0,0 +1,23 @@ +import factory # type: ignore +from faker import Faker + +from feeds.models import Author + +fake = Faker() + + +class AuthorFactory(factory.django.DjangoModelFactory): + class Meta: + model = Author + django_get_or_create = ("slug",) + + # Although there is a many-to-many relationship between Article + # and Author we don't create a post_generation hook for authors + # since that adds extra code for no added value. It's better to + # have a single 'entry-point' for generating Articles and pass + # an Author object as an argument if we want to generate a list + # of Articles for a given Author. + + name = factory.Faker("name") + slug = factory.Faker("slug") + description = factory.Faker("html_text") diff --git a/src/feeds/tests/factories/category.py b/src/feeds/tests/factories/category.py new file mode 100644 index 0000000..6f81b3a --- /dev/null +++ b/src/feeds/tests/factories/category.py @@ -0,0 +1,14 @@ +import factory # type: ignore +from faker import Faker + +from feeds.models import Category + +fake = Faker() + + +class CategoryFactory(factory.django.DjangoModelFactory): + class Meta: + model = Category + + name = factory.Faker("word") + slug = factory.Faker("slug") diff --git a/src/feeds/tests/factories/feed.py b/src/feeds/tests/factories/feed.py new file mode 100644 index 0000000..6ba003f --- /dev/null +++ b/src/feeds/tests/factories/feed.py @@ -0,0 +1,42 @@ +import random + +import factory # type: ignore +from faker import Faker + +from feeds.models import Feed + +fake = Faker() + + +# The distributions are used to set the probability that a Feed will have +# a given number of categories. + +category_distribution = [0] * 0 + [1] * 70 + [2] * 20 + [10] * 3 + + +class FeedFactory(factory.django.DjangoModelFactory): + class Meta: + model = Feed + + name = factory.Faker("word") + url = factory.Faker("url") + source = factory.SubFactory("feeds.tests.factories.SourceFactory") + auto_publish = factory.Faker("boolean", chance_of_getting_true=90) + + @factory.post_generation + def categories(self, create, extracted, **kwargs): + if create: + if extracted: + self.categories.set(extracted) + else: + from .category import Category, CategoryFactory + + count = random.choice(category_distribution) + + if Category.objects.exists(): + categories = list(Category.objects.all()) + for n in range(count): + self.categories.add(random.choice(categories)) + else: + for n in range(count): + self.categories.add(CategoryFactory()) diff --git a/src/feeds/tests/factories/source.py b/src/feeds/tests/factories/source.py new file mode 100644 index 0000000..9a9a4cd --- /dev/null +++ b/src/feeds/tests/factories/source.py @@ -0,0 +1,29 @@ +import factory # type: ignore +from faker import Faker + +from feeds.models import Source + +fake = Faker() + + +class SourceFactory(factory.django.DjangoModelFactory): + class Meta: + model = Source + django_get_or_create = ("slug",) + + name = factory.Faker("name") + slug = factory.Faker("slug") + url = factory.Faker("url") + data = {"default": "value"} # type: ignore + + @factory.post_generation # type: ignore + def data(self, create, extracted, **kwargs): + if extracted: + self.data = extracted + else: + self.data = { + "og:title": self.name, + "og:url": self.url, + } + if create: + self.save() diff --git a/src/feeds/tests/factories/tag.py b/src/feeds/tests/factories/tag.py new file mode 100644 index 0000000..f0ff20a --- /dev/null +++ b/src/feeds/tests/factories/tag.py @@ -0,0 +1,17 @@ +import factory # type: ignore +from faker import Faker + +from feeds.models import Tag + +fake = Faker() + + +class TagFactory(factory.django.DjangoModelFactory): + class Meta: + model = Tag + django_get_or_create = ("slug",) + + name = factory.Faker("word") + slug = factory.Faker("slug") + summary = factory.Faker("paragraph") + description = factory.Faker("html_text") diff --git a/src/feeds/tests/factories/user.py b/src/feeds/tests/factories/user.py new file mode 100644 index 0000000..cb470dc --- /dev/null +++ b/src/feeds/tests/factories/user.py @@ -0,0 +1,18 @@ +from django.contrib.auth.models import User + +import factory # type: ignore + + +class UserFactory(factory.django.DjangoModelFactory): + class Meta: + model = User + django_get_or_create = ("username",) + + username = factory.Faker("user_name") + first_name = factory.Faker("first_name") + last_name = factory.Faker("last_name") + email = factory.Faker("email") + + is_staff = False + is_active = True + is_superuser = False diff --git a/src/feeds/tests/mixins/__init__.py b/src/feeds/tests/mixins/__init__.py new file mode 100644 index 0000000..8a53c3a --- /dev/null +++ b/src/feeds/tests/mixins/__init__.py @@ -0,0 +1,3 @@ +from .admin_tests import AdminTests + +__all__ = ("AdminTests",) diff --git a/src/feeds/tests/mixins/admin_tests.py b/src/feeds/tests/mixins/admin_tests.py new file mode 100644 index 0000000..abee12b --- /dev/null +++ b/src/feeds/tests/mixins/admin_tests.py @@ -0,0 +1,67 @@ +from django.contrib.contenttypes.models import ContentType +from django.urls import reverse + +import pytest + +from feeds.tests.factories import UserFactory + + +@pytest.mark.django_db +class AdminTests: + """Tests for the Django Admin + + The tests only check the changelist view, since that is where most of + the errors will occur. There is little value in testing the add, change + or delete views since we would just be testing Django itself. When these + are customised or if the on_delete rules in the underlying models do not + allow cascaded deletes, generic tests would either not work or add very + little value. + + """ + + # The FactoryBoy class for creating objects + factory_class = None + # The number of objects to create + batch_size = 10 + + @classmethod + def setUpTestData(cls): + cls.admin = UserFactory(is_staff=True, is_superuser=True) + cls.model = cls.factory_class._meta.model # noqa + cls.app_label = cls.model._meta.app_label # noqa + cls.model_name = cls.model._meta.model_name # noqa + cls.objects = cls.factory_class.create_batch(cls.batch_size) + + def clear_query_caches(self): # noqa + ContentType.objects.clear_cache() + + def setUp(self): + self.client.force_login(self.admin) # noqa + + def test_changelist_view(self): + """Verify the changelist view is displayed without any errors""" + url = reverse("admin:%s_%s_changelist" % (self.app_label, self.model_name)) + response = self.client.get(url) # noqa + assert response.status_code == 200 + + def test_changelist_search(self): + """Verify search_fields does not contain any invalid lookups""" + url = reverse("admin:%s_%s_changelist" % (self.app_label, self.model_name)) + response = self.client.get(url, {"q": "test"}, follow=True) # noqa + assert response.status_code == 200 + + def test_changelist_query_count(self): + """Verify the number of queries is fixed when a new object is added""" + url = reverse("admin:%s_%s_changelist" % (self.app_label, self.model_name)) + + self.clear_query_caches() + with self.assertNumQueries(self.query_count): # noqa + self.client.get(url) # noqa + + # Add another object + self.factory_class() + + self.clear_query_caches() + # The number of queries should remain the same + with self.assertNumQueries(self.query_count): # noqa + self.client.get(url) # noqa diff --git a/src/feeds/tests/models/__init__.py b/src/feeds/tests/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/models/test_article.py b/src/feeds/tests/models/test_article.py new file mode 100644 index 0000000..b3d8782 --- /dev/null +++ b/src/feeds/tests/models/test_article.py @@ -0,0 +1,29 @@ +import datetime as dt + +from django.utils import timezone + +import pytest + +from feeds.models import Article +from feeds.tests.factories import ArticleFactory + +pytestmark = pytest.mark.django_db + + +def test_published(): + today = timezone.now() + ArticleFactory.create(title="first", date=today, publish=True) + ArticleFactory.create(title="second", date=today, publish=False) + queryset = Article.objects.published() + assert queryset.count() == 1 + assert queryset.first().title == "first" + + +def test_for_date(): + today = timezone.now() + ArticleFactory.create(title="first", date=today - dt.timedelta(days=1)) + ArticleFactory.create(title="second", date=today) + ArticleFactory.create(title="first", date=today + dt.timedelta(days=1)) + queryset = Article.objects.for_date(today) + assert queryset.count() == 1 + assert queryset.first().title == "second" diff --git a/src/feeds/tests/models/test_feed.py b/src/feeds/tests/models/test_feed.py new file mode 100644 index 0000000..f2341c6 --- /dev/null +++ b/src/feeds/tests/models/test_feed.py @@ -0,0 +1,49 @@ +from django.core.exceptions import ValidationError +from django.utils import timezone + +import pytest + +from feeds.models import Feed +from feeds.tests.factories import FeedFactory + +pytestmark = pytest.mark.django_db + + +def test_schedule_invalid(): + with pytest.raises(ValidationError): + FeedFactory(schedule="0 10, 12 * * *").full_clean() + + +def test_schedule_minutes_valid(): + FeedFactory(schedule="0 10 * * *").full_clean() + FeedFactory(schedule="* 10 * * *").full_clean() + + +def test_schedule_minutes_invalid(): + with pytest.raises(ValidationError): + FeedFactory(schedule="15 10 * * *").full_clean() + + +@pytest.mark.freeze_time +def test_feed_scheduled(freezer): + feed = FeedFactory.create(enabled=True, schedule="0 10 * * *") + freezer.move_to(timezone.now().replace(minute=0, hour=10)) + feeds = Feed.objects.scheduled_for(timezone.now()) + assert feed.pk in [feed.pk for feed in feeds] + + +@pytest.mark.freeze_time +def test_unscheduled_feed_skipped(monkeypatch, freezer): + feed = FeedFactory.create(enabled=True, schedule="0 11 * * *") + freezer.move_to(timezone.now().replace(minute=0, hour=10)) + feeds = Feed.objects.scheduled_for(timezone.now()) + assert feed.pk not in [feed.pk for feed in feeds] + + +@pytest.mark.freeze_time +def test_default_schedule_used(monkeypatch, freezer, settings): + feed = FeedFactory.create(enabled=True, schedule="") + settings.FEEDS_LOAD_SCHEDULE = "0 10 * * *" + freezer.move_to(timezone.now().replace(minute=0, hour=10)) + feeds = Feed.objects.scheduled_for(timezone.now()) + assert feed.pk in [feed.pk for feed in feeds] diff --git a/src/feeds/tests/providers/__init__.py b/src/feeds/tests/providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/providers/html.py b/src/feeds/tests/providers/html.py new file mode 100644 index 0000000..729c7ad --- /dev/null +++ b/src/feeds/tests/providers/html.py @@ -0,0 +1,9 @@ +from faker import Faker +from faker.providers import BaseProvider + +fake = Faker() + + +class Provider(BaseProvider): + def html_text(self): # noqa + return "

%s

%s

" % (fake.sentence(), fake.paragraph()) diff --git a/src/feeds/tests/services/__init__.py b/src/feeds/tests/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/services/test_load_feed.py b/src/feeds/tests/services/test_load_feed.py new file mode 100644 index 0000000..4f7d0c9 --- /dev/null +++ b/src/feeds/tests/services/test_load_feed.py @@ -0,0 +1,183 @@ +from django.template import Context, Template +from django.utils import timezone + +import pytest + +from feeds.models import Article +from feeds.services import feeds +from feeds.services.feeds import normalize_title +from feeds.tests.factories import AliasFactory, ArticleFactory, FeedFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture() +def feed_template(): + return """ + + + + {{ title }} + {{ url }} + {{ identifier }} + {{ date_str }} + {{ date_str }} + {{ author }} + {{ summary }} + {% for tag in tags %} + + {% endfor %} + + + """ + + +@pytest.fixture() +def feed_context(): + # The context contains values for the feed template and building Articles. + # Dates are included as datetimes and RFC 822 format strings so the context + # can be used to fill out the feed template and populate Article models. + now = timezone.now().replace(microsecond=0) + return { + "title": "Article title.", + "url": "https://www.example.com/entry/", + "date": now, + "date_str": now.strftime("%a, %d %b %Y %H:%M:%S %z"), + "identifier": "Article:Identifier", + "author": "Article Author", + "summary": "Article Summary", + "tags": ["First", "Second"], + } + + +def test_article_added(monkeypatch, feed_template, feed_context): + # feedparser can parse a file, url or string so we can use the get_url() + # function in the feeds module to inject a string instead of the Blog's + # url and get feedparser to parse it and return the entries. + + def mock_return(feed): + template = Template(feed_template) + context = Context(feed_context) + return template.render(context) + + monkeypatch.setattr(feeds, "get_source", mock_return) + feed = FeedFactory.create(enabled=True) + feeds.load_feed(feed) + articles = Article.objects.all() + + # Results contain added Article + assert len(articles) == 1 + + # Verify the rss attributes are set to the values from the feed + # Ensure microseconds on the date the entry was published are set + # to zero, rather than the value when the entry is saved in case + # the date is used to see if an entry has changed when the feed + # is next loaded. + + article = articles[0] + + assert article.title == normalize_title(feed_context["title"]) + assert article.authors.first().name == feed_context["author"] + assert article.url == feed_context["url"] + assert article.date == feed_context["date"] + assert article.date.microsecond == 0 + assert article.summary == feed_context["summary"] + assert article.identifier == feed_context["identifier"] + assert article.source == feed.source + assert article.publish == feed.auto_publish + + # Verify the tags are added to the JSON data field. Only one + # tag is used to keep the feed template and this test, simple. + assert article.data["feed:tags"] == feed_context["tags"] + + # Verify the categories are added to the article + assert article.categories.count() == feed.categories.count() != 0 + assert article.categories.first() == feed.categories.first() + + +def test_article_updated(monkeypatch, feed_template, feed_context): + feed = FeedFactory.create(enabled=True, auto_publish=True) + params = { + k: v + for k, v in feed_context.items() + if not k.endswith("date_str") and k not in ("author", "tags") + } + existing = ArticleFactory.create(feed=feed, publish=feed.auto_publish, **params) + + feed_context["title"] = "Updated title" + updated_title = feed_context["title"] + + feed.auto_publish = False + feed.save() + + def mock_return(feed): + template = Template(feed_template) + context = Context(feed_context) + return template.render(context) + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(feed) + articles = Article.objects.filter(modified__gt=existing.modified) + + # Results contains updated Article + assert len(articles) == 1 + + # The title of the entry was updated. All other attributes + # remain unchanged. + + article = articles[0] + + assert article.modified > existing.modified + assert article.title == updated_title + assert article.url == feed_context["url"] + assert article.date == feed_context["date"] + assert article.date.microsecond == 0 + assert article.summary == feed_context["summary"] + assert article.identifier == feed_context["identifier"] + assert article.publish is True + + +def test_article_author_alias(monkeypatch, feed_template, feed_context): + alias = AliasFactory.create() + feed = alias.feed + author = alias.author + + feed_context["author"] = alias.name + + def mock_return(feed): + template = Template(feed_template) + context = Context(feed_context) + return template.render(context) + + monkeypatch.setattr(feeds, "get_source", mock_return) + + feeds.load_feed(feed) + + article = Article.objects.all().first() + + assert article.authors.first() == author + + +titles = [ + (" Hello World", "Hello World"), # leading space + (" Hello World", "Hello World"), # multiple leading spaces + ("Hello World ", "Hello World"), # trailing space + ("Hello World ", "Hello World"), # multiple trailing spaces + ("Hello World.", "Hello World"), # trailing period + ("Hello World...", "Hello World..."), # trailing ellipsis + ("Hello- World", "Hello - World"), # no space before hyphen + ("Hello -World", "Hello - World"), # no space after hyphen + ("Hello , World", "Hello, World"), # space before comma + ("Hello,World", "Hello, World"), # no space after comma + ("Hello(World)", "Hello (World)"), # no space before bracket + ("(Hello)World", "(Hello) World"), # no space after bracket +] + + +@pytest.mark.parametrize("before, expected", titles) +def test_normalise_title(before, expected): + after = feeds.normalize_title(before) + assert after == expected diff --git a/src/feeds/tests/services/test_load_feed_errors.py b/src/feeds/tests/services/test_load_feed_errors.py new file mode 100644 index 0000000..a2a8225 --- /dev/null +++ b/src/feeds/tests/services/test_load_feed_errors.py @@ -0,0 +1,231 @@ +from django.utils import timezone + +import pytest + +from feeds.models import Article, Author +from feeds.services import feeds +from feeds.tests.factories import FeedFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture() +def feed_template(): + return """ + + + + %(title)s + %(url)s + %(identifier)s + %(date)s + %(date)s + %(author)s + %(summary)s + + + + """ + + +@pytest.fixture() +def feed_context(): + return { + "title": "ARTICLE TITLE.", + "url": "https://www.example.com/entry/", + "date": timezone.now().strftime("%a, %d %b %Y %H:%M:%S %z"), + "identifier": "Article:Identifier", + "author": "Article Author", + "summary": "Article Summary", + "tags": "Tag", + } + + +def test_blank_identifier(monkeypatch, feed_template, feed_context): + """Feed entries with an empty identifier field are skipped""" + + def mock_return(feed): + feed_context["identifier"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_missing_identifier(monkeypatch, feed_template, feed_context): + """Feed entries with no identifier are loaded using the url as an identifier""" + feed_template = feed_template.replace("%(identifier)s", "") + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is True + + +def test_blank_title(monkeypatch, feed_template, feed_context): + """Feed entries with an empty title field are skipped""" + + def mock_return(feed): + feed_context["title"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_missing_title(monkeypatch, feed_template, feed_context): + """Feed entries with no title field are skipped""" + feed_template = feed_template.replace("%(title)s", "") + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_blank_link(monkeypatch, feed_template, feed_context): + """Feed entries with an empty link field are skipped""" + + def mock_return(feed): + feed_context["url"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_missing_link(monkeypatch, feed_template, feed_context): + """Feed entries with no link field are skipped""" + feed_template = feed_template.replace("%(url)s", "") + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_blank_date(monkeypatch, feed_template, feed_context): + """Feed entries with an empty published field are skipped""" + + def mock_return(feed): + feed_context["date"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_missing_date(monkeypatch, feed_template, feed_context): + """Feed entries with no published field are skipped""" + feed_template = feed_template.replace("%(date)s", "") + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_blank_author(monkeypatch, feed_template, feed_context): + """Feed entries with an empty author field are loaded""" + + def mock_return(feed): + feed_context["author"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().authors.count() == 0 + + +def test_missing_author(monkeypatch, feed_template, feed_context): + """Feed entries with no author field are loaded""" + feed_template = feed_template.replace( + "%(author)s", "" + ) + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().authors.count() == 0 + + +def test_multiple_authors(monkeypatch, feed_template, feed_context): + """Feeds where multiple Authors exist with the same name are loaded""" + Author.objects.create(name=feed_context["author"]) + Author.objects.create(name=feed_context["author"]) + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().authors.count() == 1 + + +def test_blank_summary(monkeypatch, feed_template, feed_context): + """Feed entries with an empty summary field are loaded""" + + def mock_return(feed): + feed_context["summary"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().summary == "" + + +def test_missing_summary(monkeypatch, feed_template, feed_context): + """Feed entries with no summary field are loaded""" + feed_template = feed_template.replace( + '%(summary)s', "" + ) + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().summary == "" + + +def test_blank_tags(monkeypatch, feed_template, feed_context): + """Feed entries with an empty tag field are loaded""" + + def mock_return(feed): + feed_context["tags"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().data["feed:tags"] == [] + + +def test_missing_tags(monkeypatch, feed_template, feed_context): + """Feed entries with no tag field are loaded""" + feed_template = feed_template.replace( + '', "" + ) + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().data["feed:tags"] == []