From 37245a7adf83ab5399be83f3bc6f11b468060cdb Mon Sep 17 00:00:00 2001 From: Stuart MacKay Date: Sat, 16 Sep 2023 13:32:22 +0100 Subject: [PATCH] Add RSS related files from the django-lynx (private) repository. The code was taken from a working project which has been in production for two and a half years. The code was refactored for this project, the views, templates, etc. were moved to the demo app, requirements updated, etc. etc. THe demo site works and works with celery to automatically load feeds according to a preset schedule. If you just want to see it in action create an admin account, add a Source, then a Feed and you can load it via an admin action. Next steps are make everything run using either a virtualenv or Docker and get it all polished for a first release. --- .env.docker | 31 + .env.venv | 41 ++ .envrc | 12 + .gitignore | 3 + Dockerfile | 36 + MANIFEST.in | 6 +- Makefile | 16 +- README.md | 68 +- TODO.md | 19 + demo/__init__.py | 1 + demo/apps.py | 5 + demo/{conf => }/asgi.py | 2 +- demo/celery.py | 73 ++ demo/conf/urls.py | 17 - demo/paginators.py | 68 ++ demo/{conf => }/settings.py | 48 +- demo/static/images/rss.svg | 57 ++ demo/static/js/index.js | 23 + demo/tasks.py | 8 + demo/templates/demo/articles.html | 72 ++ demo/templates/demo/author.html | 68 ++ demo/templates/demo/authors.html | 55 ++ demo/templates/demo/snippets/pagination.html | 26 + .../demo/snippets/tag_author_cloud.html | 13 + demo/templates/demo/snippets/tag_cloud.html | 13 + .../demo/snippets/tag_source_cloud.html | 13 + demo/templates/demo/source.html | 95 +++ demo/templates/demo/sources.html | 62 ++ demo/templates/demo/tag.html | 78 +++ demo/templates/demo/tags.html | 57 ++ demo/tests/test_article_view.py | 27 + demo/tests/test_articles_view.py | 53 ++ demo/tests/test_author_view.py | 61 ++ demo/tests/test_authors_view.py | 31 + demo/tests/test_source_view.py | 61 ++ demo/tests/test_sources_view.py | 31 + demo/tests/test_tag_view.py | 75 ++ demo/tests/test_tags_view.py | 31 + demo/urls.py | 72 ++ demo/views/__init__.py | 20 + demo/views/article.py | 29 + demo/views/articles.py | 37 + demo/views/author.py | 44 ++ demo/views/authors.py | 21 + demo/views/source.py | 45 ++ demo/views/sources.py | 24 + demo/views/tag.py | 44 ++ demo/views/tags.py | 23 + demo/views/utils.py | 7 + demo/{conf => }/wsgi.py | 2 +- docker-compose.yml | 58 ++ docker-init.sh | 3 + manage.py | 3 +- requirements/dev.in | 8 + requirements/dev.txt | 271 ++++++-- requirements/tests.in | 1 + requirements/tests.txt | 50 +- setup.cfg | 6 +- setup.py | 27 +- src/app_project/admin/__init__.py | 5 - src/app_project/admin/example.py | 9 - src/app_project/migrations/0001_initial.py | 31 - src/app_project/models/__init__.py | 5 - src/app_project/models/example.py | 6 - src/app_project/settings.py | 3 - .../templates/app_project/base.html | 3 - .../templates/app_project/index.html | 6 - src/app_project/tests/test_views.py | 8 - src/app_project/urls.py | 5 - src/app_project/views/__init__.py | 5 - src/app_project/views/index.py | 5 - src/{app_project => feeds}/__init__.py | 0 src/feeds/admin/__init__.py | 17 + src/feeds/admin/alias.py | 12 + src/feeds/admin/article.py | 162 +++++ src/feeds/admin/author.py | 85 +++ src/feeds/admin/category.py | 14 + src/feeds/admin/feed.py | 184 +++++ src/feeds/admin/prettifiers.py | 65 ++ src/feeds/admin/source.py | 67 ++ src/feeds/admin/tag.py | 119 ++++ src/{app_project => feeds}/apps.py | 3 +- .../locale/en/LC_MESSAGES/django.mo | Bin .../locale/en/LC_MESSAGES/django.po | 0 src/feeds/migrations/0001_initial.py | 655 ++++++++++++++++++ .../conf => src/feeds/migrations}/__init__.py | 0 src/feeds/models/__init__.py | 17 + src/feeds/models/alias.py | 35 + src/feeds/models/article.py | 176 +++++ src/feeds/models/author.py | 47 ++ src/feeds/models/category.py | 12 + src/feeds/models/feed.py | 142 ++++ src/feeds/models/source.py | 47 ++ src/feeds/models/tag.py | 81 +++ .../migrations => feeds/services}/__init__.py | 0 src/feeds/services/feeds.py | 327 +++++++++ src/feeds/settings.py | 55 ++ src/feeds/templates/admin/merge_authors.html | 52 ++ src/feeds/templates/admin/merge_tags.html | 52 ++ .../tests => feeds/templatetags}/__init__.py | 0 src/feeds/templatetags/feeds.py | 19 + src/feeds/tests/__init__.py | 0 src/feeds/tests/admin/__init__.py | 0 src/feeds/tests/admin/test_alias_admin.py | 12 + src/feeds/tests/admin/test_article_admin.py | 12 + src/feeds/tests/admin/test_author_admin.py | 62 ++ src/feeds/tests/admin/test_feed_admin.py | 34 + src/feeds/tests/admin/test_source_admin.py | 37 + src/feeds/tests/admin/test_tag_admin.py | 63 ++ src/feeds/tests/conditions/__init__.py | 0 src/feeds/tests/conditions/articles.py | 21 + src/feeds/tests/conditions/emails.py | 28 + src/feeds/tests/conditions/forms.py | 21 + src/feeds/tests/conditions/models.py | 16 + src/feeds/tests/conditions/querysets.py | 17 + src/feeds/tests/conditions/responses.py | 37 + src/feeds/tests/conftest.py | 23 + src/feeds/tests/factories/__init__.py | 30 + src/feeds/tests/factories/alias.py | 15 + src/feeds/tests/factories/article.py | 132 ++++ src/feeds/tests/factories/author.py | 23 + src/feeds/tests/factories/category.py | 14 + src/feeds/tests/factories/feed.py | 42 ++ src/feeds/tests/factories/source.py | 29 + src/feeds/tests/factories/tag.py | 17 + src/feeds/tests/factories/user.py | 18 + src/feeds/tests/mixins/__init__.py | 3 + src/feeds/tests/mixins/admin_tests.py | 67 ++ src/feeds/tests/models/__init__.py | 0 src/feeds/tests/models/test_article.py | 29 + src/feeds/tests/models/test_feed.py | 49 ++ src/feeds/tests/providers/__init__.py | 0 src/feeds/tests/providers/html.py | 9 + src/feeds/tests/services/__init__.py | 0 src/feeds/tests/services/test_load_feed.py | 183 +++++ .../tests/services/test_load_feed_errors.py | 231 ++++++ 136 files changed, 5752 insertions(+), 309 deletions(-) create mode 100644 .env.docker create mode 100644 .env.venv create mode 100644 .envrc create mode 100644 Dockerfile create mode 100644 demo/__init__.py create mode 100644 demo/apps.py rename demo/{conf => }/asgi.py (80%) create mode 100644 demo/celery.py delete mode 100644 demo/conf/urls.py create mode 100644 demo/paginators.py rename demo/{conf => }/settings.py (56%) create mode 100644 demo/static/images/rss.svg create mode 100644 demo/tasks.py create mode 100644 demo/templates/demo/articles.html create mode 100644 demo/templates/demo/author.html create mode 100644 demo/templates/demo/authors.html create mode 100644 demo/templates/demo/snippets/pagination.html create mode 100644 demo/templates/demo/snippets/tag_author_cloud.html create mode 100644 demo/templates/demo/snippets/tag_cloud.html create mode 100644 demo/templates/demo/snippets/tag_source_cloud.html create mode 100644 demo/templates/demo/source.html create mode 100644 demo/templates/demo/sources.html create mode 100644 demo/templates/demo/tag.html create mode 100644 demo/templates/demo/tags.html create mode 100644 demo/tests/test_article_view.py create mode 100644 demo/tests/test_articles_view.py create mode 100644 demo/tests/test_author_view.py create mode 100644 demo/tests/test_authors_view.py create mode 100644 demo/tests/test_source_view.py create mode 100644 demo/tests/test_sources_view.py create mode 100644 demo/tests/test_tag_view.py create mode 100644 demo/tests/test_tags_view.py create mode 100644 demo/urls.py create mode 100644 demo/views/__init__.py create mode 100644 demo/views/article.py create mode 100644 demo/views/articles.py create mode 100644 demo/views/author.py create mode 100644 demo/views/authors.py create mode 100644 demo/views/source.py create mode 100644 demo/views/sources.py create mode 100644 demo/views/tag.py create mode 100644 demo/views/tags.py create mode 100644 demo/views/utils.py rename demo/{conf => }/wsgi.py (80%) create mode 100644 docker-compose.yml create mode 100755 docker-init.sh delete mode 100644 src/app_project/admin/__init__.py delete mode 100644 src/app_project/admin/example.py delete mode 100644 src/app_project/migrations/0001_initial.py delete mode 100644 src/app_project/models/__init__.py delete mode 100644 src/app_project/models/example.py delete mode 100644 src/app_project/settings.py delete mode 100644 src/app_project/templates/app_project/base.html delete mode 100644 src/app_project/templates/app_project/index.html delete mode 100644 src/app_project/tests/test_views.py delete mode 100644 src/app_project/urls.py delete mode 100644 src/app_project/views/__init__.py delete mode 100644 src/app_project/views/index.py rename src/{app_project => feeds}/__init__.py (100%) create mode 100644 src/feeds/admin/__init__.py create mode 100644 src/feeds/admin/alias.py create mode 100644 src/feeds/admin/article.py create mode 100644 src/feeds/admin/author.py create mode 100644 src/feeds/admin/category.py create mode 100644 src/feeds/admin/feed.py create mode 100644 src/feeds/admin/prettifiers.py create mode 100644 src/feeds/admin/source.py create mode 100644 src/feeds/admin/tag.py rename src/{app_project => feeds}/apps.py (86%) rename src/{app_project => feeds}/locale/en/LC_MESSAGES/django.mo (100%) rename src/{app_project => feeds}/locale/en/LC_MESSAGES/django.po (100%) create mode 100644 src/feeds/migrations/0001_initial.py rename {demo/conf => src/feeds/migrations}/__init__.py (100%) create mode 100644 src/feeds/models/__init__.py create mode 100644 src/feeds/models/alias.py create mode 100644 src/feeds/models/article.py create mode 100644 src/feeds/models/author.py create mode 100644 src/feeds/models/category.py create mode 100644 src/feeds/models/feed.py create mode 100644 src/feeds/models/source.py create mode 100644 src/feeds/models/tag.py rename src/{app_project/migrations => feeds/services}/__init__.py (100%) create mode 100644 src/feeds/services/feeds.py create mode 100644 src/feeds/settings.py create mode 100644 src/feeds/templates/admin/merge_authors.html create mode 100644 src/feeds/templates/admin/merge_tags.html rename src/{app_project/tests => feeds/templatetags}/__init__.py (100%) create mode 100644 src/feeds/templatetags/feeds.py create mode 100644 src/feeds/tests/__init__.py create mode 100644 src/feeds/tests/admin/__init__.py create mode 100644 src/feeds/tests/admin/test_alias_admin.py create mode 100644 src/feeds/tests/admin/test_article_admin.py create mode 100644 src/feeds/tests/admin/test_author_admin.py create mode 100644 src/feeds/tests/admin/test_feed_admin.py create mode 100644 src/feeds/tests/admin/test_source_admin.py create mode 100644 src/feeds/tests/admin/test_tag_admin.py create mode 100644 src/feeds/tests/conditions/__init__.py create mode 100644 src/feeds/tests/conditions/articles.py create mode 100644 src/feeds/tests/conditions/emails.py create mode 100644 src/feeds/tests/conditions/forms.py create mode 100644 src/feeds/tests/conditions/models.py create mode 100644 src/feeds/tests/conditions/querysets.py create mode 100644 src/feeds/tests/conditions/responses.py create mode 100644 src/feeds/tests/conftest.py create mode 100644 src/feeds/tests/factories/__init__.py create mode 100644 src/feeds/tests/factories/alias.py create mode 100644 src/feeds/tests/factories/article.py create mode 100644 src/feeds/tests/factories/author.py create mode 100644 src/feeds/tests/factories/category.py create mode 100644 src/feeds/tests/factories/feed.py create mode 100644 src/feeds/tests/factories/source.py create mode 100644 src/feeds/tests/factories/tag.py create mode 100644 src/feeds/tests/factories/user.py create mode 100644 src/feeds/tests/mixins/__init__.py create mode 100644 src/feeds/tests/mixins/admin_tests.py create mode 100644 src/feeds/tests/models/__init__.py create mode 100644 src/feeds/tests/models/test_article.py create mode 100644 src/feeds/tests/models/test_feed.py create mode 100644 src/feeds/tests/providers/__init__.py create mode 100644 src/feeds/tests/providers/html.py create mode 100644 src/feeds/tests/services/__init__.py create mode 100644 src/feeds/tests/services/test_load_feed.py create mode 100644 src/feeds/tests/services/test_load_feed_errors.py diff --git a/.env.docker b/.env.docker new file mode 100644 index 0000000..83637ca --- /dev/null +++ b/.env.docker @@ -0,0 +1,31 @@ +# +# Environment variables used for running the demo site using docker +# + +# Service: db +# Rather than set POSTGRES_PASSWORD (required) and leave POSTGRES_USER and +# so POSTGRES_DB to default to "postgres" we set the variables to the name +# of the app so it mirrors the configuration for locally installed services +# which might be shared between sites. However, the value of doing this is +# that you easily initialize the database with dumps from production where +# the ownership of the tables is defined. + +PGHOST=db +POSTGRES_DB=feeds +POSTGRES_USER=feeds +POSTGRES_PASSWORD=feeds + +# Service: broker +# This, like the db service, simply mirrors the configuration for rabbitmq +# being installed locally and possibly shared between sites. + +RABBITMQ_DEFAULT_USER=feeds +RABBITMQ_DEFAULT_PASS=feeds +RABBITMQ_DEFAULT_VHOST=feeds + +# Services: web, celery, celery-beat +# Environment variables used in Django settings + +CELERY_BROKER_URL=amqp://feeds:feeds@broker:5672/feeds + +DB_HOST=db diff --git a/.env.venv b/.env.venv new file mode 100644 index 0000000..de3186b --- /dev/null +++ b/.env.venv @@ -0,0 +1,41 @@ +# +# Environment variables used for development using a virtualenv +# + +# Copy or rename this file to .env so it can be used by direnv and/or an IDE. + +# This assumes you've installed postgresql and rabbitmq on your +# machine and have added the accounts need to connect to them: +# +# sudo -u postgres psql +# postgres=# create database feeds; +# postgres=# create user feeds with encrypted password 'feeds'; +# postgres=# grant all privileges on database feeds to feeds; +# +# sudo rabbitmqctl add_user feeds feeds +# sudo rabbitmqctl set_permissions -p / feeds ".*" ".*" ".*" +# sudo rabbitmqctl add_vhost feeds + +# Set environment variables used by the postgresql commands to connect +# to the database. This is the admin account not the account used by +# Django to connect to the database. PGPASSWORD is particularly useful +# to avoid having to enter it for each command. You will need to configure +# the server to allow username/password (md5) authentication on local +# (socket) connections and so avoid having to su to the postgres user first. +# +# /etc/postgresql//main/pg_hba.conf +# local all all md5 +# +# For more info on environment variables see, +# https://www.postgresql.org/docs/current/libpq-envars.html + +PGHOST=localhost +PGPORT=5432 +PGUSER=postgres +PGPASSWORD=postgres + +# Environment variables used by Django + +CELERY_BROKER_URL=amqp://feeds:feeds@localhost:5672/feeds + +DB_HOST=localhost diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..a8b0583 --- /dev/null +++ b/.envrc @@ -0,0 +1,12 @@ +# direnv file +# When you cd to this directory, direnv will activate the virtualenv +# and set all the environment variables in the .env file. +# See https://direnv.net/man/direnv-stdlib.1.html + +# Activate the virtualenv +# This simply replicates what venv/bin/activate does. +export VIRTUAL_ENV=venv +PATH_add "$VIRTUAL_ENV/bin" + +# Set environment variables from .env +dotenv diff --git a/.gitignore b/.gitignore index 59ea6b5..bd33e56 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ __pycache__/ # Ignore dot files except those required for the project .* !.editorconfig +!.envrc +!.env.docker +!.env.venv !.flake8 !.gitattributes !.github diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d4dba6f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# +# Dockerfile: configuration for building the images for the web app, +# celery-beat and celery-worker. +# + +# Base python image +FROM python:3.8.16-buster +# Add the site to the python path +ENV PYTHONPATH /code +# Send all output on stdout and stderr straight to the container logs +ENV PYTHONUNBUFFERED 1 +# Set the locale +ENV LC_ALL C.UTF-8 +# Terminals support 256 colours +ENV TERM xterm-256color + +# Mount the site directory + +RUN mkdir /code +WORKDIR /code + +# Install dependencies + +RUN apt-get update \ + && apt-get install -y --no-install-recommends + +RUN apt-get install -y --no-install-recommends \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install requirements + +COPY requirements/dev.txt /tmp/requirements.txt +RUN pip install --upgrade setuptools pip wheel\ + && pip install pip-tools==6.10.0\ + && pip install -r /tmp/requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index af126d1..9d11ebd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -11,11 +11,9 @@ include CHANGELOG.rst exclude setup.cfg exclude pyproject.toml -recursive-include src/app_project/locale * -recursive-include src/app_project/templates * -recursive-include src/app_project/static * +recursive-include src/feeds/ * -recursive-exclude src/app_project/tests * +recursive-exclude src/feeds/tests * recursive-exclude * .cache recursive-exclude * __pycache__ diff --git a/Makefile b/Makefile index 576d25f..fe271d0 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,8 @@ PYTHON = python3.10 site_python := /usr/bin/env $(PYTHON) root_dir := $(realpath .) -app_dir = $(root_dir)/src/app_project -demo_dir = $(root_dir)/project +app_dir = $(root_dir)/src/feeds +demo_dir = $(root_dir)/demo venv_name = venv venv_dir = $(root_dir)/$(venv_name) @@ -133,7 +133,7 @@ clean: clean-venv clean-build clean-tests clean-mypy clean-coverage clean-docs $(venv_dir): $(site_python) -m venv $(venv_dir) $(pip) install --upgrade pip setuptools wheel - $(pip) install pip-tools==6.10.0 + $(pip) install pip-tools requirements/dev.txt: requirements/dev.in requirements/docs.in requirements/tests.in $(pip-compile) requirements/dev.in @@ -189,19 +189,19 @@ translations: .PHONY: flake8 flake8: - $(flake8) $(site_dir) + $(flake8) $(app_dir) .PHONY: isort isort: - $(isort) --check $(site_dir) + $(isort) --check $(app_dir) .PHONY: black black: - $(black) --check $(site_dir) + $(black) --check $(app_dir) .PHONY: mypy mypy: - $(mypy) $(site_dir) + $(mypy) $(app_dir) .PHONY: checks checks: flake8 black isort mypy @@ -212,7 +212,7 @@ checks: flake8 black isort mypy .PHONY: coverage coverage: - $(pytest) --cov=app_project --cov-config=setup.cfg --cov-report html + $(pytest) --cov=feeds --cov-config=setup.cfg --cov-report html .PHONY: test test: diff --git a/README.md b/README.md index 699ef4a..0b91714 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,26 @@ -# Django App Project +# Django Feeds -[![Build Status](https://img.shields.io/github/actions/workflow/status/StuartMacKay/django-app-template/ci.yml?branch=master)](https://github.com/StuartMacKay/django-app-template/actions/workflows/ci.yml?query=branch%3Amaster) +[![Build Status](https://img.shields.io/github/actions/workflow/status/StuartMacKay/django-feeds/ci.yml?branch=master)](https://github.com/StuartMacKay/django-feeds/actions/workflows/ci.yml?query=branch%3Amaster) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) -## Features +Django Feeds is an aggregator for RSS and Atom feeds. -* Development with [black](https://github.com/psf/black) so everybody gets the code formatting rules they deserve -* Development with [flake8](https://flake8.pycqa.org/en/latest/) so people using ed get syntax checking -* Development with [isort](https://pycqa.github.io/isort/) for automatically sorting imports -* Development with [mypy](https://mypy-lang.org/) for type-hinting to catch errors -* Testing with [pytest](https://docs.pytest.org/), [FactoryBoy](https://factoryboy.readthedocs.io/en/stable/) and [tox](https://tox.wiki/en/latest/) -* Manage versions with [bump2version](https://pypi.org/project/bump2version/) - for semantic version numbers -* Manage dependency versions with [pip-tools](https://github.com/jazzband/pip-tools) -* Manage dependency upgrades with [pip-upgrade](https://github.com/simion/pip-upgrader) +## Features -## Quick start +## Quick Start -First, download and unzip the project files in the directory of your choice. -Then rename the project to something more useful: -```shell -mv django-app-template django-myapp -``` +## Demo -Change to the project directory and start setting things up: -```shell -cd django-myapp -``` +If you clone or download the [django-feeds](https://github.com/StuartMacKay/django-feeds) +repository there is a demonstration application that lets you see how it +all works. -First, build the virtualenv and install all the dependencies. This will -also build the library: ```shell +git clone git@github.com:StuartMacKay/django-feeds.git +cd django-feeds make install -``` - -Now run the demo site: -```shell make demo -``` - - -Run the database migrations: -```shell -./manage.py migrate -``` - -Run the tests: -```shell -make tests -``` - -Run the django server: -```shell -./manage.py runserver -``` - -Open a browser and visit http://localhost:8000 and, voila, we have a working -site. Well cover the deployment later. - -Almost all steps used the project Makefile. That's great if you're running -Linux with GNU Make and not much fun if you're not. All is not lost, however. -All the Makefile's targets contain only one or two commands, so even if you -are running Windows you should still be able to get the site running without -too much effort. +``` +It's a standard django project so if you don't have `make` available +then just look at the [Makefile](Makefile) and run the commands from +the various targets. diff --git a/TODO.md b/TODO.md index b6dbda7..484e8f9 100644 --- a/TODO.md +++ b/TODO.md @@ -1,2 +1,21 @@ # TODO +The code runs and runs well. It has been in production for almost two +years, so it is rather polished. It was factored out of the +[Voices for Independence](https://www.voices.scot) web site with s small +number of changes and is now being repackaged to make it reusable in other +projects we have in mind. + +* replace the original bootstrap css classes with semantic names, so it + can be restyled easily. + +* move 'business logic' out of the views and out of the querysets into a + services layer - an internal API. That way you're not forced + to subclass the views, and you can easily write your own. + +* Add more blocks to the templates and create more snippets so again you + can easily replace what is provided in the app. + +* Also load the feeds using cron, so you are not forced to use celery. + +* Add a REST API - at some point. diff --git a/demo/__init__.py b/demo/__init__.py new file mode 100644 index 0000000..cf2e85f --- /dev/null +++ b/demo/__init__.py @@ -0,0 +1 @@ +from .celery import app as celery_app diff --git a/demo/apps.py b/demo/apps.py new file mode 100644 index 0000000..a536b9b --- /dev/null +++ b/demo/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class Config(AppConfig): + name = "demo" diff --git a/demo/conf/asgi.py b/demo/asgi.py similarity index 80% rename from demo/conf/asgi.py rename to demo/asgi.py index c0d7d61..6516f3b 100644 --- a/demo/conf/asgi.py +++ b/demo/asgi.py @@ -11,6 +11,6 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.conf.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings") application = get_asgi_application() diff --git a/demo/celery.py b/demo/celery.py new file mode 100644 index 0000000..eca5503 --- /dev/null +++ b/demo/celery.py @@ -0,0 +1,73 @@ +import logging +import os + +import structlog +from celery import Celery # type: ignore +from celery.schedules import crontab +from celery.signals import setup_logging # type: ignore +from django_structlog.celery.steps import DjangoStructLogInitStep # type: ignore + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings") + +app = Celery() + +# Load the configuration +app.config_from_object("demo.settings") +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() +# Configure workers to use structlog +app.steps["worker"].add(DjangoStructLogInitStep) + +# A celery task runs every hour, on the error and checked the schedule for +# each active feed. If the schedule, also a crontab, matches the current +# time then the feed is loaded. This two-level approach comes in useful for +# feeds that are either updated in frequently, so there's no need to cook +# the planet by checking every hour, or, dealing with throttling by services +# such as Cloudflare, where too many request s + +FEEDS_TASK_SCHEDULE = os.environ.get("FEEDS_TASK_SCHEDULE", "0 * * * *") +minute, hour, day_of_week, day_of_month, month_of_year = FEEDS_TASK_SCHEDULE.split() + +app.conf.beat_schedule = { + "load-feeds": { + "task": "demo.tasks.load_feeds", + "schedule": crontab( + minute=minute, + hour=hour, + day_of_week=day_of_week, + day_of_month=day_of_month, + month_of_year=month_of_year, + ), + }, + "daily-digest": { + "task": "feed.tasks.daily_digest", + "schedule": crontab(minute="0", hour="2"), # 2am, each day + }, +} + + +@setup_logging.connect +def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs): # noqa + from django.conf import settings + + config = settings.LOGGING + logging.config.dictConfig(config) # type: ignore + + # noinspection DuplicatedCode + structlog.configure( + processors=[ + structlog.stdlib.filter_by_level, + structlog.processors.TimeStamper(fmt="iso"), + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + structlog.stdlib.PositionalArgumentsFormatter(), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.processors.UnicodeDecoder(), + structlog.stdlib.ProcessorFormatter.wrap_for_formatter, + ], + context_class=structlog.threadlocal.wrap_dict(dict), + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, # type: ignore + cache_logger_on_first_use=True, + ) diff --git a/demo/conf/urls.py b/demo/conf/urls.py deleted file mode 100644 index 921ba72..0000000 --- a/demo/conf/urls.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -URL Configuration for the demonstration site. - -""" -from django.contrib import admin -from django.urls import path, include, reverse_lazy -from django.views.generic import RedirectView - -import debug_toolbar # type: ignore - - -urlpatterns = [ - path("", RedirectView.as_view(url=reverse_lazy("app_index"))), - path("admin/", admin.site.urls), - path("app/", include("app_project.urls")), - path("__debug__/toolbar/", include(debug_toolbar.urls)), # type: ignore -] diff --git a/demo/paginators.py b/demo/paginators.py new file mode 100644 index 0000000..c6c77cf --- /dev/null +++ b/demo/paginators.py @@ -0,0 +1,68 @@ +from datetime import timedelta +from math import ceil + +from django.core.paginator import Paginator +from django.utils import timezone +from django.utils.functional import cached_property + +from text_unidecode import unidecode # type: ignore + + +class DayPaginator(Paginator): + """ + The DayPaginator is used to show a week's worth of Articles on each + page, starting with today. In principle a page can be empty, though + one a number of Sources are active this is unlikely. + """ + + def page(self, number): + number = self.validate_number(number) + top = (number - 1) * self.per_page + bottom = top + self.per_page + tomorrow = timezone.now().replace(hour=0, minute=0, second=0, microsecond=0) + tomorrow += timedelta(days=1) + since = tomorrow - timedelta(days=bottom) + until = tomorrow - timedelta(days=top) + results = self.object_list.filter(date__gte=since, date__lt=until) + return self._get_page(results, number, self) + + @cached_property + def num_pages(self): + if self.count == 0: + return 0 + first = timezone.now() + last = self.object_list.last().date + return ceil((first - last).days / self.per_page) + + +class AlphabetPaginator(Paginator): + def __init__(self, object_list, per_page, orphans=0, allow_empty_first_page=True): + super().__init__(object_list, per_page, orphans, allow_empty_first_page) + + # Build an index of the first page on which each letter + # occurs. That allows a jump table to be displayed which + # takes the reader to the page where items starting with + # a given letter begin. + + self.index = {} + + attr_name = self.object_list.query.order_by[0] + + for idx, object in enumerate(self.object_list): + # Only index objects if the field is not blank + if value := str(getattr(object, attr_name)): + # normalise accented characters + letter = unidecode(value[0].upper()) + + if letter not in self.index: + page_number = int(idx / self.per_page) + 1 + self.index[letter] = page_number + + # Add the normalised letter to the object so a mini-heading + # can be displayed using the ifchanged template tag + object.index_letter = letter + + # Add the first index page where the letter occurs. That + # allows a 'continued' label to display at the top of the + # page + object.index_page = self.index[letter] diff --git a/demo/conf/settings.py b/demo/settings.py similarity index 56% rename from demo/conf/settings.py rename to demo/settings.py index 115b01e..5ec6ce6 100644 --- a/demo/conf/settings.py +++ b/demo/settings.py @@ -1,14 +1,12 @@ """ -MVS (Minimal Viable Settings) for running a basic site. +Demo site settings """ import os - DEBUG = True -CONF_DIR = os.path.dirname(os.path.abspath(__file__)) -DEMO_DIR = os.path.dirname(CONF_DIR) +DEMO_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(DEMO_DIR) SECRET_KEY = "+^%b!5_!ul7prtm_y0w_xluJg32aqna&+)&thhy3)jqr2g*0%s" @@ -21,7 +19,11 @@ "django.contrib.messages", "django.contrib.staticfiles", "debug_toolbar", - "app_project.apps.Config", + "django_celery_beat", + "django_extensions", + "tagulous", + "feeds.apps.Config", + "demo.apps.Config", ) MIDDLEWARE = [ @@ -30,22 +32,23 @@ "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", + "debug_toolbar.middleware.DebugToolbarMiddleware", ] -ROOT_URLCONF = "demo.conf.urls" +ROOT_URLCONF = "demo.urls" STATIC_URL = "/static/" TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [os.path.join(DEMO_DIR, "templates")], + "DIRS": [], "APP_DIRS": True, "OPTIONS": { "context_processors": [ "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", - "django.template.context_processors.request" + "django.template.context_processors.request", ], }, }, @@ -53,22 +56,33 @@ DATABASES = { "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": os.path.join(ROOT_DIR, "db.sqlite3"), + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": "feeds", + "USER": "feeds", + "PASSWORD": "feeds", + "HOST": os.environ["DB_HOST"], + "PORT": 5432, } } DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - "LOCATION": "site", - } -} - INTERNAL_IPS = [ "127.0.0.1", ] USE_TZ = True + +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "") + +CELERY_TASK_ALWAYS_EAGER = CELERY_BROKER_URL == "" + +CELERY_ACCEPT_CONTENT = ["json"] + +CELERY_WORKER_HIJACK_ROOT_LOGGER = False + +CELERY_TASK_ROUTES = { + "demo.tasks.load_feeds": {"delivery_mode": "transient"}, +} + +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" diff --git a/demo/static/images/rss.svg b/demo/static/images/rss.svg new file mode 100644 index 0000000..7bcbdac --- /dev/null +++ b/demo/static/images/rss.svg @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demo/static/js/index.js b/demo/static/js/index.js index e69de29..3319270 100644 --- a/demo/static/js/index.js +++ b/demo/static/js/index.js @@ -0,0 +1,23 @@ +function articleClick() { + 'use strict'; + let links = document.getElementsByClassName('article-link'); + for (let i = 0; i < links.length; i++) { + links[i].addEventListener('click', function(event) { + let xhttp = new XMLHttpRequest(); + let url = this.getAttribute("href"); + event.preventDefault(); + xhttp.open("POST", this.dataset.link, true); + xhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); + xhttp.send("article=" + this.dataset.pk); + setTimeout(function(){ + window.location.href = url; + }, 100); + }); + } +} + + +document.addEventListener('DOMContentLoaded', (event) => { + 'use strict'; + articleClick(); +}); diff --git a/demo/tasks.py b/demo/tasks.py new file mode 100644 index 0000000..3ad8eb2 --- /dev/null +++ b/demo/tasks.py @@ -0,0 +1,8 @@ +from feeds.services import feeds + +from .celery import app + + +@app.task() +def load_feeds() -> None: + feeds.load_feeds() diff --git a/demo/templates/demo/articles.html b/demo/templates/demo/articles.html new file mode 100644 index 0000000..88814e7 --- /dev/null +++ b/demo/templates/demo/articles.html @@ -0,0 +1,72 @@ +{% extends "base.html" %} +{% load feeds i18n %} + +{% block title %}{{ block.super }} - {% translate "Articles" %}{% endblock %} + +{% block sidebar %} + {% if tags %} +
+ {% block tags %} + {% translate "This week..." as title %} + {% include "demo/snippets/tag_cloud.html" %} + {% endblock %} +
+ {% endif %} +{% endblock %} + +{% block main %} +
+ {% for date, articles in object_groups.items %} +
+
+
{{ date|date:"M" }}
+
{{ date|date:"j" }}
+
+
    + {% for article in articles %} +
  • +
    + {{ article.title }} + {% for label in article|categories:"label" %} + {{ label.label|capfirst }} + {% endfor %} +
    + + {% if article.has_authors %} + {% for author in article.authors.all %} + {% if not forloop.first %} ,{% endif %}{{ author.name }} + {% endfor %} + - + {% endif %} + {{ article.source.name }} + {% if article.archive_url %} + - {% translate "Archived" %} + {% endif %} + {% if article.views %} + - {% blocktranslate count views=article.views %} + {{ views }} view + {% plural %} + {{ views }} views + {% endblocktranslate %} + {% endif %} + + {% if article.comment %} +
    {{ article.comment|safe }}
    + {% endif %} +
    • + {% endfor %} +
+
+ {% endfor %} + + {% block pagination %} + {% include "demo/snippets/pagination.html" with previous_page_title="Next Week" next_page_title="Previous Week"%} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/author.html b/demo/templates/demo/author.html new file mode 100644 index 0000000..a455bbf --- /dev/null +++ b/demo/templates/demo/author.html @@ -0,0 +1,68 @@ +{% extends "base.html" %} +{% load feeds i18n static %} + +{% block title %} + {% blocktranslate with name=author.name %} + Articles by {{ name }} + {% endblocktranslate %} +{% endblock %} + +{% block content %} +
+

{{ author.name }}

+ {% if author.description %} +

{{ author.description|safe }}

+ {% endif %} + + {% block tags %} + {% if tags %} + {% include "demo/snippets/tag_author_cloud.html" %} + {% endif %} + {% endblock %} + +
+ +
+
    + {% for article in page_obj.object_list %} +
  • +
    + {{ article.title }} + {% if show_labels %} + {% for label in article|categories:"label" %} + {{ label.label|capfirst }} + {% endfor %} + {% endif %} +
    + + {{ article.date|date:"jS M Y" }} - + {{ article.source.name }} + {% if article.archive_url %} + - {% translate "Archived" %} + {% endif %} + {% if article.views %} + - {% blocktranslate count views=article.views %} + {{ views }} view + {% plural %} + {{ views }} views + {% endblocktranslate %} + {% endif %} + + {% if article.comment %} +
    {{ article.comment|safe }}
    + {% endif %} +
    • + {% endfor %} +
+ + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/authors.html b/demo/templates/demo/authors.html new file mode 100644 index 0000000..4388234 --- /dev/null +++ b/demo/templates/demo/authors.html @@ -0,0 +1,55 @@ +{% extends "base.html" %} +{% load i18n %} + +{% block sidebar %} +
+

{% translate "Authors" %}

+
+ {% translate "Browse" %} +
+ {% block description %}{% endblock %} +
+{% endblock %} + +{% block main %} +
+
    + {% for letter, page_number in page_obj.paginator.index.items %} +
  • + {{ letter }} +
    • + {% endfor %} +
+ +
    + {% for object in page_obj.object_list %} +
  • + {% block object-detail %} + {% ifchanged object.index_letter %} +
    + {{ object.index_letter }} + {% if object.index_page < page_obj.number %} + ({% translate "continued" %}) + {% endif %} +
    + {% endifchanged %} +
    + {% if object.count %}{{ forloop.counter0|add:page_obj.start_index }}. {% endif %}{{ object.name }} +
    + + {{ object.article_count|default_if_none:"0" }}{{ " " }}{% if object.article_count == 1 %}{% translate "Post" %}{% else %}{% translate "Posts" %}{% endif %} + {% if object.article_count %} + - {% translate "Latest" %}: {{ object.articles.all.0.date|date:"jS M Y" }} + {% endif %} + + {% endblock %} +
    • + {% endfor %} +
+ + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/snippets/pagination.html b/demo/templates/demo/snippets/pagination.html new file mode 100644 index 0000000..e7e8844 --- /dev/null +++ b/demo/templates/demo/snippets/pagination.html @@ -0,0 +1,26 @@ +{% load i18n %} +{% if is_paginated %} +
+ +
+{% endif %} diff --git a/demo/templates/demo/snippets/tag_author_cloud.html b/demo/templates/demo/snippets/tag_author_cloud.html new file mode 100644 index 0000000..bc2099a --- /dev/null +++ b/demo/templates/demo/snippets/tag_author_cloud.html @@ -0,0 +1,13 @@ +{% load i18n %} +
+ {% translate "Topics covered..." as default_title %} + {% if title %}{{ title }}{% else %}{{ default_title }}{% endif %} + +
diff --git a/demo/templates/demo/snippets/tag_cloud.html b/demo/templates/demo/snippets/tag_cloud.html new file mode 100644 index 0000000..af5ad55 --- /dev/null +++ b/demo/templates/demo/snippets/tag_cloud.html @@ -0,0 +1,13 @@ +{% load i18n %} +
+ {% translate "Topics covered..." as default_title %} + {% if title %}{{ title }}{% else %}{{ default_title }}{% endif %} + +
diff --git a/demo/templates/demo/snippets/tag_source_cloud.html b/demo/templates/demo/snippets/tag_source_cloud.html new file mode 100644 index 0000000..e64c1b6 --- /dev/null +++ b/demo/templates/demo/snippets/tag_source_cloud.html @@ -0,0 +1,13 @@ +{% load i18n %} +
+ {% translate "Topics covered..." as default_title %} + {% if title %}{{ title }}{% else %}{{ default_title }}{% endif %} + +
diff --git a/demo/templates/demo/source.html b/demo/templates/demo/source.html new file mode 100644 index 0000000..ead70f2 --- /dev/null +++ b/demo/templates/demo/source.html @@ -0,0 +1,95 @@ +{% extends "base.html" %} +{% load feeds i18n static %} + +{% block title %}{% translate "Source" %}: {{ source.name }}{% endblock %} + +{% block content %} +
+

{{ source.name }}

+ {% if source.description %} +

{{ source.description|safe }}

+ {% endif %} + + {% block tags %} + {% if tags %} + {% include "demo/snippets/tag_source_cloud.html" %} + {% endif %} + {% endblock %} + +
+ {% if source.url or source.feed_set.all %} +

{% translate "Links" %}

+ {% endif %} +
+ {% if source.url %} + + + + + + {% endif %} + + {% for feed in source.feed_set.all %} +
+ + + + + +
+ {% endfor %} + +
+
+ +
+ +
+
    + {% for article in page_obj.object_list %} +
  • +
    + {{ article.title }} + {% if show_labels %} + {% for label in article|categories:"label" %} + {{ label.label|capfirst }} + {% endfor %} + {% endif %} +
    + + {{ article.date|date:"jS M Y" }} - + {% if article.has_authors %} + {% for author in article.authors.all %} + {% if not forloop.first %} ,{% endif %}{{ author.name }} + {% endfor %} + - + {% endif %} + {% if article.archive_url %} + {% translate "Archived" %} + - + {% endif %} + {% if article.views %} + {% blocktranslate count views=article.views %} + {{ views }} view + {% plural %} + {{ views }} views + {% endblocktranslate %} + {% endif %} + + {% if article.comment %} +
    {{ article.comment|safe }}
    + {% endif %} +
    • + {% endfor %} +
+ + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/sources.html b/demo/templates/demo/sources.html new file mode 100644 index 0000000..414b042 --- /dev/null +++ b/demo/templates/demo/sources.html @@ -0,0 +1,62 @@ +{% extends "base.html" %} +{% load i18n %} + +{% block title %}{% translate "Sources" %}{% endblock %} + +{% block sidebar %} +
+

{% translate "Sources" %}

+
+ {% translate "Browse" %} +
+
+ {% translate "Popular" %} +
+ {% block description %}{% endblock %} +
+{% endblock %} + +{% block main %} +
+
    + {% for letter, page_number in page_obj.paginator.index.items %} +
  • + {{ letter }} +
    • + {% endfor %} +
+ + {% block object-list %} +
    + {% for object in page_obj.object_list %} +
  • + {% block object-detail %} + {% ifchanged object.index_letter %} +
    + {{ object.index_letter }} + {% if object.index_page < page_obj.number %} + ({% translate "continued" %}) + {% endif %} +
    + {% endifchanged %} +
    + {{ object.name }} +
    + + {{ object.article_count|default_if_none:"0" }}{{ " " }}{% if object.article_count == 1 %}{% translate "Post" %}{% else %}{% translate "Posts" %}{% endif %} + {% if object.article_count %} + - {% translate "Latest" %}: {{ object.articles.all.0.date|date:"jS M Y" }} + {% endif %} + + {% endblock %} +
    • + {% endfor %} +
+ {% endblock %} + + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/tag.html b/demo/templates/demo/tag.html new file mode 100644 index 0000000..45703f7 --- /dev/null +++ b/demo/templates/demo/tag.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} +{% load feeds i18n %} + +{% block title %}{% translate "Tag" %}: {{ tag.name }}{% endblock %} + +{% block sidebar %} +
+

{{ tag.name }}

+ {% if tag.description %} +
+ {{ tag.description|safe }} +
+ {% endif %} + {% if tag.related.exists %} +

{% translate "Related Tags" %}

+
    + {% for tag in tag.related.all %} +
  • + {{ tag.name }} +
    • + {% endfor %} +
+ {% endif %} +
+{% endblock %} + +{% block main %} +
+
    + {% for article in page_obj.object_list %} +
  • +
    + {{ article.title }} + {% if show_labels %} + {% for label in article|categories:"label" %} + {{ label.label|capfirst }} + {% endfor %} + {% endif %} +
    + + {{ article.date|date:"jS M Y" }} - + {% if article.has_authors %} + {% for author in article.authors.all %} + {% if not forloop.first %} ,{% endif %}{{ author.name }} + {% endfor %} + - + {% endif %} + {{ article.source.name }} + {% if article.archive_url %} + - {% translate "Archived" %} + {% endif %} + {% if article.views %} + - {% blocktranslate count views=article.views %} + {{ views }} view + {% plural %} + {{ views }} views + {% endblocktranslate %} + {% endif %} + + {% if article.comment %} +
    {{ article.comment|safe }}
    + {% endif %} +
    • + {% endfor %} +
+ + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/templates/demo/tags.html b/demo/templates/demo/tags.html new file mode 100644 index 0000000..9458bfe --- /dev/null +++ b/demo/templates/demo/tags.html @@ -0,0 +1,57 @@ +{% extends "base.html" %} +{% load i18n %} + + {% block title %}{% translate "Tags" %}{% endblock %} + +{% block sidebar %} +
+

{% translate "Tags" %}

+
+ {% translate "Browse" %} +
+ {% block description %}{% endblock %} +
+{% endblock %} + +{% block main %} +
+
    + {% for letter, page_number in page_obj.paginator.index.items %} +
  • + {{ letter }} +
    • + {% endfor %} +
+ +
    + {% for object in page_obj.object_list %} +
  • + {% block object-detail %} + {% ifchanged object.index_letter %} +
    + {{ object.index_letter }} + {% if object.index_page < page_obj.number %} + ({% translate "continued" %}) + {% endif %} +
    + {% endifchanged %} +
    + {% if object.count %}{{ forloop.counter0|add:page_obj.start_index }}. {% endif %}{{ object.name }} +
    + + {{ object.article_count|default_if_none:"0" }}{{ " " }}{% if object.article_count == 1 %}{% translate "Post" %}{% else %}{% translate "Posts" %}{% endif %} + {% if object.article_count %} + - {% translate "Latest" %}: {{ object.article_set.all.0.date|date:"jS M Y" }} + {% endif %} + + {% endblock %} +
    • + {% endfor %} +
+ + {% block pagination %} + {% include "demo/snippets/pagination.html" %} + {% endblock %} + +
+{% endblock %} diff --git a/demo/tests/test_article_view.py b/demo/tests/test_article_view.py new file mode 100644 index 0000000..b8da608 --- /dev/null +++ b/demo/tests/test_article_view.py @@ -0,0 +1,27 @@ +import pytest +from django.urls import reverse + +from feeds.tests.conditions.articles import article_was_clicked +from feeds.tests.conditions.responses import redirects_to +from feeds.tests.factories import ArticleFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def article(): + return ArticleFactory() + + +def test_view(client, article): + response = client.get( + reverse("article", kwargs={"code": article.code}), follow=False + ) + assert redirects_to(response, article.url) + assert article_was_clicked(article) + + +def test_click_view(client, article): + response = client.post(reverse("article-click"), data={"code": article.code}) + assert response.status_code == 204 + assert article_was_clicked(article) diff --git a/demo/tests/test_articles_view.py b/demo/tests/test_articles_view.py new file mode 100644 index 0000000..815f91d --- /dev/null +++ b/demo/tests/test_articles_view.py @@ -0,0 +1,53 @@ +import datetime as dt + +from django.urls import reverse +from django.utils import timezone + +import pytest + +from feeds.tests.conditions.articles import ( + only_articles_between_dates, + only_published_articles, + only_tags_for_articles, +) +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.factories import ( + ArticleFactory, + AuthorFactory, + CategoryFactory, + TagFactory, +) + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def url(): + return reverse("articles") + + +@pytest.fixture +def articles(): + CategoryFactory.create(name="label/repost") + CategoryFactory.create(name="label/updated") + TagFactory.create_batch(20) + AuthorFactory.create_batch(10) + return ArticleFactory.create_batch(100) + + +def test_get(client, url, articles): + response = client.get(url) + articles = response.context["object_list"] + tags = response.context["tags"] + end_date = dt.date.today() + start_date = end_date - dt.timedelta(days=6) + assert is_paginated(response) + assert is_ordered(response) + assert only_published_articles(articles, timezone.now()) + assert only_tags_for_articles(articles, tags) + assert only_articles_between_dates(articles, start_date, end_date) + + +def test_query_count(client, url, articles, django_assert_num_queries): + with django_assert_num_queries(8): + client.get(url) diff --git a/demo/tests/test_author_view.py b/demo/tests/test_author_view.py new file mode 100644 index 0000000..3ab62e9 --- /dev/null +++ b/demo/tests/test_author_view.py @@ -0,0 +1,61 @@ +from django.urls import reverse +from django.utils import timezone + +import pytest + +from feeds.tests.conditions.articles import ( + only_published_articles, + only_tags_for_articles, +) +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.conditions.responses import redirects_to, warning_message_shown +from feeds.tests.factories import ( + ArticleFactory, + AuthorFactory, + CategoryFactory, + SourceFactory, + TagFactory, +) + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def author(): + return AuthorFactory() + + +@pytest.fixture +def url(author): + return reverse("author", args=(author.slug,)) + + +@pytest.fixture +def articles(author): + CategoryFactory.create(name="label/repost") + CategoryFactory.create(name="label/updated") + TagFactory.create_batch(20) + SourceFactory.create_batch(20) + return ArticleFactory.create_batch(100, authors=[author]) + + +def test_get(client, url, author, articles): + response = client.get(url) + articles = response.context["object_list"] + tags = response.context["tags"] + assert is_paginated(response) + assert is_ordered(response) + assert only_published_articles(articles, timezone.now()) + assert only_tags_for_articles(articles, tags) + + +def test_errors(client): + response = client.get(reverse("author", args=("no-such-author",)), follow=True) + assert response.status_code == 200 + assert redirects_to(response, reverse("authors")) + assert warning_message_shown(response) + + +def test_query_count(client, url, articles, django_assert_num_queries): + with django_assert_num_queries(4): + client.get(url) diff --git a/demo/tests/test_authors_view.py b/demo/tests/test_authors_view.py new file mode 100644 index 0000000..9372afb --- /dev/null +++ b/demo/tests/test_authors_view.py @@ -0,0 +1,31 @@ +from django.urls import reverse + +import pytest + +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.factories import ArticleFactory, AuthorFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def url(): + return reverse("authors") + + +@pytest.fixture +def authors(): + authors = AuthorFactory.create_batch(50) + ArticleFactory.create_batch(100) + return authors + + +def test_get(client, url, authors): + response = client.get(url) + assert is_paginated(response) + assert is_ordered(response) + + +def test_query_count(client, url, authors, django_assert_num_queries): + with django_assert_num_queries(2): + client.get(url) diff --git a/demo/tests/test_source_view.py b/demo/tests/test_source_view.py new file mode 100644 index 0000000..b51f7e7 --- /dev/null +++ b/demo/tests/test_source_view.py @@ -0,0 +1,61 @@ +from django.urls import reverse +from django.utils import timezone + +import pytest + +from feeds.tests.conditions.articles import ( + only_published_articles, + only_tags_for_articles, +) +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.conditions.responses import redirects_to, warning_message_shown +from feeds.tests.factories import ( + ArticleFactory, + AuthorFactory, + CategoryFactory, + SourceFactory, + TagFactory, +) + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def source(): + return SourceFactory() + + +@pytest.fixture +def url(source): + return reverse("source", args=(source.slug,)) + + +@pytest.fixture +def articles(source): + CategoryFactory.create(name="label/repost") + CategoryFactory.create(name="label/updated") + TagFactory.create_batch(20) + AuthorFactory.create_batch(10) + return ArticleFactory.create_batch(100, source=source) + + +def test_get(client, url, articles): + response = client.get(url) + articles = response.context["object_list"] + tags = response.context["tags"] + assert is_paginated(response) + assert is_ordered(response) + assert only_published_articles(articles, timezone.now()) + assert only_tags_for_articles(articles, tags) + + +def test_errors(client): + response = client.get(reverse("source", args=("no-such-source",)), follow=True) + assert response.status_code == 200 + assert redirects_to(response, reverse("sources")) + assert warning_message_shown(response) + + +def test_query_count(client, url, articles, django_assert_num_queries): + with django_assert_num_queries(6): + client.get(url) diff --git a/demo/tests/test_sources_view.py b/demo/tests/test_sources_view.py new file mode 100644 index 0000000..7682fb0 --- /dev/null +++ b/demo/tests/test_sources_view.py @@ -0,0 +1,31 @@ +from django.urls import reverse + +import pytest + +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.factories import SourceFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def url(): + return reverse("sources") + + +@pytest.fixture +def sources(): + return SourceFactory.create_batch(50) + + +def test_get(client, url, sources): + response = client.get(url) + assert is_paginated(response) + assert is_ordered(response) + + +def test_query_count(client, url, sources, django_assert_num_queries): + # Includes a query to fetch the current site though this + # query does not appear in the debug toolbar + with django_assert_num_queries(2): + client.get(url) diff --git a/demo/tests/test_tag_view.py b/demo/tests/test_tag_view.py new file mode 100644 index 0000000..6fe554b --- /dev/null +++ b/demo/tests/test_tag_view.py @@ -0,0 +1,75 @@ +from django.urls import reverse +from django.utils import timezone + +import pytest + +from feeds.models import Author, Source +from feeds.tests.conditions.articles import only_published_articles +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.conditions.responses import redirects_to, warning_message_shown +from feeds.tests.factories import ( + ArticleFactory, + AuthorFactory, + CategoryFactory, + SourceFactory, + TagFactory, +) + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def tag(): + return TagFactory() + + +@pytest.fixture +def url(tag): + return reverse("tag", args=(tag.slug,)) + + +@pytest.fixture +def articles(tag): + CategoryFactory.create(name="label/repost") + CategoryFactory.create(name="label/updated") + SourceFactory.create_batch(20) + AuthorFactory.create_batch(10) + return ArticleFactory.create_batch(100, tags=[tag]) + + +def test_get(client, url, articles): + response = client.get(url) + articles = response.context["object_list"] + assert is_paginated(response) + assert is_ordered(response) + assert only_published_articles(articles, timezone.now()) + + +def test_errors(client): + response = client.get(reverse("tag", args=("no-such-tag",)), follow=True) + assert response.status_code == 200 + assert redirects_to(response, reverse("tags")) + assert warning_message_shown(response) + + +def test_tag_for_author(client, tag, articles): + author = Author.objects.first() + url = reverse("tag-author", args=(tag.slug, author.slug)) + response = client.get(url) + expected = [obj.pk for obj in author.articles.published()] + actual = [obj.pk for obj in response.context["object_list"]] + assert sorted(actual) == sorted(expected) + + +def test_tag_for_source(client, tag, articles): + source = Source.objects.first() + url = reverse("tag-source", args=(tag.slug, source.slug)) + response = client.get(url) + expected = [obj.pk for obj in source.articles.published()] + actual = [obj.pk for obj in response.context["object_list"]] + assert sorted(actual) == sorted(expected) + + +def test_query_count(client, url, articles, django_assert_num_queries): + with django_assert_num_queries(2): + client.get(url) diff --git a/demo/tests/test_tags_view.py b/demo/tests/test_tags_view.py new file mode 100644 index 0000000..4143e6f --- /dev/null +++ b/demo/tests/test_tags_view.py @@ -0,0 +1,31 @@ +from django.urls import reverse + +import pytest + +from feeds.tests.conditions.querysets import is_ordered, is_paginated +from feeds.tests.factories import TagFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def url(): + return reverse("tags") + + +@pytest.fixture +def tags(): + return TagFactory.create_batch(100) + + +def test_get(client, url, tags): + response = client.get(url) + assert is_paginated(response) + assert is_ordered(response) + + +def test_query_count(client, url, tags, django_assert_num_queries): + # Includes a query to fetch the current site though this + # query does not appear in the debug toolbar + with django_assert_num_queries(2): + client.get(url) diff --git a/demo/urls.py b/demo/urls.py new file mode 100644 index 0000000..0d29c73 --- /dev/null +++ b/demo/urls.py @@ -0,0 +1,72 @@ +from django.urls import path + +""" +URL Configuration for the demonstration site. + +""" +from django.contrib import admin +from django.urls import include, path + +import debug_toolbar # type: ignore + +from demo import views + +urlpatterns = [ + path( + "", + views.ArticlesView.as_view(), + name="articles", + ), + path( + "article/click/", + views.ArticleClickView.as_view(), + name="article-click", + ), + path( + "article//", + views.ArticleView.as_view(), + name="article", + ), + path( + "author/", + views.AuthorsView.as_view(), + name="authors", + ), + path( + "author//", + views.AuthorView.as_view(), + name="author", + ), + path( + "source/", + views.SourcesView.as_view(), + name="sources", + ), + path( + "source//", + views.SourceView.as_view(), + name="source", + ), + path( + "tag/", + views.TagsView.as_view(), + name="tags", + ), + path( + "tag//", + views.TagView.as_view(), + name="tag", + ), + path( + "tag//author//", + views.TagView.as_view(), + name="tag-author", + ), + path( + "tag//source//", + views.TagView.as_view(), + name="tag-source", + ), + path("admin/", admin.site.urls), + path("__debug__/toolbar/", include(debug_toolbar.urls)), # type: ignore +] diff --git a/demo/views/__init__.py b/demo/views/__init__.py new file mode 100644 index 0000000..342de67 --- /dev/null +++ b/demo/views/__init__.py @@ -0,0 +1,20 @@ +from .article import ArticleClickView, ArticleView +from .articles import ArticlesView +from .author import AuthorView +from .authors import AuthorsView +from .source import SourceView +from .sources import SourcesView +from .tag import TagView +from .tags import TagsView + +__all__ = ( + "ArticleView", + "ArticleClickView", + "AuthorView", + "AuthorsView", + "ArticlesView", + "SourceView", + "SourcesView", + "TagView", + "TagsView", +) diff --git a/demo/views/article.py b/demo/views/article.py new file mode 100644 index 0000000..d4d5ded --- /dev/null +++ b/demo/views/article.py @@ -0,0 +1,29 @@ +from django.http import HttpResponse +from django.utils.decorators import method_decorator +from django.views import View, generic +from django.views.decorators.csrf import csrf_exempt + +from feeds.models import Article + + +class ArticleView(generic.RedirectView): + # In the off chance there are Articles with duplicate codes return the + # url for the latest once since that's the one most likely to have been + # clicked. If no such Article exist then return None so the RedirectView + # responds with 410 Gone. + + def get_redirect_url(self, *args, **kwargs): + Article.objects.viewed(kwargs["code"]) + if article := Article.objects.with_code(kwargs["code"]): + return article.url + + +@method_decorator(csrf_exempt, name="dispatch") +class ArticleClickView(View): + # Record outbound link clicks via POST so no trail on who clicked what + # is left in the web server logs. + + def post(self, request): + code = request.POST.get("code") + Article.objects.viewed(code) + return HttpResponse(status=204) diff --git a/demo/views/articles.py b/demo/views/articles.py new file mode 100644 index 0000000..dbaff75 --- /dev/null +++ b/demo/views/articles.py @@ -0,0 +1,37 @@ +from django.views import generic + +from demo.paginators import DayPaginator +from demo.views.utils import shuffle +from feeds.models import Article, Tag + + +class ArticlesView(generic.ListView): + template_name = "demo/articles.html" + paginator_class = DayPaginator + paginate_by = 7 + + def get_queryset(self): + return ( + Article.objects.published() + .select_related("source") + .prefetch_related("authors", "categories") + .order_by("-date") + ) + + def get_groups_by_date(self, object_list): # noqa + groups = {} + for article in object_list: + date = article.date.date() + groups.setdefault(date, []) + groups[date].append(article) + return groups + + def get_tags(self, object_list): # noqa + return shuffle(Tag.objects.for_articles(object_list).weighted()) + + def get_context_data(self, *, object_list=None, **kwargs): + context = super().get_context_data(**kwargs) + objects = context["object_list"] + context["tags"] = self.get_tags(objects) + context["object_groups"] = self.get_groups_by_date(objects) + return context diff --git a/demo/views/author.py b/demo/views/author.py new file mode 100644 index 0000000..251da41 --- /dev/null +++ b/demo/views/author.py @@ -0,0 +1,44 @@ +from typing import List + +from django.contrib import messages +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ +from django.views import generic + +from demo.views.utils import shuffle +from feeds.models import Article, Author, Tag + + +class AuthorView(generic.ListView): + template_name = "demo/author.html" + paginate_by = 20 + + def get_queryset(self): + return ( + Article.objects.published() + .by_author(self.kwargs["slug"]) + .select_related("source") + .order_by("-date") + ) + + def get_author(self) -> Author: + return Author.objects.get(slug=self.kwargs["slug"]) + + def get_tags(self, articles) -> List[Tag]: + return shuffle(Tag.objects.for_articles(articles).weighted()) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["tags"] = self.get_tags(context["object_list"]) + return context + + def get(self, request, *args, **kwargs): + try: + author = self.get_author() + except Author.DoesNotExist: + messages.warning(request, _("An author with this name could not be found")) + return redirect(reverse("authors")) + self.object_list = self.get_queryset() + context = self.get_context_data(author=author) + return self.render_to_response(context) diff --git a/demo/views/authors.py b/demo/views/authors.py new file mode 100644 index 0000000..a72a4f1 --- /dev/null +++ b/demo/views/authors.py @@ -0,0 +1,21 @@ +from django.db.models import Count, Prefetch +from django.views import generic + +from demo.paginators import AlphabetPaginator +from feeds.models import Article, Author + + +class AuthorsView(generic.ListView): + template_name = "demo/authors.html" + paginator_class = AlphabetPaginator + paginate_by = 25 + + def get_queryset(self): + return ( + Author.objects.published() + .order_by("name") + .annotate(article_count=Count("articles")) + .prefetch_related( + Prefetch("articles", queryset=Article.objects.all().order_by("-date")) + ) + ) diff --git a/demo/views/source.py b/demo/views/source.py new file mode 100644 index 0000000..8e35751 --- /dev/null +++ b/demo/views/source.py @@ -0,0 +1,45 @@ +from typing import List + +from django.contrib import messages +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ +from django.views import generic + +from demo.views.utils import shuffle +from feeds.models import Article, Source, Tag + + +class SourceView(generic.ListView): + template_name = "demo/source.html" + paginate_by = 20 + + def get_queryset(self): + return ( + Article.objects.published() + .by_source(self.kwargs["slug"]) + .select_related("source") + .prefetch_related("authors") + .order_by("-date") + ) + + def get_source(self) -> Source: + return Source.objects.get(slug=self.kwargs["slug"]) + + def get_tags(self, articles) -> List[Tag]: + return shuffle(Tag.objects.for_articles(articles).weighted()) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["tags"] = self.get_tags(context["object_list"]) + return context + + def get(self, request, *args, **kwargs): + try: + source = self.get_source() + except Source.DoesNotExist: + messages.warning(request, _("A site with this name could not be found")) + return redirect(reverse("sources")) + self.object_list = self.get_queryset() + context = self.get_context_data(source=source) + return self.render_to_response(context) diff --git a/demo/views/sources.py b/demo/views/sources.py new file mode 100644 index 0000000..2d70dff --- /dev/null +++ b/demo/views/sources.py @@ -0,0 +1,24 @@ +from django.db.models import Count, Prefetch +from django.views import generic + +from demo.paginators import AlphabetPaginator +from feeds.models import Article, Source + + +class SourcesView(generic.ListView): + template_name = "demo/sources.html" + paginator_class = AlphabetPaginator + paginate_by = 25 + + def get_queryset(self): + return ( + Source.objects.all() + .order_by("name") + .annotate(article_count=Count("articles")) + .prefetch_related( + Prefetch( + "articles", + queryset=Article.objects.published().order_by("-date"), + ) + ) + ) diff --git a/demo/views/tag.py b/demo/views/tag.py new file mode 100644 index 0000000..60293cd --- /dev/null +++ b/demo/views/tag.py @@ -0,0 +1,44 @@ +from django.contrib import messages +from django.db.models import Q +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ +from django.views import generic + +from feeds.models import Article, Tag + + +class TagView(generic.ListView): + template_name = "demo/tag.html" + paginate_by = 20 + + def get_queryset(self): + return ( + Article.objects.published() + .by_tag(self.kwargs["slug"]) + .filter(self.get_filters()) + .select_related("source") + .prefetch_related("authors") + .order_by("-date") + ) + + def get_filters(self): + filters = Q() + if author := self.kwargs.get("author", ""): # noqa + filters &= Q(authors__slug=author.strip()) + if source := self.kwargs.get("source", ""): # noqa + filters &= Q(source__slug=source.strip()) + return filters + + def get_tag(self): + return Tag.objects.get(slug=self.kwargs["slug"]) + + def get(self, request, *args, **kwargs): + try: + tag = self.get_tag() + except Tag.DoesNotExist: + messages.warning(request, _("A tag with this title could not be found")) + return redirect(reverse("tags")) + self.object_list = self.get_queryset() + context = self.get_context_data(tag=tag) + return self.render_to_response(context) diff --git a/demo/views/tags.py b/demo/views/tags.py new file mode 100644 index 0000000..a703548 --- /dev/null +++ b/demo/views/tags.py @@ -0,0 +1,23 @@ +from django.db.models import Count, Prefetch +from django.views import generic + +from demo.paginators import AlphabetPaginator +from feeds.models import Article, Tag + + +class TagsView(generic.ListView): + template_name = "demo/tags.html" + paginator_class = AlphabetPaginator + paginate_by = 25 + + def get_queryset(self): + return ( + Tag.objects.all() + .order_by("name") + .annotate(article_count=Count("article")) + .prefetch_related( + Prefetch( + "article_set", queryset=Article.objects.all().order_by("-date") + ) + ) + ) diff --git a/demo/views/utils.py b/demo/views/utils.py new file mode 100644 index 0000000..48b9daa --- /dev/null +++ b/demo/views/utils.py @@ -0,0 +1,7 @@ +import random + + +def shuffle(objects): + random.seed(random.randint(1, 10000)) + random.shuffle(objects) + return objects diff --git a/demo/conf/wsgi.py b/demo/wsgi.py similarity index 80% rename from demo/conf/wsgi.py rename to demo/wsgi.py index 062381a..0a0e4d0 100644 --- a/demo/conf/wsgi.py +++ b/demo/wsgi.py @@ -11,6 +11,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.conf.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings") application = get_wsgi_application() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5721165 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,58 @@ +version: '3.1' + +services: + db: + image: postgres:15 + ports: + - "15432:5432" + volumes: + - ${DB_DATA_DIR-./.data/db}:/var/lib/postgresql/data + env_file: + - .env.docker + + broker: + image: rabbitmq:3.11.9-management + env_file: + - .env.docker + + web: + stdin_open: true + tty: true + build: . + command: /code/docker-init.sh + volumes: &web-volumes + - ./:/code + ports: + - "8000:8000" + depends_on: &web-depends-on + - db + - cache + - broker + env_file: + - .env.docker + + flower: + image: mher/flower + ports: + - "15555:5555" + command: celery flower --broker_api=http://feeds:feeds@broker:15672/api/feeds + depends_on: + - broker + env_file: + - .env.docker + + celery-beat: + build: . + command: celery --app feeds beat --loglevel debug + volumes: *web-volumes + depends_on: *web-depends-on + env_file: + - .env.docker + + celery-worker: + build: . + command: celery --app feeds worker --loglevel debug + volumes: *web-volumes + depends_on: *web-depends-on + env_file: + - .env.docker diff --git a/docker-init.sh b/docker-init.sh new file mode 100755 index 0000000..adc409a --- /dev/null +++ b/docker-init.sh @@ -0,0 +1,3 @@ +#!/bin/bash +python /code/manage.py migrate +python /code/manage.py runserver 0.0.0.0:8000 diff --git a/manage.py b/manage.py index 13e3aa2..86cc0b0 100755 --- a/manage.py +++ b/manage.py @@ -2,9 +2,8 @@ import os import sys - if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.conf.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings") from django.core.management import execute_from_command_line diff --git a/requirements/dev.in b/requirements/dev.in index 09d518d..a62cc84 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -8,6 +8,14 @@ -e . +# Packages required by the demo site + +bs4 +celery +django-celery-beat +requests +text_unidecode + # bumpversion is no longer maintained so we use bump2version until the # point where bump2version emerges as the official bumpversion, see # https://github.com/c4urself/bump2version/issues/86 diff --git a/requirements/dev.txt b/requirements/dev.txt index 7d6131a..c039ab9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,69 +1,130 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile requirements/dev.in # --e file:///home/stuart/Development/django-app-template +-e file:///home/stuart/Development/django-feeds # via -r requirements/dev.in alabaster==0.7.13 # via sphinx -asgiref==3.6.0 - # via django -attrs==22.2.0 - # via pytest +amqp==5.1.1 + # via kombu +asgiref==3.7.2 + # via + # django + # django-structlog babel==2.12.1 # via sphinx -black==23.1.0 +beautifulsoup4==4.12.2 + # via bs4 +billiard==4.1.0 + # via celery +black==23.9.1 # via -r requirements/tests.in -bleach==6.0.0 - # via readme-renderer +bs4==0.0.1 + # via -r requirements/dev.in bump2version==1.0.1 # via -r requirements/dev.in -cachetools==5.3.0 +cachetools==5.3.1 # via tox -certifi==2022.12.7 +celery==5.3.4 + # via + # -r requirements/dev.in + # django-celery-beat + # django-structlog +certifi==2023.7.22 # via requests cffi==1.15.1 # via cryptography -chardet==5.1.0 +chardet==5.2.0 # via tox -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests -click==8.1.3 - # via black +click==8.1.7 + # via + # black + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.0 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery colorama==0.4.6 # via tox -coverage[toml]==7.2.1 +colorclass==2.2.2 + # via pip-upgrader +coverage[toml]==7.3.1 # via pytest-cov -cryptography==39.0.2 +cron-descriptor==1.4.0 + # via django-celery-beat +croniter==1.4.1 + # via django-rss-feeds +cryptography==41.0.4 # via secretstorage -distlib==0.3.6 +distlib==0.3.7 # via virtualenv -django==4.1.7 - # via django-library-project +django==3.2.21 + # via + # django-celery-beat + # django-debug-toolbar + # django-extensions + # django-rss-feeds + # django-structlog + # django-stubs + # django-stubs-ext + # django-tagulous + # django-timezone-field +django-celery-beat==2.5.0 + # via -r requirements/dev.in +django-debug-toolbar==4.2.0 + # via -r requirements/dev.in +django-extensions==3.2.3 + # via django-rss-feeds +django-ipware==5.0.0 + # via django-structlog +django-structlog[celery]==5.3.0 + # via django-rss-feeds +django-stubs==4.2.4 + # via -r requirements/dev.in +django-stubs-ext==4.2.2 + # via django-stubs +django-tagulous==1.3.3 + # via django-rss-feeds +django-timezone-field==6.0.1 + # via django-celery-beat +docopt==0.6.2 + # via pip-upgrader docutils==0.18.1 # via # readme-renderer # sphinx # sphinx-rtd-theme -exceptiongroup==1.1.0 +exceptiongroup==1.1.3 # via pytest -factory-boy==3.2.1 +factory-boy==3.3.0 # via pytest-factoryboy -faker==17.6.0 +faker==19.6.2 # via factory-boy -filelock==3.9.0 +feedparser==6.0.10 + # via django-rss-feeds +filelock==3.12.4 # via # tox # virtualenv -flake8==6.0.0 +flake8==6.1.0 # via -r requirements/tests.in +freezegun==1.2.2 + # via pytest-freezegun idna==3.4 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==6.0.0 +importlib-metadata==6.8.0 # via # keyring # twine @@ -73,7 +134,7 @@ iniconfig==2.0.0 # via pytest isort==5.12.0 # via -r requirements/tests.in -jaraco-classes==3.2.3 +jaraco-classes==3.3.0 # via keyring jeepney==0.8.0 # via @@ -81,143 +142,199 @@ jeepney==0.8.0 # secretstorage jinja2==3.1.2 # via sphinx -keyring==23.13.1 +keyring==24.2.0 # via twine -lack==0.0.0 - # via -r requirements/tests.in -markdown-it-py==2.2.0 +kombu==5.3.2 + # via celery +markdown-it-py==3.0.0 # via rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via jinja2 mccabe==0.7.0 # via flake8 mdurl==0.1.2 # via markdown-it-py -more-itertools==9.1.0 +more-itertools==10.1.0 # via jaraco-classes -mypy==1.1.1 - # via -r requirements/tests.in +mypy==1.5.1 + # via + # -r requirements/tests.in + # django-stubs mypy-extensions==1.0.0 # via # black # mypy -packaging==23.0 +nh3==0.2.14 + # via readme-renderer +packaging==23.1 # via # black + # pip-upgrader # pyproject-api # pytest # sphinx # tox -pathspec==0.11.0 - # via - # black - # twine +pathspec==0.11.2 + # via black +pip-upgrader==1.4.15 + # via -r requirements/dev.in pkginfo==1.9.6 # via twine -platformdirs==3.1.1 +platformdirs==3.10.0 # via # black # tox # virtualenv -pluggy==1.0.0 +pluggy==1.3.0 # via # pytest # tox -pycodestyle==2.10.0 +prompt-toolkit==3.0.39 + # via click-repl +psycopg2-binary==2.9.7 + # via django-rss-feeds +pycodestyle==2.11.0 # via flake8 pycparser==2.21 # via cffi -pyflakes==3.0.1 +pyflakes==3.1.0 # via flake8 -pygments==2.14.0 +pygments==2.16.1 # via # readme-renderer # rich # sphinx -pyproject-api==1.5.0 +pyproject-api==1.6.1 # via tox -pytest==7.2.2 +pytest==7.4.2 # via # -r requirements/tests.in # pytest-cov # pytest-django # pytest-factoryboy -pytest-cov==4.0.0 + # pytest-freezegun +pytest-cov==4.1.0 # via -r requirements/tests.in pytest-django==4.5.2 # via -r requirements/tests.in pytest-factoryboy==2.5.1 # via -r requirements/tests.in +pytest-freezegun==0.4.2 + # via -r requirements/tests.in +python-crontab==3.0.0 + # via django-celery-beat python-dateutil==2.8.2 - # via faker -readme-renderer==37.3 + # via + # celery + # croniter + # django-rss-feeds + # faker + # freezegun + # python-crontab +pytz==2023.3.post1 + # via django +readme-renderer==42.0 # via twine -requests==2.28.2 +requests==2.31.0 # via + # -r requirements/dev.in + # pip-upgrader # requests-toolbelt # sphinx # twine -requests-toolbelt==0.10.1 +requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.3.2 +rich==13.5.3 # via twine secretstorage==3.3.3 # via keyring +sgmllib3k==1.0.0 + # via feedparser six==1.16.0 - # via - # bleach - # python-dateutil + # via python-dateutil snowballstemmer==2.2.0 # via sphinx -sphinx==6.1.3 +soupsieve==2.5 + # via beautifulsoup4 +sphinx==7.2.6 # via # -r requirements/docs.in # sphinx-rtd-theme -sphinx-rtd-theme==1.2.0 + # sphinxcontrib-applehelp + # sphinxcontrib-devhelp + # sphinxcontrib-htmlhelp + # sphinxcontrib-jquery + # sphinxcontrib-qthelp + # sphinxcontrib-serializinghtml +sphinx-rtd-theme==1.3.0 # via -r requirements/docs.in -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==1.0.7 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==1.0.5 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.0.4 # via sphinx -sphinxcontrib-jquery==2.0.0 +sphinxcontrib-jquery==4.1 # via sphinx-rtd-theme sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==1.0.6 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==1.1.9 # via sphinx -sqlparse==0.4.3 - # via django +sqlparse==0.4.4 + # via + # django + # django-debug-toolbar +structlog==23.1.0 + # via django-structlog +terminaltables==3.1.10 + # via pip-upgrader +text-unidecode==1.3 + # via -r requirements/dev.in tomli==2.0.1 # via + # black # coverage + # django-stubs # mypy # pyproject-api # pytest # tox -tox==4.4.6 +tox==4.11.3 # via -r requirements/dev.in twine==4.0.2 # via -r requirements/dev.in -typing-extensions==4.5.0 +types-pytz==2023.3.1.1 + # via django-stubs +types-pyyaml==6.0.12.11 + # via django-stubs +typing-extensions==4.8.0 # via + # asgiref + # black + # django-stubs + # django-stubs-ext # mypy # pytest-factoryboy -urllib3==1.26.14 +tzdata==2023.3 + # via + # celery + # django-celery-beat +urllib3==2.0.5 # via # requests # twine -virtualenv==20.20.0 +vine==5.0.0 + # via + # amqp + # celery + # kombu +virtualenv==20.24.5 # via tox -webencodings==0.5.1 - # via bleach -zipp==3.15.0 +wcwidth==0.2.6 + # via prompt-toolkit +zipp==3.17.0 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/tests.in b/requirements/tests.in index ecd5d57..5c82ef0 100644 --- a/requirements/tests.in +++ b/requirements/tests.in @@ -6,4 +6,5 @@ mypy pytest pytest-django pytest-factoryboy +pytest-freezegun pytest-cov diff --git a/requirements/tests.txt b/requirements/tests.txt index 6fb9879..6e2b428 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,25 +1,25 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile requirements/tests.in # -attrs==22.2.0 - # via pytest -black==23.1.0 +black==23.9.1 # via -r requirements/tests.in -click==8.1.3 +click==8.1.7 # via black -coverage[toml]==7.2.1 +coverage[toml]==7.3.1 # via pytest-cov -exceptiongroup==1.1.0 +exceptiongroup==1.1.3 # via pytest -factory-boy==3.2.1 +factory-boy==3.3.0 # via pytest-factoryboy -faker==17.6.0 +faker==19.6.2 # via factory-boy -flake8==6.0.0 +flake8==6.1.0 # via -r requirements/tests.in +freezegun==1.2.2 + # via pytest-freezegun inflection==0.5.1 # via pytest-factoryboy iniconfig==2.0.0 @@ -28,40 +28,45 @@ isort==5.12.0 # via -r requirements/tests.in mccabe==0.7.0 # via flake8 -mypy==1.1.1 +mypy==1.5.1 # via -r requirements/tests.in mypy-extensions==1.0.0 # via # black # mypy -packaging==23.0 +packaging==23.1 # via # black # pytest -pathspec==0.11.0 +pathspec==0.11.2 # via black -platformdirs==3.1.1 +platformdirs==3.10.0 # via black -pluggy==1.0.0 +pluggy==1.3.0 # via pytest -pycodestyle==2.10.0 +pycodestyle==2.11.0 # via flake8 -pyflakes==3.0.1 +pyflakes==3.1.0 # via flake8 -pytest==7.2.2 +pytest==7.4.2 # via # -r requirements/tests.in # pytest-cov # pytest-django # pytest-factoryboy -pytest-cov==4.0.0 + # pytest-freezegun +pytest-cov==4.1.0 # via -r requirements/tests.in pytest-django==4.5.2 # via -r requirements/tests.in pytest-factoryboy==2.5.1 # via -r requirements/tests.in +pytest-freezegun==0.4.2 + # via -r requirements/tests.in python-dateutil==2.8.2 - # via faker + # via + # faker + # freezegun six==1.16.0 # via python-dateutil tomli==2.0.1 @@ -70,7 +75,8 @@ tomli==2.0.1 # coverage # mypy # pytest -typing-extensions==4.5.0 +typing-extensions==4.8.0 # via + # black # mypy # pytest-factoryboy diff --git a/setup.cfg b/setup.cfg index 2500be9..98379cb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ tag = True tag_name = v{new_version} sign_tags = True -[bumpversion:file:src/app_project/__init__.py] +[bumpversion:file:src/feeds/__init__.py] search = __version__ = "{current_version}" replace = __version__ = "{new_version}" @@ -88,7 +88,7 @@ line_length = 88 default_section = THIRDPARTY known_django = django -known_first_party = app_project +known_first_party = feeds sections = FUTURE, STDLIB, @@ -103,7 +103,7 @@ skip = [tool:pytest] -DJANGO_SETTINGS_MODULE = demo.conf.settings +DJANGO_SETTINGS_MODULE = demo.settings testpaths = src diff --git a/setup.py b/setup.py index 95a045d..15c52fb 100644 --- a/setup.py +++ b/setup.py @@ -17,29 +17,34 @@ def read(filename): setup( - name="django-library-project", + name="django-rss-feeds", version="0.0.0", - description="A deployable Django app.", + description="An aggregator for RSS and Atom feeds.", long_description=read("README.md"), long_description_content_type="text/x-rst", author="Stuart MacKay", author_email="smackay@flagstonesoftware.com", - keywords="django, app, template", - url="https://github.com/StuartMacKay/django-app-template", - packages=[ - "app_project", - "app_project/migrations" - ], + keywords="django, rss, atom", + url="https://github.com/StuartMacKay/django-feeds", + packages=["feeds", "feeds/migrations"], package_dir={"": "src"}, include_package_data=True, zip_safe=False, python_requires=">=3.8", - install_requires="Django>=3.2", + install_requires=[ + "Django>=3.2,<3.3", + "croniter", + "django-extensions", + "django-structlog[celery]", + "django-tagulous", + "feedparser", + "python-dateutil", + "psycopg2-binary", + ], license="License :: OSI Approved :: Apache Software License", classifiers=[ "Development Status :: 3 - Alpha", "Framework :: Django :: 3.2", - "Framework :: Django :: 4.0", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Natural Language :: English", @@ -47,5 +52,5 @@ def read(filename): "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.10", - ] + ], ) diff --git a/src/app_project/admin/__init__.py b/src/app_project/admin/__init__.py deleted file mode 100644 index a816332..0000000 --- a/src/app_project/admin/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .example import ExampleAdmin - -__all__ = ( - "ExampleAdmin", -) diff --git a/src/app_project/admin/example.py b/src/app_project/admin/example.py deleted file mode 100644 index adba364..0000000 --- a/src/app_project/admin/example.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.contrib import admin - -from app_project import models - - -@admin.register(models.Example) -class ExampleAdmin(admin.ModelAdmin): - list_display = ("name",) - list_filter = ("name",) diff --git a/src/app_project/migrations/0001_initial.py b/src/app_project/migrations/0001_initial.py deleted file mode 100644 index b868677..0000000 --- a/src/app_project/migrations/0001_initial.py +++ /dev/null @@ -1,31 +0,0 @@ -# Generated by Django 3.1.1 on 2020-09-06 13:13 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [] - - operations = [ - migrations.CreateModel( - name="Example", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "name", - models.CharField(blank=True, max_length=32, verbose_name="Name"), - ), - ], - ), - ] diff --git a/src/app_project/models/__init__.py b/src/app_project/models/__init__.py deleted file mode 100644 index f04a629..0000000 --- a/src/app_project/models/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .example import Example - -__all__ = ( - "Example", -) diff --git a/src/app_project/models/example.py b/src/app_project/models/example.py deleted file mode 100644 index 0302494..0000000 --- a/src/app_project/models/example.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.db import models -from django.utils.translation import gettext_lazy as _ - - -class Example(models.Model): - name = models.CharField(verbose_name=_("Name"), max_length=32, blank=True) diff --git a/src/app_project/settings.py b/src/app_project/settings.py deleted file mode 100644 index ecfa14d..0000000 --- a/src/app_project/settings.py +++ /dev/null @@ -1,3 +0,0 @@ -# app-specific settings.py -# These will be added to the main settings when the app is loaded. -# See ./apps.py diff --git a/src/app_project/templates/app_project/base.html b/src/app_project/templates/app_project/base.html deleted file mode 100644 index 7219eb5..0000000 --- a/src/app_project/templates/app_project/base.html +++ /dev/null @@ -1,3 +0,0 @@ -{% extends "base.html" %} -{% load static i18n %} - diff --git a/src/app_project/templates/app_project/index.html b/src/app_project/templates/app_project/index.html deleted file mode 100644 index b8fe5d6..0000000 --- a/src/app_project/templates/app_project/index.html +++ /dev/null @@ -1,6 +0,0 @@ -{% extends "app_project/base.html" %} -{% load static i18n %} - -{% block content %} -

App Project

-{% endblock %} diff --git a/src/app_project/tests/test_views.py b/src/app_project/tests/test_views.py deleted file mode 100644 index f7bf597..0000000 --- a/src/app_project/tests/test_views.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.test import TestCase -from django.urls import reverse - - -class IndexViewTests(TestCase): - def test_view(self): - response = self.client.get(reverse("app_index")) - self.assertEqual(200, response.status_code) diff --git a/src/app_project/urls.py b/src/app_project/urls.py deleted file mode 100644 index 0752870..0000000 --- a/src/app_project/urls.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.urls import path - -from .views import IndexView - -urlpatterns = [path("", IndexView.as_view(), name="app_index")] diff --git a/src/app_project/views/__init__.py b/src/app_project/views/__init__.py deleted file mode 100644 index 7789c52..0000000 --- a/src/app_project/views/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .index import IndexView - -__all__ = ( - "IndexView", -) diff --git a/src/app_project/views/index.py b/src/app_project/views/index.py deleted file mode 100644 index 6490fec..0000000 --- a/src/app_project/views/index.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.views.generic import TemplateView - - -class IndexView(TemplateView): - template_name = "app_project/index.html" diff --git a/src/app_project/__init__.py b/src/feeds/__init__.py similarity index 100% rename from src/app_project/__init__.py rename to src/feeds/__init__.py diff --git a/src/feeds/admin/__init__.py b/src/feeds/admin/__init__.py new file mode 100644 index 0000000..0da0a10 --- /dev/null +++ b/src/feeds/admin/__init__.py @@ -0,0 +1,17 @@ +from .alias import AliasAdmin +from .article import ArticleAdmin +from .author import AuthorAdmin +from .category import CategoryAdmin +from .feed import FeedAdmin +from .source import SourceAdmin +from .tag import TagAdmin + +__all__ = ( + "AliasAdmin", + "ArticleAdmin", + "AuthorAdmin", + "CategoryAdmin", + "FeedAdmin", + "SourceAdmin", + "TagAdmin", +) diff --git a/src/feeds/admin/alias.py b/src/feeds/admin/alias.py new file mode 100644 index 0000000..202d005 --- /dev/null +++ b/src/feeds/admin/alias.py @@ -0,0 +1,12 @@ +from django.contrib.admin import ModelAdmin, register + +from feeds.models import Alias + + +@register(Alias) +class AliasAdmin(ModelAdmin): + list_display = ("name", "author", "feed") + search_fields = ("name", "author__name", "feed__name") + autocomplete_fields = ("author", "feed") + readonly_fields = ("created", "modified") + fields = ("name", "author", "feed", "created", "modified") diff --git a/src/feeds/admin/article.py b/src/feeds/admin/article.py new file mode 100644 index 0000000..ce69536 --- /dev/null +++ b/src/feeds/admin/article.py @@ -0,0 +1,162 @@ +from urllib.parse import quote + +from django import forms +from django.contrib import admin +from django.db import models +from django.forms import Textarea +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +import tagulous.admin # type: ignore + +from feeds.admin.prettifiers import prettify +from feeds.models import Article + + +def retitle(title): + class Wrapper(admin.FieldListFilter): + def __new__(cls, *args, **kwargs): + instance = admin.FieldListFilter.create(*args, **kwargs) + instance.title = title + return instance + + return Wrapper + + +class IsArchivedFilter(admin.SimpleListFilter): + title = _("Archived") + parameter_name = "archived" + + def lookups(self, request, model_admin): + return ( + ("yes", _("Yes")), + ("no", _("No")), + ) + + def queryset(self, request, queryset): + if self.value() == "yes": + return queryset.exclude(archive_url__exact="") + if self.value() == "no": + return queryset.filter(archive_url__exact="") + + +class IsTaggedFilter(admin.SimpleListFilter): + title = _("Tagged") + parameter_name = "tagged" + + def lookups(self, request, model_admin): + return ( + ("yes", _("Yes")), + ("no", _("No")), + ) + + def queryset(self, request, queryset): + if self.value() == "yes": + return queryset.exclude(tags=None) + if self.value() == "no": + return queryset.filter(tags=None) + + +class ArticleForm(forms.ModelForm): + class Meta: + model = Article + fields = "__all__" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + search_url = "https://archive.md/" + quote(self.instance.url) + message = _( + "The link to the archived article. " + 'Search for an existing copy' + ) + self.fields["archive_url"].help_text = mark_safe(message % search_url) + + +class ArticleAdmin(admin.ModelAdmin): + list_display = ( + "title", + "date", + "source", + "published", + "archived", + "tagged", + ) + + list_filter = ( + ("publish", retitle("Published")), + IsArchivedFilter, + IsTaggedFilter, + "categories", + ) + + search_fields = ("title",) + + ordering = ("-date",) + + readonly_fields = ( + "code", + "pretty_data", + "created", + "modified", + ) + + autocomplete_fields = ("authors", "source", "tags") + + form = ArticleForm + + fields = ( + "title", + "authors", + "url", + "source", + "date", + "tags", + "categories", + "publish", + "code", + "archive_url", + "summary", + "comment", + "data", + "pretty_data", + "created", + "modified", + ) + + formfield_overrides = { + models.TextField: {"widget": Textarea(attrs={"rows": "5", "cols": "80"})}, + models.JSONField: {"widget": Textarea(attrs={"rows": "10", "cols": "80"})}, + } + + @admin.display(boolean=True, description=_("Published")) + def published(self, obj): + return bool(obj.publish) + + @admin.display(boolean=True, description=_("Archived")) + def archived(self, obj): + return bool(obj.archive_url) + + @admin.display(boolean=True, description=_("Tagged")) + def tagged(self, obj): + return bool(obj.tags.all().count()) + + @admin.display(description=_("Formatted")) + def pretty_data(self, instance): + return mark_safe(prettify(instance.data)) + + def get_queryset(self, request): + return ( + super(ArticleAdmin, self) + .get_queryset(request) + .select_related("source") + .prefetch_related("authors", "categories", "tags") + ) + + def lookup_allowed(self, lookup: str, value: str) -> bool: + if lookup in ("tags__slug__in", "source__slug__in", "authors__slug__in"): + return True + return super().lookup_allowed(lookup, value) + + +tagulous.admin.register(Article, ArticleAdmin) diff --git a/src/feeds/admin/author.py b/src/feeds/admin/author.py new file mode 100644 index 0000000..e2486a5 --- /dev/null +++ b/src/feeds/admin/author.py @@ -0,0 +1,85 @@ +from django.contrib import admin, messages +from django.contrib.admin import ModelAdmin, helpers, register +from django.contrib.admin.utils import model_ngettext +from django.db.models import Count, QuerySet +from django.http import HttpRequest +from django.shortcuts import redirect, render +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ + +from feeds.models import Author + + +@register(Author) +class AuthorAdmin(ModelAdmin): + list_display = ("name", "slug", "articles") + search_fields = ("name",) + + fields = ( + "name", + "slug", + "description", + "created", + "modified", + ) + + readonly_fields = ( + "created", + "modified", + "slug", + ) + + actions = ("merge_authors", "show_articles") + + @admin.display(description=_("Articles")) + def articles(self, obj): + return obj.count + + @admin.action(description=_("Merge selected authors")) + def merge_authors(modeladmin, request, queryset): + if request.POST.get("post"): + selected = Author.objects.get(pk=request.POST["selected"]) + + for obj in queryset: + if obj.pk == selected.pk: + continue + for article in obj.articles.all(): + article.authors.remove(obj) + article.authors.add(selected) + obj.delete() + + if count := queryset.count(): + modeladmin.message_user( + request, + _("Successfully merged %(count)d %(items)s.") + % {"count": count, "items": model_ngettext(modeladmin.opts, count)}, + messages.SUCCESS, + ) + return None + + context = { + **modeladmin.admin_site.each_context(request), + "title": _("Merge authors"), + "queryset": queryset, + "opts": modeladmin.model._meta, + "action_checkbox_name": helpers.ACTION_CHECKBOX_NAME, + "media": modeladmin.media, + } + + return render(request, "admin/merge_authors.html", context=context) + + @admin.action(description=_("Show Articles for selected Authors")) + def show_articles(modeladmin, request, queryset): + url = "%s?authors__slug__in=%s" % ( + reverse("admin:feeds_article_changelist"), + ",".join([obj.slug for obj in queryset]), + ) + return redirect(url) + + def get_queryset(self, request: HttpRequest) -> QuerySet: + return ( + super() + .get_queryset(request) + .prefetch_related("articles") + .annotate(count=Count("articles")) + ) diff --git a/src/feeds/admin/category.py b/src/feeds/admin/category.py new file mode 100644 index 0000000..eee0975 --- /dev/null +++ b/src/feeds/admin/category.py @@ -0,0 +1,14 @@ +from django.contrib import admin + +import tagulous # type: ignore + +from feeds.models import Category + + +class CategoryAdmin(admin.ModelAdmin): + + list_display = ("name", "count", "protected") + search_fields = ("name",) + + +tagulous.admin.register(Category, CategoryAdmin) diff --git a/src/feeds/admin/feed.py b/src/feeds/admin/feed.py new file mode 100644 index 0000000..5ca97c7 --- /dev/null +++ b/src/feeds/admin/feed.py @@ -0,0 +1,184 @@ +import logging + +from django import forms +from django.contrib import admin, messages +from django.contrib.admin import ModelAdmin +from django.db import DataError +from django.utils.http import urlencode +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +import tagulous.admin # type: ignore + +from feeds.models import Feed +from feeds.services import feeds + +log = logging.getLogger(__name__) + + +class StatusFilter(admin.SimpleListFilter): + title = _("Status") + parameter_name = "status" + + ranges = { + "1xx": (100, 200), + "2xx": (200, 300), + "3xx": (300, 400), + "4xx": (400, 500), + "5xx": (500, 600), + } + + def lookups(self, request, model_admin): + return ( + ("1xx", _("1xx")), + ("2xx", _("2xx")), + ("3xx", _("3xx")), + ("4xx", _("4xx")), + ("5xx", _("5xx")), + ("null", _("Unknown")), + ) + + def queryset(self, request, queryset): + value = self.value() + if value == "null": + return queryset.filter(status__isnull=True) + elif value in self.ranges: + begin, end = self.ranges[value] + return queryset.filter(status__gte=begin, status__lt=end) + elif value: + return queryset.none() + + +class FailingFilter(admin.SimpleListFilter): + title = _("Failing") + parameter_name = "failing" + + def lookups(self, request, model_admin): + return ( + ("0", _("No")), + ("1", _("Yes")), + ) + + def queryset(self, request, queryset): + value = self.value() + if value == "0": + return queryset.filter(failures=0) + elif value == "1": + return queryset.filter(failures__gt=0) + elif value: + return queryset.none() + + +class FeedForm(forms.ModelForm): + class Meta: + model = Feed + fields = ( + "name", + "source", + "url", + "categories", + "enabled", + "schedule", + "auto_publish", + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if url := self.data.get("url", self.initial.get("url")): + validate_url = "https://validator.w3.org/feed/check.cgi?" + urlencode( + {"url": url} + ) + message = _('Validate the current url') + self.fields["url"].help_text = mark_safe(message % validate_url) + + def clean_schedule(self): + value = self.cleaned_data["schedule"] + + # Remove any extra spaces to reduce the chances that an invalid + # crontab string will be entered. + + value = value.replace(" ", " ") + value = value.replace(" ,", ",") + value = value.replace(", ", ",") + value = value.replace(" -", "-") + value = value.replace("- ", "-") + value = value.replace(" /", "/") + value = value.replace("/ ", "/") + + return value.strip() + + +class FeedAdmin(ModelAdmin): + list_display = ( + "name", + "enabled", + "auto_publish", + "loaded", + "failures", + "status", + ) + + list_filter = ( + "categories", + StatusFilter, + FailingFilter, + "enabled", + "auto_publish", + ) + + search_fields = ("name",) + + readonly_fields = ( + "created", + "modified", + "loaded", + "failures", + "etag", + "last_modified", + "status", + ) + + actions = [ + "load_feed", + ] + + form = FeedForm + + autocomplete_fields = ("source",) + + @admin.action(description=_("Load selected feeds")) + def load_feed(modeladmin, request, queryset): # noqa + for feed in queryset: + try: + # force the feed to be loaded by resetting the fields + # used to send the Last-Modified or ETag headers. + + feed.last_modified = None + feed.etag = None + + # Call load_feed at the module level so this action + # can be tested by mocking the function. + + result = feeds.load_feed(feed) + except DataError: + log.exception("Could not load feed", extra={"feed": feed.name}) + result = False + + if result: + msg = _('The feed "%s" was loaded successfully.' % feed.name) + messages.info(request, msg) + else: + msg = _('There was an error loading the feed "%s"' % feed.name) + messages.error(request, msg) + + def get_queryset(self, request): + return super(FeedAdmin, self).get_queryset(request) + + def get_readonly_fields(self, request, obj=None): + if obj: + return super().get_readonly_fields(request, obj) + return [] + + +tagulous.admin.register(Feed, FeedAdmin) diff --git a/src/feeds/admin/prettifiers.py b/src/feeds/admin/prettifiers.py new file mode 100644 index 0000000..3597a31 --- /dev/null +++ b/src/feeds/admin/prettifiers.py @@ -0,0 +1,65 @@ +import json + +from pygments import highlight # type: ignore +from pygments.formatters.html import HtmlFormatter # type: ignore +from pygments.lexers.data import JsonLexer # type: ignore + +__all__ = [ + "prettify", +] + + +class PrettyEncoder(json.JSONEncoder): + def __init__(self, *args, **kwargs): + super(PrettyEncoder, self).__init__(*args, **kwargs) + self.current_indent = 0 + self.current_indent_str = "" + + def encode(self, o): + # Special Processing for lists + if isinstance(o, (list, tuple)): + primitives_only = True + for item in o: + if isinstance(item, (list, tuple, dict)): + primitives_only = False + break + output = [] + if primitives_only: + for item in o: + output.append(json.dumps(item)) + return "[ " + ", ".join(output) + " ]" + else: + self.current_indent += self.indent + self.current_indent_str = "".join( + [" " for x in range(self.current_indent)] + ) + for item in o: + output.append(self.current_indent_str + self.encode(item)) + self.current_indent -= self.indent + self.current_indent_str = " " * self.current_indent + return "[\n" + ",\n".join(output) + "\n" + self.current_indent_str + "]" + elif isinstance(o, dict): + output = [] + self.current_indent += self.indent + self.current_indent_str = " " * self.current_indent + items = sorted(o.items()) if self.sort_keys else o.items() + for key, value in items: + output.append( + self.current_indent_str + + json.dumps(key) + + ": " + + self.encode(value) + ) + self.current_indent -= self.indent + self.current_indent_str = " " * self.current_indent + return "{\n" + ",\n".join(output) + "\n" + self.current_indent_str + "}" + else: + return json.dumps(o) + + +def prettify(data): + response = json.dumps(data, sort_keys=True, indent=4, cls=PrettyEncoder) + formatter = HtmlFormatter(style="colorful") + response = highlight(response, JsonLexer(), formatter) + style = "
" + return style + response diff --git a/src/feeds/admin/source.py b/src/feeds/admin/source.py new file mode 100644 index 0000000..87241dc --- /dev/null +++ b/src/feeds/admin/source.py @@ -0,0 +1,67 @@ +from django.contrib import admin +from django.contrib.admin import ModelAdmin, register +from django.db import models +from django.forms import Textarea +from django.shortcuts import redirect +from django.urls import reverse +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from feeds.admin.prettifiers import prettify +from feeds.models import Source + + +@register(Source) +class SourceAdmin(ModelAdmin): + list_display = ( + "name", + "has_url", + "has_description", + ) + + search_fields = ("name",) + + actions = ["show_articles"] + + readonly_fields = ( + "pretty_data", + "created", + "modified", + "slug", + ) + + fields = ( + "name", + "url", + "description", + "data", + "pretty_data", + "created", + "modified", + "slug", + ) + + formfield_overrides = { + models.TextField: {"widget": Textarea(attrs={"rows": "5", "cols": "80"})}, + models.JSONField: {"widget": Textarea(attrs={"rows": "10", "cols": "80"})}, + } + + @admin.display(boolean=True) + def has_url(self, obj): + return bool(obj.url) + + @admin.display(boolean=True) + def has_description(self, obj): + return bool(obj.description) + + @admin.display(description=_("Formatted")) + def pretty_data(self, instance): + return mark_safe(prettify(instance.data)) + + @admin.action(description=_("Show Articles for selected Sources")) + def show_articles(modeladmin, request, queryset): + url = "%s?source__slug__in=%s" % ( + reverse("admin:feeds_article_changelist"), + ",".join([obj.slug for obj in queryset]), + ) + return redirect(url) diff --git a/src/feeds/admin/tag.py b/src/feeds/admin/tag.py new file mode 100644 index 0000000..bfaa21f --- /dev/null +++ b/src/feeds/admin/tag.py @@ -0,0 +1,119 @@ +from django.contrib import admin, messages +from django.contrib.admin import helpers, register +from django.contrib.admin.utils import model_ngettext +from django.db import models +from django.db.models import Count, QuerySet +from django.forms import Textarea +from django.http import HttpRequest +from django.shortcuts import redirect, render +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ + +from feeds.models import Tag + + +class ArticleCountFilter(admin.SimpleListFilter): + title = _("No. of Articles") + parameter_name = "number_of_articles" + + def lookups(self, request, model_admin): + return ( + (0, _("0")), + (1, _("1")), + (2, _("2")), + (3, _("3")), + ) + + def queryset(self, request, queryset): + if value := self.value(): + try: + return queryset.filter(count=int(value)) + except ValueError: + return queryset.none() + return queryset + + +@register(Tag) +class TagAdmin(admin.ModelAdmin): + list_display = ("name", "slug", "has_summary", "articles") + + list_filter = (ArticleCountFilter,) + + search_fields = ("name",) + + fields = ( + "name", + "slug", + "summary", + "description", + "related", + "created", + "modified", + ) + + readonly_fields = ("created", "modified") + + actions = ["merge_tags", "show_articles"] + + autocomplete_fields = ("related",) + + formfield_overrides = { + models.TextField: {"widget": Textarea(attrs={"rows": "5", "cols": "80"})}, + } + + @admin.display(boolean=True, description=_("Has Summary")) + def has_summary(self, obj): + return bool(obj.summary) + + @admin.display(description=_("Articles")) + def articles(self, obj): + return obj.count + + @admin.action(description=_("Merge selected Tags")) + def merge_tags(modeladmin, request, queryset): + if request.POST.get("post"): + selected = Tag.objects.get(pk=request.POST["selected"]) + + for obj in queryset: + if obj.pk == selected.pk: + continue + for article in obj.article_set.all(): + article.tags.remove(obj) + article.tags.add(selected) + obj.delete() + + if count := queryset.count(): + modeladmin.message_user( + request, + _("Successfully merged %(count)d %(items)s.") + % {"count": count, "items": model_ngettext(modeladmin.opts, count)}, + messages.SUCCESS, + ) + return None + + context = { + **modeladmin.admin_site.each_context(request), + "title": _("Merge Tags"), + "queryset": queryset, + "opts": modeladmin.model._meta, + "action_checkbox_name": helpers.ACTION_CHECKBOX_NAME, + "media": modeladmin.media, + } + + return render(request, "admin/merge_tags.html", context=context) + + @admin.action(description=_("Show Articles for selected Tags")) + def show_articles(modeladmin, request, queryset): + url = "%s?tags__slug__in=%s" % ( + reverse("admin:feeds_article_changelist"), + ",".join([obj.slug for obj in queryset]), + ) + return redirect(url) + + def get_queryset(self, request: HttpRequest) -> QuerySet: + return ( + super() + .get_queryset(request) + .prefetch_related("article_set") + .annotate(count=Count("article")) + ) diff --git a/src/app_project/apps.py b/src/feeds/apps.py similarity index 86% rename from src/app_project/apps.py rename to src/feeds/apps.py index e2e05f8..5222e09 100644 --- a/src/app_project/apps.py +++ b/src/feeds/apps.py @@ -12,8 +12,7 @@ def setup_app_settings(): class Config(AppConfig): - name = "app_project" - verbose_name = "App Project" + name = "feeds" def ready(self): setup_app_settings() diff --git a/src/app_project/locale/en/LC_MESSAGES/django.mo b/src/feeds/locale/en/LC_MESSAGES/django.mo similarity index 100% rename from src/app_project/locale/en/LC_MESSAGES/django.mo rename to src/feeds/locale/en/LC_MESSAGES/django.mo diff --git a/src/app_project/locale/en/LC_MESSAGES/django.po b/src/feeds/locale/en/LC_MESSAGES/django.po similarity index 100% rename from src/app_project/locale/en/LC_MESSAGES/django.po rename to src/feeds/locale/en/LC_MESSAGES/django.po diff --git a/src/feeds/migrations/0001_initial.py b/src/feeds/migrations/0001_initial.py new file mode 100644 index 0000000..8c2777d --- /dev/null +++ b/src/feeds/migrations/0001_initial.py @@ -0,0 +1,655 @@ +# Generated by Django 4.1.7 on 2023-03-13 19:46 + +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields +import feeds.models.article +import feeds.models.feed +import tagulous.models.fields +import tagulous.models.models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Author", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "name", + models.CharField( + help_text="The name of the author", + max_length=100, + verbose_name="Name", + ), + ), + ( + "slug", + django_extensions.db.fields.AutoSlugField( + blank=True, + editable=False, + help_text="The slug uniquely identifying the author", + max_length=100, + populate_from="name", + unique=True, + verbose_name="Slug", + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="A short profile of the author", + verbose_name="Description", + ), + ), + ], + options={ + "verbose_name": "Author", + "verbose_name_plural": "Authors", + }, + ), + migrations.CreateModel( + name="Category", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255, unique=True)), + ("slug", models.SlugField()), + ( + "count", + models.IntegerField( + default=0, + help_text="Internal counter of how many times this tag is in use", + ), + ), + ( + "protected", + models.BooleanField( + default=False, + help_text="Will not be deleted when the count reaches 0", + ), + ), + ("path", models.TextField()), + ( + "label", + models.CharField( + help_text="The name of the tag, without ancestors", + max_length=255, + ), + ), + ( + "level", + models.IntegerField( + default=1, help_text="The level of the tag in the tree" + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="feeds.category", + ), + ), + ], + options={ + "verbose_name": "Category", + "verbose_name_plural": "Categories", + }, + bases=(tagulous.models.models.BaseTagTreeModel, models.Model), + ), + migrations.CreateModel( + name="Source", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "name", + models.CharField( + help_text="The name of the source", + max_length=100, + verbose_name="Name", + ), + ), + ( + "slug", + django_extensions.db.fields.AutoSlugField( + blank=True, + editable=False, + help_text="The slug uniquely identifying the source", + max_length=100, + populate_from="name", + unique=True, + verbose_name="Slug", + ), + ), + ( + "url", + models.URLField( + blank=True, + help_text="The URL for source's web site", + verbose_name="Web site", + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="A description of the source", + verbose_name="Description", + ), + ), + ( + "data", + models.JSONField( + blank=True, + default=dict, + help_text="Data describing a source", + verbose_name="Data", + ), + ), + ], + options={ + "verbose_name": "Source", + "verbose_name_plural": "Sources", + }, + ), + migrations.CreateModel( + name="Tag", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "name", + models.CharField( + default="", + help_text="The name of the tag", + max_length=100, + verbose_name="Name", + ), + ), + ( + "slug", + django_extensions.db.fields.AutoSlugField( + blank=True, + editable=False, + help_text="The slug uniquely identifying the tag", + max_length=100, + populate_from="name", + unique=True, + verbose_name="Slug", + ), + ), + ( + "summary", + models.TextField( + blank=True, + help_text="A short summary of what the tag covers", + verbose_name="Summary", + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="A more detailed description of the tag. May be in HTML", + verbose_name="Description", + ), + ), + ( + "related", + models.ManyToManyField( + blank=True, + help_text="The set of tag(s) that are related to this one", + to="feeds.tag", + verbose_name="Related tags", + ), + ), + ], + options={ + "verbose_name": "Tag", + "verbose_name_plural": "Tags", + }, + ), + migrations.CreateModel( + name="Feed", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "name", + models.CharField( + help_text="The name of the feed", + max_length=100, + verbose_name="Name", + ), + ), + ( + "url", + models.URLField( + help_text="The URL for the RSS feed (RSS or Atom)", + verbose_name="URL", + ), + ), + ( + "enabled", + models.BooleanField( + default=False, + help_text="Enable loading of the RSS feed", + verbose_name="Enabled", + ), + ), + ( + "schedule", + models.CharField( + blank=True, + help_text="Crontab entry which describe the times the feed will be downloaded.Leave blank to use the default schedule defined for all feeds.", + max_length=100, + validators=[feeds.models.feed.validate_crontab], + verbose_name="Schedule", + ), + ), + ( + "auto_publish", + models.BooleanField( + default=False, + help_text="Automatically publish Articles when added from an RSS Feed", + verbose_name="Auto Publish", + ), + ), + ( + "loaded", + models.DateTimeField( + blank=True, + help_text="The date that the RSS feed was last loaded", + null=True, + verbose_name="Loaded", + ), + ), + ( + "failures", + models.IntegerField( + default=0, + help_text="The number of consecutive times a feed has failed to load.", + verbose_name="Failures", + ), + ), + ( + "status", + models.IntegerField( + blank=True, + help_text="The HTTP status code from the last time the feed was fetched", + null=True, + verbose_name="Status", + ), + ), + ( + "etag", + models.CharField( + blank=True, + help_text="The Etag header from the RSS feed (Atom feeds only)", + max_length=100, + null=True, + verbose_name="ETag", + ), + ), + ( + "last_modified", + models.DateTimeField( + blank=True, + help_text="The last-modified header from the RSS feed", + null=True, + verbose_name="Last Modified", + ), + ), + ( + "categories", + tagulous.models.fields.TagField( + _set_tag_meta=True, + blank=True, + force_lowercase=True, + help_text="The categories of articles published by the feed", + to="feeds.category", + tree=True, + verbose_name="Categories", + ), + ), + ( + "source", + models.ForeignKey( + help_text="The web site which hosts the feed", + on_delete=django.db.models.deletion.PROTECT, + to="feeds.source", + verbose_name="Source", + ), + ), + ], + options={ + "verbose_name": "Feed", + "verbose_name_plural": "Feeds", + }, + ), + migrations.CreateModel( + name="Article", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "title", + models.CharField( + help_text="The title of the article", + max_length=1000, + verbose_name="Title", + ), + ), + ( + "url", + models.URLField( + help_text="The link to the article on the source's web site", + max_length=2000, + verbose_name="URL", + ), + ), + ( + "code", + models.CharField( + default=feeds.models.article.article_code, + help_text="The code that used to identify the article", + max_length=6, + verbose_name="Code", + ), + ), + ( + "archive_url", + models.URLField( + blank=True, + help_text="The link to the archived article using a service such as archive.today", + max_length=2000, + verbose_name="Archive URL", + ), + ), + ( + "date", + models.DateTimeField( + help_text="The date the article was published by the source", + verbose_name="Date", + ), + ), + ( + "summary", + models.TextField( + blank=True, + help_text="A summary of the article", + verbose_name="Summary", + ), + ), + ( + "identifier", + models.CharField( + blank=True, + help_text="The unique identifier for the Article from the Feed", + max_length=2000, + verbose_name="Identifier", + ), + ), + ( + "comment", + models.TextField( + blank=True, + help_text="An editorial comment about the article", + verbose_name="Comment", + ), + ), + ( + "publish", + models.BooleanField( + db_index=True, + default=False, + help_text="Publish the article on the site", + verbose_name="Publish", + ), + ), + ( + "views", + models.PositiveIntegerField( + default=0, + help_text="The number of times the article has been viewed (read)", + verbose_name="Views", + ), + ), + ( + "data", + models.JSONField( + blank=True, + default=dict, + help_text="Data describing an article", + verbose_name="Data", + ), + ), + ( + "authors", + models.ManyToManyField( + help_text="The author(s) who wrote the article", + related_name="articles", + to="feeds.author", + verbose_name="Authors", + ), + ), + ( + "categories", + tagulous.models.fields.TagField( + _set_tag_meta=True, + blank=True, + help_text="The categories that describes the article contents", + to="feeds.category", + tree=True, + verbose_name="Categories", + ), + ), + ( + "feed", + models.ForeignKey( + blank=True, + help_text="The feed from the host web site", + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="articles", + to="feeds.feed", + verbose_name="RSS Feed", + ), + ), + ( + "source", + models.ForeignKey( + help_text="The source (blog, news site, etc.) which published the article", + on_delete=django.db.models.deletion.CASCADE, + related_name="articles", + to="feeds.source", + verbose_name="Source", + ), + ), + ( + "tags", + models.ManyToManyField( + blank=True, + help_text="Keywords for subjects covered in the content", + to="feeds.tag", + verbose_name="Tags", + ), + ), + ], + options={ + "verbose_name": "Article", + "verbose_name_plural": "Articles", + "get_latest_by": "date", + }, + ), + migrations.CreateModel( + name="Alias", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "name", + models.CharField( + blank=True, + help_text="A name used to lookup an author from a feed entry", + max_length=100, + verbose_name="Name", + ), + ), + ( + "author", + models.ForeignKey( + help_text="The Author that the feed entry author maps to", + on_delete=django.db.models.deletion.CASCADE, + related_name="aliases", + to="feeds.author", + verbose_name="Author", + ), + ), + ( + "feed", + models.ForeignKey( + help_text="The feed where the alias is used", + on_delete=django.db.models.deletion.CASCADE, + to="feeds.feed", + verbose_name="Feed", + ), + ), + ], + options={ + "verbose_name": "Alias", + "verbose_name_plural": "Aliases", + }, + ), + ] diff --git a/demo/conf/__init__.py b/src/feeds/migrations/__init__.py similarity index 100% rename from demo/conf/__init__.py rename to src/feeds/migrations/__init__.py diff --git a/src/feeds/models/__init__.py b/src/feeds/models/__init__.py new file mode 100644 index 0000000..7ef2698 --- /dev/null +++ b/src/feeds/models/__init__.py @@ -0,0 +1,17 @@ +from .alias import Alias +from .article import Article +from .author import Author +from .category import Category +from .feed import Feed +from .source import Source +from .tag import Tag + +__all__ = ( + "Alias", + "Article", + "Author", + "Category", + "Feed", + "Source", + "Tag", +) diff --git a/src/feeds/models/alias.py b/src/feeds/models/alias.py new file mode 100644 index 0000000..e90412d --- /dev/null +++ b/src/feeds/models/alias.py @@ -0,0 +1,35 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from django_extensions.db.models import TimeStampedModel + + +class Alias(TimeStampedModel): + class Meta: + verbose_name = _("Alias") + verbose_name_plural = _("Aliases") + + name = models.CharField( + verbose_name=_("Name"), + help_text=_("A name used to lookup an author from a feed entry"), + max_length=100, + blank=True, + ) + + author = models.ForeignKey( + to="feeds.Author", + related_name="aliases", + on_delete=models.CASCADE, + verbose_name=_("Author"), + help_text=_("The Author that the feed entry author maps to"), + ) + + feed = models.ForeignKey( + to="feeds.Feed", + on_delete=models.CASCADE, + verbose_name=_("Feed"), + help_text=_("The feed where the alias is used"), + ) + + def __str__(self): + return self.name diff --git a/src/feeds/models/article.py b/src/feeds/models/article.py new file mode 100644 index 0000000..e6f64e1 --- /dev/null +++ b/src/feeds/models/article.py @@ -0,0 +1,176 @@ +import datetime as dt + +from django.db import models +from django.db.models import F +from django.utils.crypto import get_random_string +from django.utils.translation import gettext_lazy as _ + +from django_extensions.db.models import TimeStampedModel +from tagulous.models import TagField + + +def article_code(): + return get_random_string(6) + + +class ArticleQuerySet(models.QuerySet): + def published(self) -> "ArticleQuerySet": + return self.filter(publish=True) + + def for_date(self, date: dt.date) -> "ArticleQuerySet": + return self.filter(date__date=date) + + def since_date(self, date: dt.date) -> "ArticleQuerySet": + return self.filter(date__date__gt=date) + + def by_author(self, slug) -> "ArticleQuerySet": + return self.filter(authors__slug=slug) + + def by_source(self, slug) -> "ArticleQuerySet": + return self.filter(source__slug=slug) + + def by_tag(self, slug) -> "ArticleQuerySet": + return self.filter(tags__slug=slug) + + def with_code(self, code) -> "Article": + return self.filter(code=code).latest("created") + + def with_identifier(self, identifier) -> "ArticleQuerySet": + return self.filter(identifier=identifier) + + def viewed(self, code) -> None: + self.filter(code=code).update(count=F("views") + 1) + + +ArticleManager = models.Manager.from_queryset(ArticleQuerySet) # type: ignore + + +class Article(TimeStampedModel): + class Meta: + verbose_name = _("Article") + verbose_name_plural = _("Articles") + get_latest_by = "date" + + title = models.CharField( + verbose_name=_("Title"), + help_text=_("The title of the article"), + max_length=1000, + ) + + authors = models.ManyToManyField( + to="feeds.Author", + related_name="articles", + verbose_name=_("Authors"), + help_text=_("The author(s) who wrote the article"), + ) + + url = models.URLField( + verbose_name=_("URL"), + help_text=_("The link to the article on the source's web site"), + max_length=2000, + ) + + code = models.CharField( + verbose_name=_("Code"), + help_text=_("The code that used to identify the article"), + max_length=6, + default=article_code, + ) + + archive_url = models.URLField( + verbose_name=_("Archive URL"), + help_text=_( + "The link to the archived article using a service such as archive.today" + ), + max_length=2000, + blank=True, + ) + + date = models.DateTimeField( + verbose_name=_("Date"), + help_text=_("The date the article was published by the source"), + ) + + summary = models.TextField( + verbose_name=_("Summary"), + help_text=_("A summary of the article"), + blank=True, + ) + + source = models.ForeignKey( + to="feeds.Source", + related_name="articles", + on_delete=models.CASCADE, + verbose_name=_("Source"), + help_text=_("The source (blog, news site, etc.) which published the article"), + ) + + feed = models.ForeignKey( + to="feeds.Feed", + related_name="articles", + on_delete=models.PROTECT, + verbose_name=_("RSS Feed"), + help_text=_("The feed from the host web site"), + null=True, + blank=True, + ) + + identifier = models.CharField( + verbose_name=_("Identifier"), + help_text=_("The unique identifier for the Article from the Feed"), + max_length=2000, + blank=True, + ) + + comment = models.TextField( + verbose_name=_("Comment"), + help_text=_("An editorial comment about the article"), + blank=True, + ) + + publish = models.BooleanField( + verbose_name=_("Publish"), + help_text=_("Publish the article on the site"), + default=False, + db_index=True, + ) + + categories = TagField( + verbose_name=_("Categories"), + help_text=_("The categories that describes the article contents"), + to="feeds.Category", + blank=True, + ) + + # Tags are added after the article is added to the database. + # The idea is that articles on the same subjects from different + # sources have the same set of Tags - more or less. + + tags = models.ManyToManyField( + verbose_name=_("Tags"), + help_text=_("Keywords for subjects covered in the content"), + to="feeds.Tag", + blank=True, + ) + + views = models.PositiveIntegerField( + verbose_name=_("Views"), + help_text="The number of times the article has been viewed (read)", + default=0, + ) + + data = models.JSONField( + verbose_name=_("Data"), + help_text=_("Data describing an article"), + default=dict, + blank=True, + ) + + objects = ArticleManager() # type: ignore + + def __str__(self) -> str: + return self.title + + def has_authors(self): + names = [author.name for author in self.authors.all()] + return names and " ".join(names) != self.source.name diff --git a/src/feeds/models/author.py b/src/feeds/models/author.py new file mode 100644 index 0000000..f2c93a2 --- /dev/null +++ b/src/feeds/models/author.py @@ -0,0 +1,47 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from django_extensions.db.fields import AutoSlugField +from django_extensions.db.models import TimeStampedModel + + +class AuthorQuerySet(models.QuerySet): + def published(self) -> "AuthorQuerySet": + return self.filter(articles__publish=True).distinct() + + def with_slug(self, slug) -> "AuthorQuerySet": + return self.filter(slug=slug) + + +AuthorManager = models.Manager.from_queryset(AuthorQuerySet) # type: ignore + + +class Author(TimeStampedModel): # type: ignore + class Meta: + verbose_name = _("Author") + verbose_name_plural = _("Authors") + + name = models.CharField( + verbose_name=_("Name"), + help_text=_("The name of the author"), + max_length=100, + ) + + slug = AutoSlugField( + verbose_name=_("Slug"), + help_text=_("The slug uniquely identifying the author"), + populate_from="name", + max_length=100, + unique=True, + ) + + description = models.TextField( + verbose_name=_("Description"), + help_text=_("A short profile of the author"), + blank=True, + ) + + objects = AuthorManager() # type: ignore + + def __str__(self): + return self.name diff --git a/src/feeds/models/category.py b/src/feeds/models/category.py new file mode 100644 index 0000000..ef96509 --- /dev/null +++ b/src/feeds/models/category.py @@ -0,0 +1,12 @@ +from django.utils.translation import gettext_lazy as _ + +import tagulous # type: ignore + + +class Category(tagulous.models.TagTreeModel): + class Meta: + verbose_name = _("Category") + verbose_name_plural = _("Categories") + + class TagMeta: + force_lowercase = True diff --git a/src/feeds/models/feed.py b/src/feeds/models/feed.py new file mode 100644 index 0000000..db738d5 --- /dev/null +++ b/src/feeds/models/feed.py @@ -0,0 +1,142 @@ +import datetime as dt +from typing import List + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.translation import gettext_lazy as _ + +import tagulous # type: ignore +from croniter import croniter +from django_extensions.db.models import TimeStampedModel + + +def validate_crontab(value): + if not croniter.is_valid(value): + raise ValidationError( + _('"%(value)s" is not a valid crontab entry'), + params={"value": value}, + ) + + minutes = value.split()[0] + + if minutes != "0" and minutes != "*": + raise ValidationError( + _( + "The celery task to load feeds runs every hour on the hour" + "so the minutes field (the first) must be set to '0' or '*'" + ) + ) + + +class FeedQuerySet(models.QuerySet): + def disabled(self) -> "FeedQuerySet": + return self.filter(enabled=False) + + def enabled(self) -> "FeedQuerySet": + return self.filter(enabled=True) + + def scheduled_for(self, timestamp: dt.datetime) -> List["Feed"]: + feeds: List["Feed"] = [] + for feed in Feed.objects.enabled(): + schedule: str = feed.schedule or settings.FEEDS_LOAD_SCHEDULE + if croniter.match(schedule, timestamp): + feeds.append(feed) + return feeds + + +FeedManager = models.Manager.from_queryset(FeedQuerySet) # type: ignore + + +class Feed(TimeStampedModel): # type: ignore + class Meta: + verbose_name = _("Feed") + verbose_name_plural = _("Feeds") + + name = models.CharField( + verbose_name=_("Name"), + help_text=_("The name of the feed"), + max_length=100, + ) + + source = models.ForeignKey( + to="feeds.Source", + on_delete=models.PROTECT, + verbose_name=_("Source"), + help_text=_("The web site which hosts the feed"), + ) + + url = models.URLField( + verbose_name=_("URL"), + help_text=_("The URL for the RSS feed (RSS or Atom)"), + ) + + categories = tagulous.models.TagField( + verbose_name=_("Categories"), + help_text=_("The categories of articles published by the feed"), + to="feeds.Category", + blank=True, + ) + + enabled = models.BooleanField( + verbose_name=_("Enabled"), + help_text=_("Enable loading of the RSS feed"), + default=False, + ) + + schedule = models.CharField( + validators=[validate_crontab], + verbose_name=_("Schedule"), + help_text=_( + "Crontab entry which describe the times the feed will be downloaded." + "Leave blank to use the default schedule defined for all feeds." + ), + max_length=100, + blank=True, + ) + + auto_publish = models.BooleanField( + verbose_name=_("Auto Publish"), + help_text=_("Automatically publish Articles when added from an RSS Feed"), + default=False, + ) + + loaded = models.DateTimeField( + verbose_name=_("Loaded"), + help_text=_("The date that the RSS feed was last loaded"), + null=True, + blank=True, + ) + + failures = models.IntegerField( + verbose_name=_("Failures"), + help_text=_("The number of consecutive times a feed has failed to load."), + default=0, + ) + + status = models.IntegerField( + verbose_name=_("Status"), + help_text=_("The HTTP status code from the last time the feed was fetched"), + null=True, + blank=True, + ) + + etag = models.CharField( + verbose_name=_("ETag"), + help_text=_("The Etag header from the RSS feed (Atom feeds only)"), + max_length=100, + null=True, + blank=True, + ) + + last_modified = models.DateTimeField( + verbose_name=_("Last Modified"), + help_text=_("The last-modified header from the RSS feed"), + null=True, + blank=True, + ) + + objects = FeedManager() # type: ignore + + def __str__(self): + return self.name diff --git a/src/feeds/models/source.py b/src/feeds/models/source.py new file mode 100644 index 0000000..ab82c99 --- /dev/null +++ b/src/feeds/models/source.py @@ -0,0 +1,47 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from django_extensions.db.fields import AutoSlugField +from django_extensions.db.models import TimeStampedModel + + +class Source(TimeStampedModel): # type: ignore + class Meta: + verbose_name = _("Source") + verbose_name_plural = _("Sources") + + name = models.CharField( + verbose_name=_("Name"), + help_text=_("The name of the source"), + max_length=100, + ) + + slug = AutoSlugField( + verbose_name=_("Slug"), + help_text=_("The slug uniquely identifying the source"), + populate_from="name", + max_length=100, + unique=True, + ) + + url = models.URLField( + verbose_name=_("Web site"), + help_text=_("The URL for source's web site"), + blank=True, + ) + + description = models.TextField( + verbose_name=_("Description"), + help_text=_("A description of the source"), + blank=True, + ) + + data = models.JSONField( + verbose_name=_("Data"), + help_text=_("Data describing a source"), + default=dict, + blank=True, + ) + + def __str__(self): + return self.name diff --git a/src/feeds/models/tag.py b/src/feeds/models/tag.py new file mode 100644 index 0000000..5d70e6e --- /dev/null +++ b/src/feeds/models/tag.py @@ -0,0 +1,81 @@ +from typing import Dict, List + +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from django_extensions.db.fields import AutoSlugField +from django_extensions.db.models import TimeStampedModel + + +class TagQuerySet(models.QuerySet): + def for_articles(self, articles) -> "TagQuerySet": + return self.filter(article__in=articles) + + def weighted(self) -> "List[Tag]": + results: Dict[Tag, int] = {} + + for tag in self: + results.setdefault(tag, 0) + results[tag] += 1 + + counts = results.values() + + if counts: + highest = max(counts) + lowest = min(counts) + delta = highest - lowest + 1 + categories = 6 + interval = delta / categories + + for tag, count in results.items(): + tag.weight = int((count - lowest) / interval) + 1 + + return list(results.keys()) + + +TagManager = models.Manager.from_queryset(TagQuerySet) # type: ignore + + +class Tag(TimeStampedModel): # type: ignore + class Meta: + verbose_name = _("Tag") + verbose_name_plural = _("Tags") + + name = models.CharField( + verbose_name=_("Name"), + help_text=_("The name of the tag"), + max_length=100, + default="", + ) + + slug = AutoSlugField( + verbose_name=_("Slug"), + help_text=_("The slug uniquely identifying the tag"), + populate_from="name", + max_length=100, + unique=True, + ) + + summary = models.TextField( + verbose_name=_("Summary"), + help_text=_("A short summary of what the tag covers"), + blank=True, + ) + + description = models.TextField( + verbose_name=_("Description"), + help_text=_("A more detailed description of the tag. May be in HTML"), + blank=True, + ) + + related = models.ManyToManyField( + to="self", + verbose_name=_("Related tags"), + help_text=_("The set of tag(s) that are related to this one"), + blank=True, + ) + + objects = TagManager() # type: ignore + + def __str__(self): + return self.name diff --git a/src/app_project/migrations/__init__.py b/src/feeds/services/__init__.py similarity index 100% rename from src/app_project/migrations/__init__.py rename to src/feeds/services/__init__.py diff --git a/src/feeds/services/feeds.py b/src/feeds/services/feeds.py new file mode 100644 index 0000000..02f14de --- /dev/null +++ b/src/feeds/services/feeds.py @@ -0,0 +1,327 @@ +import logging +import re +from datetime import datetime +from html import unescape +from typing import List, Optional +from urllib.error import HTTPError, URLError + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.core.validators import URLValidator +from django.utils import timezone +from django.utils.text import slugify + +import feedparser # type: ignore +from dateutil.parser import parse as parse_date +from feedparser import FeedParserDict + +from feeds.models import Alias, Article, Author, Feed + +__all__ = ("load_feed", "get_user_agent") + + +validate_url = URLValidator() + + +def load_feeds() -> None: + now = timezone.now() + feeds = Feed.objects.scheduled_for(now) + count = 0 + + log = logging.getLogger(f"{__name__}.load_feeds") + log.debug("Feeds scheduled", extra={"num_feeds": len(feeds)}) + + for feed in feeds: + if load_feed(feed): + count += 1 + + log.debug("Feeds loaded", extra={"num_feeds": count}) + + +def load_feed(feed: Feed) -> bool: + # The Last-Modified and ETag headers, from the previous fetch, are + # sent when fetching a feed, so we only load entries if the feed + # has been updated. + + # Since we do conditional fetches, the feeds can be updated frequently. + # That lets avoid worrying about retries for network and remote + # server errors since it will probably have been resolved by the + # next fetch. + + log = logging.getLogger(f"{__name__}.load_feed") + + url = feed.url + etag = feed.etag + modified = feed.last_modified + + log.debug( + "Feed request", + extra={"feed": feed.name, "url": url, "last_modified": modified, "etag": etag}, + ) + + feed.loaded = timezone.now() + + try: + response = feedparser.parse( + get_source(feed), agent=get_user_agent(), modified=modified, etag=etag + ) + except (HTTPError, URLError): + log.exception("Feed not fetched", extra={"feed": feed.name}) + feed.failures += 1 + feed.save() + return False + + # The status field is not present in the response when a feed + # is loaded from a string or a file. + + feed.status = response.get("status") + + value = response.headers.get("last-modified") + modified = parse_date(value) if value else None + etag = response.headers.get("etag") + content_length = response.headers.get("content-length") + num_entries = len(response.entries) + + log.debug( + "Feed response", + extra={ + "feed": feed.name, + "status": feed.status, + "last_modified": modified, + "etag": etag, + "content_length": content_length, + }, + ) + + # Strictly speaking, if the feed is unchanged then no XML was returned, + # so we don't know if the feed is valid since nothing was parsed. However, + # setting the valid flag to None implies some form of error occurred when + # in fact the feed is operating normally. + + if feed.status == 304: + log.debug("Feed is unchanged", extra={"feed": feed.name}) + feed.failures = 0 + feed.save() + return False + + # Assume anything other than a 200 OK status at this point means that the + # feed XML was not returned. This avoids confusingly reporting a parse + # error if an HTML error page was returned, for example, if the feed url + # changed. Note that the status will be None when running tests. + + if feed.status and feed.status != 200: + log.error( + "Feed not loaded", + extra={ + "feed": feed.name, + "status": feed.status, + "num_entries": num_entries, + }, + ) + feed.failures += 1 + feed.save() + return False + + # One source of feed parsing failures is non-printing characters, which + # presumably were the copy and pasted into the post when it was created. + # We don't want abandon the load at this point as it might be something + # that the feed author can correct and since we're having trouble other + # people are as well, so it's worth contacting them over this. + + if response.bozo: + exc = response.get("bozo_exception", None) + log.error("Feed not parsed", exc_info=exc) + feed.failures += 1 + feed.save() + return False + + feed.etag = etag + feed.last_modified = modified + feed.failures = 0 + feed.save() + + num_created = 0 + + for item in response.entries: + values = { + "identifier": get_identifier(item), + "title": get_title(item), + "url": get_url(item), + "date": get_published(item), + "summary": get_summary(item), + "tags": get_tags(item), + } + + # Simply skip over any items that are "badly formed". This is + # preferable to simply letting exceptions be raised when saving + # an Article as "badly formed" feed are not that unusual, so it's + # better to ignore any missing data as it's likely to be present + # at a later time, for example, an author forgets to add a title + # to the article and publishes the post. + + required = ["identifier", "title", "url", "date"] + + if not all([v for k, v in values.items() if k in required]): + continue + + article: Article = get_article(str(values.pop("identifier"))) + + article.title = values["title"] # type: ignore + article.url = values["url"] # type: ignore + article.date = values["date"] # type: ignore + article.summary = values["summary"] # type: ignore + article.data["feed:tags"] = values["tags"] + + if article.pk is None: + num_created += 1 + article.feed = feed + article.source = feed.source + article.publish = feed.auto_publish + + article.save() + + for category in feed.categories.all(): + article.categories.add(category) + + for author in get_authors(item, feed): + article.authors.add(author) + + # Note the slight change in wording compared to when the feed status + # was 304, see above. That makes it easier to tell what happened simply + # by reading the message and not looking at the other values logged. + + message = "Feed was unchanged" if num_created == 0 else "Feed was loaded" + + log.debug( + message, + extra={ + "feed": feed.name, + "num_entries": num_entries, + "num_created": num_created, + }, + ) + + return True + + +def get_source(feed: Feed) -> str: + # Feedparse can parse a file, url or string so this method exists to + # allow an XML string to be injected for testing. + return feed.url + + +def get_user_agent() -> str: + return settings.FEEDS_USER_AGENT + + +def get_article(identifier: str) -> Article: + if Article.objects.with_identifier(identifier).exists(): + article = Article.objects.with_identifier(identifier).get() + else: + article = Article(identifier=identifier) + return article + + +def get_identifier(item: FeedParserDict) -> str: + # Use the entry link as a last resort as the feed would likely fail + # validation. This happens surprisingly often. + return item.get("id", item.get("link", "")) + + +def get_url(item: FeedParserDict) -> str: + try: + if link := item.get("link", ""): + validate_url(link) + except ValidationError: + link = "" + + return link + + +def get_title(item: FeedParserDict) -> str: + # Some feeds, e.g. cartoons, might just contain an image so there is no title + return normalize_title(unescape(item.get("title", ""))) + + +def get_published(item: FeedParserDict) -> datetime: + if date := item.get("published", None): + date = parse_date(date) + return date + + +def get_updated(item: FeedParserDict) -> datetime: + if date := item.get("updated", None): + date = parse_date(date) + return date + + +def get_authors(item: FeedParserDict, feed: Feed) -> List[Author]: + authors: List[Author] = [] + + names = [ + author.get("name") # noqa + for author in item.get("authors", []) + if author.get("name") # noqa + ] + + for name in names: + author: Optional[Author] + alias: Optional[Alias] = Alias.objects.filter(name=name, feed=feed).first() + + if alias: + author = alias.author + else: + slug: str = slugify(name) + + try: + # Check if the Author exists using the slug. That way case changes + # in the Author's name do not result in multiple authors. + author, created = Author.objects.get_or_create( + slug=slug, defaults={"name": name} + ) + except Author.MultipleObjectsReturned: + log = logging.getLogger(f"{__name__}.load_feed") + log.exception("Multiple Authors found", extra={"author": name}) + author = Author.objects.with_slug(slug).first() + + if author: + authors.append(author) + + return authors + + +def get_summary(item: FeedParserDict) -> str: + return unescape(item.get("summary", "")) + + +def get_tags(item: FeedParserDict) -> List[str]: + return sorted([tag.get("term") for tag in item.get("tags", []) if tag.get("term")]) + + +def normalize_title(value: str) -> str: + # Skip changes if the title is empty + if not value: + return value + # Strip double-quotes around a title + if value[0] == value[-1] == '"': + value = value[1:-1] + # Strip single-quotes around a title + if value[0] == value[-1] == "'": + value = value[1:-1] + # Remove any trailing periods but leave an ellipsis alone + if value[-1] == "." and value[-2] != ".": + value = value[:-1] + # Remove extra whitespace + value = " ".join(value.split()) + # Put a space before and after a hyphen + value = re.sub(r"(\w(\")?)-", r"\1 -", value) + value = re.sub(r"-((\")?(\w))", r"- \1", value) + # Put a space after a comma + value = re.sub(r"(\w),(\w)", r"\1, \2", value) + # Remove a space before a comma + value = re.sub(r"(\w) ,", r"\1,", value) + # Put a space before an opening bracket + value = re.sub(r"(\w)\(", r"\1 (", value) + # Put a space after a closing bracket + value = re.sub(r"\)(\w)", r") \1", value) + return value diff --git a/src/feeds/settings.py b/src/feeds/settings.py new file mode 100644 index 0000000..957ceba --- /dev/null +++ b/src/feeds/settings.py @@ -0,0 +1,55 @@ +""" +Django settings for django-feeds. + +""" +import os + +from django.core.exceptions import ImproperlyConfigured + +from croniter import croniter + +# ######### +# FEEDS +# ######### + +# Feeds are loaded using a celery task. You can schedule how often this task +# runs using the FEEDS_TASK_SCHEDULE setting which defaults to run every hour +# on the hour. Rather than fetch every feed each time you can set a schedule +# on individual feeds to reduce server load and bandwidth. If no schedule is +# set then the FEEDS_LOAD_SCHEDULE setting is used. When the celery task runs +# it compares the feed's schedule with the current time and if they match, +# the feed is loaded. +# +# IMPORTANT: the times represented by the FEEDS_TASK_SCHEDULE setting and the +# FEEDS_LOAD_SCHEDULE setting must coincide otherwise the feed will never be +# loaded. Since the celery task runs on the hour, you can add a little margin +# for error by setting the minute field for the feed schedule to "*" rather +# than "0". That way, if the task gets blocked temporarily as long as it gets +# executed in the next 59 minutes, the schedules will match and the feed will +# still be loaded. + +FEEDS_TASK_SCHEDULE = os.environ.get("FEEDS_TASK_SCHEDULE", "0 * * * *") + +if not croniter.is_valid(FEEDS_TASK_SCHEDULE): + raise ImproperlyConfigured("FEEDS_TASK_SCHEDULE setting is not a valid cron entry") + +FEEDS_LOAD_SCHEDULE = os.environ.get("FEEDS_LOAD_SCHEDULE", FEEDS_TASK_SCHEDULE) + +if not croniter.is_valid(FEEDS_LOAD_SCHEDULE): + raise ImproperlyConfigured("FEEDS_LOAD_SCHEDULE setting is not a valid cron entry") + +# A default user-agent string that is used when loading RSS feeds. Some sites +# will return an error is the user-agent is not given. + +FEEDS_USER_AGENT = os.environ.get("FEEDS_USER_AGENT", "Django Feeds") + +# ############ +# Tagulous +# ############ + +SERIALIZATION_MODULES = { + "xml": "tagulous.serializers.xml_serializer", + "json": "tagulous.serializers.json", + "python": "tagulous.serializers.python", + "yaml": "tagulous.serializers.pyyaml", +} diff --git a/src/feeds/templates/admin/merge_authors.html b/src/feeds/templates/admin/merge_authors.html new file mode 100644 index 0000000..81e1334 --- /dev/null +++ b/src/feeds/templates/admin/merge_authors.html @@ -0,0 +1,52 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n admin_urls static %} + +{% block extrahead %} + {{ block.super }} + {{ media }} + +{% endblock %} + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %} + +{% block breadcrumbs %} +
+{% translate 'Home' %} +› {{ opts.app_config.verbose_name }} +› {{ opts.verbose_name_plural|capfirst }} +› {% translate 'Merge' %} +
+{% endblock %} + +{% block content %} +

+ {% translate "Selected authors" %} +

+ +
{% csrf_token %} +
+
    + {% for obj in queryset %} +
  • + {{ obj.name }} + +
    • + {% endfor %} +
+ +
+ + +
+ + + + + {% translate "No, take me back" %} +
+
+{% endblock %} diff --git a/src/feeds/templates/admin/merge_tags.html b/src/feeds/templates/admin/merge_tags.html new file mode 100644 index 0000000..a4da2b6 --- /dev/null +++ b/src/feeds/templates/admin/merge_tags.html @@ -0,0 +1,52 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n admin_urls static %} + +{% block extrahead %} + {{ block.super }} + {{ media }} + +{% endblock %} + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %} + +{% block breadcrumbs %} +
+{% translate 'Home' %} +› {{ opts.app_config.verbose_name }} +› {{ opts.verbose_name_plural|capfirst }} +› {% translate 'Merge' %} +
+{% endblock %} + +{% block content %} +

+ {% translate "Selected tags" %} +

+ +
{% csrf_token %} +
+
    + {% for obj in queryset %} +
  • + {{ obj.name }} + +
    • + {% endfor %} +
+ +
+ + +
+ + + + + {% translate "No, take me back" %} +
+
+{% endblock %} diff --git a/src/app_project/tests/__init__.py b/src/feeds/templatetags/__init__.py similarity index 100% rename from src/app_project/tests/__init__.py rename to src/feeds/templatetags/__init__.py diff --git a/src/feeds/templatetags/feeds.py b/src/feeds/templatetags/feeds.py new file mode 100644 index 0000000..d95c9b9 --- /dev/null +++ b/src/feeds/templatetags/feeds.py @@ -0,0 +1,19 @@ +from django import template + +register = template.Library() + + +@register.filter +def categories(value, arg): + """Returns all the Category objects on an Article or Feed with a given prefix. + + The set of matching Categories objects is created using a list comprehension + as it assumes the categories were prefetched whereas filtering the QuerySet + would trigger another fetch from the database. + + """ + return [ + category + for category in value.categories.all() + if category.name.startswith(arg) and category.level > 1 + ] diff --git a/src/feeds/tests/__init__.py b/src/feeds/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/admin/__init__.py b/src/feeds/tests/admin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/admin/test_alias_admin.py b/src/feeds/tests/admin/test_alias_admin.py new file mode 100644 index 0000000..a43a0b7 --- /dev/null +++ b/src/feeds/tests/admin/test_alias_admin.py @@ -0,0 +1,12 @@ +from django.test import TestCase + +import pytest + +from feeds.tests.factories import AliasFactory +from feeds.tests.mixins import AdminTests + + +@pytest.mark.django_db +class AliasAdminTests(AdminTests, TestCase): + factory_class = AliasFactory + query_count = 5 diff --git a/src/feeds/tests/admin/test_article_admin.py b/src/feeds/tests/admin/test_article_admin.py new file mode 100644 index 0000000..6521030 --- /dev/null +++ b/src/feeds/tests/admin/test_article_admin.py @@ -0,0 +1,12 @@ +from django.test import TestCase + +import pytest + +from feeds.tests.factories import ArticleFactory +from feeds.tests.mixins import AdminTests + + +# @pytest.mark.django_db +# class ArticleAdminTests(AdminTests, TestCase): +# factory_class = ArticleFactory +# query_count = 9 diff --git a/src/feeds/tests/admin/test_author_admin.py b/src/feeds/tests/admin/test_author_admin.py new file mode 100644 index 0000000..4de6c63 --- /dev/null +++ b/src/feeds/tests/admin/test_author_admin.py @@ -0,0 +1,62 @@ +from random import choice + +from django.test import TestCase + +import pytest +from django.urls import reverse + +from feeds.models import Author +from feeds.tests.factories import ArticleFactory, AuthorFactory +from feeds.tests.mixins import AdminTests + +pytestmark = pytest.mark.django_db + + +class AuthorAdminTests(AdminTests, TestCase): + factory_class = AuthorFactory + query_count = 6 + + +def test_merge_authors_action(admin_client): + authors = AuthorFactory.create_batch(5) + article = ArticleFactory(authors=authors) + selected = choice(authors) + + url = reverse("admin:feeds_author_changelist") + data = { + "post": "yes", + "_selected_action": [author.pk for author in authors], + "action": "merge_authors", + "selected": selected.pk, + } + + admin_client.post(url, data, follow=True) + + # Merged authors are deleted + actual = [author.pk for author in Author.objects.all()] + assert actual == [selected.pk] + + # Merged author are replaced + actual = [author.pk for author in article.authors.all()] + assert actual == [selected.pk] + + +def test_show_articles_for_authors_action(admin_client): + authors = AuthorFactory.create_batch(5) + selected = choice(authors) + ArticleFactory.create_batch(10) + + url = reverse("admin:feeds_author_changelist") + data = { + "post": "yes", + "_selected_action": [selected.pk], + "action": "show_articles", + } + + response = admin_client.post(url, data, follow=True) + + request = response.request + slugs = selected.slug + + assert request["PATH_INFO"] == reverse("admin:feeds_article_changelist") + assert request["QUERY_STRING"] == f"authors__slug__in={slugs}" diff --git a/src/feeds/tests/admin/test_feed_admin.py b/src/feeds/tests/admin/test_feed_admin.py new file mode 100644 index 0000000..54e9ff4 --- /dev/null +++ b/src/feeds/tests/admin/test_feed_admin.py @@ -0,0 +1,34 @@ +from unittest import mock + +from django.test import TestCase + +import pytest +from django.urls import reverse + +from feeds.tests.factories import FeedFactory +from feeds.tests.mixins import AdminTests + + +pytestmark = pytest.mark.django_db + + +class FeedAdminTests(AdminTests, TestCase): + factory_class = FeedFactory + query_count = 6 + + +def mock_load_feed(feed): + pass + + +@mock.patch("feeds.services.feeds.load_feed") +def test_feed_load_feed_action(mocked, admin_client): + mocked.side_effect = mock_load_feed + feed = FeedFactory.create() + url = reverse("admin:feeds_feed_changelist") + data = {"action": "load_feed", "_selected_action": [str(feed.pk)]} + + response = admin_client.post(url, data, follow=True) + + assert response.status_code == 200 + assert mocked.call_count == 1 diff --git a/src/feeds/tests/admin/test_source_admin.py b/src/feeds/tests/admin/test_source_admin.py new file mode 100644 index 0000000..cbd66d3 --- /dev/null +++ b/src/feeds/tests/admin/test_source_admin.py @@ -0,0 +1,37 @@ +from random import choice + +from django.test import TestCase + +import pytest +from django.urls import reverse + +from feeds.tests.factories import ArticleFactory, SourceFactory +from feeds.tests.mixins import AdminTests + +pytestmark = pytest.mark.django_db + + +class SourceAdminTests(AdminTests, TestCase): + factory_class = SourceFactory + query_count = 5 + + +def test_show_articles_for_sources_action(admin_client): + sources = SourceFactory.create_batch(5) + selected = choice(sources) + ArticleFactory.create_batch(10) + + url = reverse("admin:feeds_source_changelist") + data = { + "post": "yes", + "_selected_action": [selected.pk], + "action": "show_articles", + } + + response = admin_client.post(url, data, follow=True) + + request = response.request + slugs = selected.slug + + assert request["PATH_INFO"] == reverse("admin:feeds_article_changelist") + assert request["QUERY_STRING"] == f"source__slug__in={slugs}" diff --git a/src/feeds/tests/admin/test_tag_admin.py b/src/feeds/tests/admin/test_tag_admin.py new file mode 100644 index 0000000..01ab956 --- /dev/null +++ b/src/feeds/tests/admin/test_tag_admin.py @@ -0,0 +1,63 @@ +from random import choice + +from django.test import TestCase + +import pytest +from django.urls import reverse + +from feeds.models import Tag +from feeds.tests.factories import ArticleFactory, TagFactory +from feeds.tests.mixins import AdminTests + + +pytestmark = pytest.mark.django_db + + +class TagAdminTests(AdminTests, TestCase): + factory_class = TagFactory + query_count = 6 + + +def test_merge_tags_action(admin_client): + tags = TagFactory.create_batch(5) + article = ArticleFactory(tags=tags) + selected = choice(tags) + + url = reverse("admin:feeds_tag_changelist") + data = { + "post": "yes", + "_selected_action": [tag.pk for tag in tags], + "action": "merge_tags", + "selected": selected.pk, + } + + admin_client.post(url, data, follow=True) + + # Merged tags are deleted + actual = [tag.pk for tag in Tag.objects.all()] + assert actual == [selected.pk] + + # Merged tags are replaced + actual = [tag.pk for tag in article.tags.all()] + assert actual == [selected.pk] + + +def test_show_articles_for_tag_action(admin_client): + tags = TagFactory.create_batch(5) + selected = choice(tags) + ArticleFactory.create_batch(10) + + url = reverse("admin:feeds_tag_changelist") + data = { + "post": "yes", + "_selected_action": [selected.pk], + "action": "show_articles", + } + + response = admin_client.post(url, data, follow=True) + + request = response.request + slugs = selected.slug + + assert request["PATH_INFO"] == reverse("admin:feeds_article_changelist") + assert request["QUERY_STRING"] == f"tags__slug__in={slugs}" diff --git a/src/feeds/tests/conditions/__init__.py b/src/feeds/tests/conditions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/conditions/articles.py b/src/feeds/tests/conditions/articles.py new file mode 100644 index 0000000..f4028de --- /dev/null +++ b/src/feeds/tests/conditions/articles.py @@ -0,0 +1,21 @@ +from feeds.models import Article + + +def only_published_articles(articles, timestamp) -> bool: + return all(article.date and article.date < timestamp for article in articles) + + +def article_was_clicked(obj: Article) -> bool: + views = obj.views + obj.refresh_from_db() + return views + 1 == obj.views + + +def only_tags_for_articles(articles, tags) -> bool: + expected = set([tag.pk for article in articles for tag in article.tags.all()]) + actual = [tag.pk for tag in tags] + return sorted(actual) == sorted(expected) + + +def only_articles_between_dates(articles, start_date, end_date) -> bool: + return all(start_date <= article.date.date() <= end_date for article in articles) diff --git a/src/feeds/tests/conditions/emails.py b/src/feeds/tests/conditions/emails.py new file mode 100644 index 0000000..7456e76 --- /dev/null +++ b/src/feeds/tests/conditions/emails.py @@ -0,0 +1,28 @@ +from django.core import mail + + +def matches(obj, **kwargs): + results = [] + for k, v in kwargs.items(): + if hasattr(obj, k) and getattr(obj, k): + results.append(True) + else: + results.append(False) + return all(results) + + +def emails_equal(e1, e2) -> bool: + return ( + e1.to == e2.to and e1.from_email == e2.from_email and e1.subject == e2.subject + ) + + +def email_was_sent(sender, receiver) -> bool: + def _matches(item): + return matches(item, to=[receiver], from_email=sender) + + return any(map(_matches, mail.outbox)) + + +def email_not_sent() -> bool: + return len(mail.outbox) == 0 diff --git a/src/feeds/tests/conditions/forms.py b/src/feeds/tests/conditions/forms.py new file mode 100644 index 0000000..eb22640 --- /dev/null +++ b/src/feeds/tests/conditions/forms.py @@ -0,0 +1,21 @@ +def data_is_empty(form) -> bool: + return form.data == {} + + +def cleaned_data_is_empty(form) -> bool: + return form.cleaned_data == {} + + +def form_is_bound(response, key="form") -> bool: + form = response.context.get(key) + return form.is_bound + + +def form_is_valid(response, key="form") -> bool: + form = response.context.get(key) + return form.is_valid() + + +def form_is_reset(response, key="form") -> bool: + form = response.context.get(key) + return form and data_is_empty(form) and cleaned_data_is_empty(form) diff --git a/src/feeds/tests/conditions/models.py b/src/feeds/tests/conditions/models.py new file mode 100644 index 0000000..5aecd46 --- /dev/null +++ b/src/feeds/tests/conditions/models.py @@ -0,0 +1,16 @@ +from django.forms import model_to_dict + + +def get_fields(obj): + return model_to_dict(obj, fields=[field.name for field in obj._meta.fields]) # noqa + + +def get_updated(obj): + return obj._meta.model.objects.get(pk=obj.id) # noqa + + +def changed_fields(obj, *names) -> bool: + d1 = get_fields(obj) + d2 = get_fields(get_updated(obj)) + changed = [k for k, v in d1.items() if v != d2[k]] + return sorted(names) == sorted(changed) diff --git a/src/feeds/tests/conditions/querysets.py b/src/feeds/tests/conditions/querysets.py new file mode 100644 index 0000000..79f537f --- /dev/null +++ b/src/feeds/tests/conditions/querysets.py @@ -0,0 +1,17 @@ +from typing import Union + +from django.db.models import QuerySet +from django.template.response import TemplateResponse + + +def is_paginated(response: TemplateResponse, key: str = "paginator") -> bool: + paginator = response.context[key] + return paginator.num_pages > 1 + + +def is_ordered(obj: Union[QuerySet, TemplateResponse], key: str = "paginator") -> bool: + if isinstance(obj, QuerySet): # type: ignore + return obj.ordered + elif isinstance(obj, TemplateResponse): + return obj.context[key].object_list.ordered + return False diff --git a/src/feeds/tests/conditions/responses.py b/src/feeds/tests/conditions/responses.py new file mode 100644 index 0000000..68ee514 --- /dev/null +++ b/src/feeds/tests/conditions/responses.py @@ -0,0 +1,37 @@ +from django.contrib import messages + + +def redirects_to(response, url) -> bool: + if hasattr(response, "redirect_chain"): + redirect_url, status_code = response.redirect_chain[-1] + else: + status_code = response.status_code + redirect_url = response.url + return status_code == 302 and redirect_url == url + + +def is_error_message(message) -> bool: + return message.level == messages.ERROR + + +def is_warning_message(message) -> bool: + return message.level == messages.WARNING + + +def is_success_message(message) -> bool: + return message.level == messages.SUCCESS + + +def success_message_shown(response) -> bool: + objs = list(response.context.get("messages", [])) + return bool(objs and len(objs) == 1) and is_success_message(objs[0]) + + +def warning_message_shown(response) -> bool: + objs = list(response.context.get("messages", [])) + return bool(objs and len(objs) == 1) and is_warning_message(objs[0]) + + +def error_message_shown(response) -> bool: + objs = list(response.context.get("messages", [])) + return bool(objs and len(objs) == 1) and is_error_message(objs[0]) diff --git a/src/feeds/tests/conftest.py b/src/feeds/tests/conftest.py new file mode 100644 index 0000000..729a6f8 --- /dev/null +++ b/src/feeds/tests/conftest.py @@ -0,0 +1,23 @@ +import pytest + + +# Override the local setting used for development to ensure that feeds +# are always loaded by default, regardless of the time that tests are +# run. This reduces the amount of setup needed for feed-related tests. +@pytest.fixture(autouse=True) +def use_flexible_load_schedule(settings): + settings.FEEDS_LOAD_SCHEDULE = "* * * * *" + + +@pytest.fixture(autouse=True) +def use_dummy_cache_backend(settings): + settings.CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + } + } + + +@pytest.fixture(autouse=True) +def disable_open_graph_loads(settings): + settings.LOAD_OPEN_GRAPH_DATA = False diff --git a/src/feeds/tests/factories/__init__.py b/src/feeds/tests/factories/__init__.py new file mode 100644 index 0000000..2ff09ef --- /dev/null +++ b/src/feeds/tests/factories/__init__.py @@ -0,0 +1,30 @@ +import factory # type: ignore + +from feeds.tests.providers import html + +from .alias import AliasFactory +from .article import ArticleFactory +from .author import AuthorFactory +from .category import CategoryFactory +from .feed import FeedFactory +from .source import SourceFactory +from .tag import TagFactory +from .user import UserFactory + +__all__ = [ + "AliasFactory", + "ArticleFactory", + "AuthorFactory", + "CategoryFactory", + "FeedFactory", + "SourceFactory", + "TagFactory", + "UserFactory", +] + +# Register the custom providers here, so they are available anywhere +# the factories are used. This could be added to the root conftest, +# but we occasionally want to be able to create instances of factories +# in the django shell, particularly for ad-hoc testing. + +factory.Faker.add_provider(html.Provider) diff --git a/src/feeds/tests/factories/alias.py b/src/feeds/tests/factories/alias.py new file mode 100644 index 0000000..c003743 --- /dev/null +++ b/src/feeds/tests/factories/alias.py @@ -0,0 +1,15 @@ +import factory # type: ignore +from faker import Faker + +from feeds.models import Alias + +fake = Faker() + + +class AliasFactory(factory.django.DjangoModelFactory): + class Meta: + model = Alias + + name = factory.Faker("user_name") + author = factory.SubFactory("feeds.tests.factories.AuthorFactory") + feed = factory.SubFactory("feeds.tests.factories.FeedFactory", enabled=True) diff --git a/src/feeds/tests/factories/article.py b/src/feeds/tests/factories/article.py new file mode 100644 index 0000000..a534dd3 --- /dev/null +++ b/src/feeds/tests/factories/article.py @@ -0,0 +1,132 @@ +import random + +from django.utils import timezone + +import factory # type: ignore +from faker import Faker + +from feeds.models import Article + +fake = Faker() + +tzinfo = timezone.get_default_timezone() + +# The distributions are used to set the probability that an Article will have +# a given number of tags, authors or clicks. + +tag_distribution = [0] * 10 + [1] * 70 + [2] * 10 + [3] * 10 +category_distribution = [0] * 70 + [1] * 20 + [2] * 10 +author_distribution = [0] * 5 + [1] * 80 + [2] * 10 + [3] * 5 +click_distribution = [0] * 5 + [1] * 80 + [2] * 10 + [3] * 5 + + +class ArticleFactory(factory.django.DjangoModelFactory): + class Meta: + model = Article + + # When articles are generated from the entries are loaded from a feed + # any trailing period is removed so all the titles across all the feeds + # are displayed consistently. Here we leave the trailing period in place + # since we don't specifically test for its presence, and it keeps the + # code simple. + # + # http_status is a custom provider which returns a status of 200 OK 95% + # of the time and 404 Not Found for the remaining 5%, which is much + # larger than the observed rate in production. The status code is not + # used to filter Articles (just yet) but we want to create a data set + # that is close to reality. + # + # The publication date is set for a random time in the past two weeks + # since the home page is paginated by week and this will generate a + # set of Articles across multiple pages. + # + # The 'publish' flag is set to True 80% of the time so the list of + # articles in any given week contains a mix of published and unpublished + # items. + # + # The post_generation hooks for tags and authors use in order + # of precedence: + # + # 1. Any objects passed as arguments to the factory. + # 2. Any existing objects from the database. + # 3. New instances. + # + # The number of objects depends on the probability distributions defined + # above. + + title = factory.Faker("sentence") + url = factory.Faker("url") + date = factory.Faker("date_time_between", start_date="-2w", tzinfo=tzinfo) + summary = factory.Faker("paragraph") + publish = factory.Faker("boolean", chance_of_getting_true=95) + source = factory.SubFactory("feeds.tests.factories.SourceFactory") + identifier = factory.Faker("url") + feed = factory.SubFactory("feeds.tests.factories.FeedFactory") + data = {"default": "value"} # type: ignore + views = factory.Faker('pyint', min_value=0, max_value=100) + + @factory.post_generation + def categories(self, create, extracted, **kwargs): + if create: + if extracted: + self.categories.set(extracted) + else: + from .category import Category, CategoryFactory + + count = random.choice(category_distribution) + + if Category.objects.exists(): + categories = list(Category.objects.all()) + for n in range(count): + self.categories.add(random.choice(categories)) + else: + for n in range(count): + self.categories.add(CategoryFactory()) + + @factory.post_generation + def tags(self, create, extracted, **kwargs): + if create: + if extracted: + self.tags.set(extracted) + else: + from .tag import Tag, TagFactory + + count = random.choice(tag_distribution) + + if Tag.objects.exists(): + tags = list(Tag.objects.all()) + for n in range(count): + self.tags.add(random.choice(tags)) + else: + for n in range(count): + self.tags.add(TagFactory()) + + @factory.post_generation + def authors(self, create, extracted, **kwargs): + if create: + if extracted: + self.authors.set(extracted) + else: + from .author import Author, AuthorFactory + + count = random.choice(author_distribution) + + if Author.objects.exists(): + authors = list(Author.objects.all()) + for n in range(count): + self.authors.add(random.choice(authors)) + else: + for n in range(count): + self.authors.add(AuthorFactory()) + + @factory.post_generation # type: ignore + def data(self, create, extracted, **kwargs): + if extracted: + self.data = extracted + else: + self.data = { + "og:title": self.title, + "og:url": self.url, + } + if create: + self.save() diff --git a/src/feeds/tests/factories/author.py b/src/feeds/tests/factories/author.py new file mode 100644 index 0000000..fd4900c --- /dev/null +++ b/src/feeds/tests/factories/author.py @@ -0,0 +1,23 @@ +import factory # type: ignore +from faker import Faker + +from feeds.models import Author + +fake = Faker() + + +class AuthorFactory(factory.django.DjangoModelFactory): + class Meta: + model = Author + django_get_or_create = ("slug",) + + # Although there is a many-to-many relationship between Article + # and Author we don't create a post_generation hook for authors + # since that adds extra code for no added value. It's better to + # have a single 'entry-point' for generating Articles and pass + # an Author object as an argument if we want to generate a list + # of Articles for a given Author. + + name = factory.Faker("name") + slug = factory.Faker("slug") + description = factory.Faker("html_text") diff --git a/src/feeds/tests/factories/category.py b/src/feeds/tests/factories/category.py new file mode 100644 index 0000000..6f81b3a --- /dev/null +++ b/src/feeds/tests/factories/category.py @@ -0,0 +1,14 @@ +import factory # type: ignore +from faker import Faker + +from feeds.models import Category + +fake = Faker() + + +class CategoryFactory(factory.django.DjangoModelFactory): + class Meta: + model = Category + + name = factory.Faker("word") + slug = factory.Faker("slug") diff --git a/src/feeds/tests/factories/feed.py b/src/feeds/tests/factories/feed.py new file mode 100644 index 0000000..6ba003f --- /dev/null +++ b/src/feeds/tests/factories/feed.py @@ -0,0 +1,42 @@ +import random + +import factory # type: ignore +from faker import Faker + +from feeds.models import Feed + +fake = Faker() + + +# The distributions are used to set the probability that a Feed will have +# a given number of categories. + +category_distribution = [0] * 0 + [1] * 70 + [2] * 20 + [10] * 3 + + +class FeedFactory(factory.django.DjangoModelFactory): + class Meta: + model = Feed + + name = factory.Faker("word") + url = factory.Faker("url") + source = factory.SubFactory("feeds.tests.factories.SourceFactory") + auto_publish = factory.Faker("boolean", chance_of_getting_true=90) + + @factory.post_generation + def categories(self, create, extracted, **kwargs): + if create: + if extracted: + self.categories.set(extracted) + else: + from .category import Category, CategoryFactory + + count = random.choice(category_distribution) + + if Category.objects.exists(): + categories = list(Category.objects.all()) + for n in range(count): + self.categories.add(random.choice(categories)) + else: + for n in range(count): + self.categories.add(CategoryFactory()) diff --git a/src/feeds/tests/factories/source.py b/src/feeds/tests/factories/source.py new file mode 100644 index 0000000..9a9a4cd --- /dev/null +++ b/src/feeds/tests/factories/source.py @@ -0,0 +1,29 @@ +import factory # type: ignore +from faker import Faker + +from feeds.models import Source + +fake = Faker() + + +class SourceFactory(factory.django.DjangoModelFactory): + class Meta: + model = Source + django_get_or_create = ("slug",) + + name = factory.Faker("name") + slug = factory.Faker("slug") + url = factory.Faker("url") + data = {"default": "value"} # type: ignore + + @factory.post_generation # type: ignore + def data(self, create, extracted, **kwargs): + if extracted: + self.data = extracted + else: + self.data = { + "og:title": self.name, + "og:url": self.url, + } + if create: + self.save() diff --git a/src/feeds/tests/factories/tag.py b/src/feeds/tests/factories/tag.py new file mode 100644 index 0000000..f0ff20a --- /dev/null +++ b/src/feeds/tests/factories/tag.py @@ -0,0 +1,17 @@ +import factory # type: ignore +from faker import Faker + +from feeds.models import Tag + +fake = Faker() + + +class TagFactory(factory.django.DjangoModelFactory): + class Meta: + model = Tag + django_get_or_create = ("slug",) + + name = factory.Faker("word") + slug = factory.Faker("slug") + summary = factory.Faker("paragraph") + description = factory.Faker("html_text") diff --git a/src/feeds/tests/factories/user.py b/src/feeds/tests/factories/user.py new file mode 100644 index 0000000..cb470dc --- /dev/null +++ b/src/feeds/tests/factories/user.py @@ -0,0 +1,18 @@ +from django.contrib.auth.models import User + +import factory # type: ignore + + +class UserFactory(factory.django.DjangoModelFactory): + class Meta: + model = User + django_get_or_create = ("username",) + + username = factory.Faker("user_name") + first_name = factory.Faker("first_name") + last_name = factory.Faker("last_name") + email = factory.Faker("email") + + is_staff = False + is_active = True + is_superuser = False diff --git a/src/feeds/tests/mixins/__init__.py b/src/feeds/tests/mixins/__init__.py new file mode 100644 index 0000000..8a53c3a --- /dev/null +++ b/src/feeds/tests/mixins/__init__.py @@ -0,0 +1,3 @@ +from .admin_tests import AdminTests + +__all__ = ("AdminTests",) diff --git a/src/feeds/tests/mixins/admin_tests.py b/src/feeds/tests/mixins/admin_tests.py new file mode 100644 index 0000000..abee12b --- /dev/null +++ b/src/feeds/tests/mixins/admin_tests.py @@ -0,0 +1,67 @@ +from django.contrib.contenttypes.models import ContentType +from django.urls import reverse + +import pytest + +from feeds.tests.factories import UserFactory + + +@pytest.mark.django_db +class AdminTests: + """Tests for the Django Admin + + The tests only check the changelist view, since that is where most of + the errors will occur. There is little value in testing the add, change + or delete views since we would just be testing Django itself. When these + are customised or if the on_delete rules in the underlying models do not + allow cascaded deletes, generic tests would either not work or add very + little value. + + """ + + # The FactoryBoy class for creating objects + factory_class = None + # The number of objects to create + batch_size = 10 + + @classmethod + def setUpTestData(cls): + cls.admin = UserFactory(is_staff=True, is_superuser=True) + cls.model = cls.factory_class._meta.model # noqa + cls.app_label = cls.model._meta.app_label # noqa + cls.model_name = cls.model._meta.model_name # noqa + cls.objects = cls.factory_class.create_batch(cls.batch_size) + + def clear_query_caches(self): # noqa + ContentType.objects.clear_cache() + + def setUp(self): + self.client.force_login(self.admin) # noqa + + def test_changelist_view(self): + """Verify the changelist view is displayed without any errors""" + url = reverse("admin:%s_%s_changelist" % (self.app_label, self.model_name)) + response = self.client.get(url) # noqa + assert response.status_code == 200 + + def test_changelist_search(self): + """Verify search_fields does not contain any invalid lookups""" + url = reverse("admin:%s_%s_changelist" % (self.app_label, self.model_name)) + response = self.client.get(url, {"q": "test"}, follow=True) # noqa + assert response.status_code == 200 + + def test_changelist_query_count(self): + """Verify the number of queries is fixed when a new object is added""" + url = reverse("admin:%s_%s_changelist" % (self.app_label, self.model_name)) + + self.clear_query_caches() + with self.assertNumQueries(self.query_count): # noqa + self.client.get(url) # noqa + + # Add another object + self.factory_class() + + self.clear_query_caches() + # The number of queries should remain the same + with self.assertNumQueries(self.query_count): # noqa + self.client.get(url) # noqa diff --git a/src/feeds/tests/models/__init__.py b/src/feeds/tests/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/models/test_article.py b/src/feeds/tests/models/test_article.py new file mode 100644 index 0000000..b3d8782 --- /dev/null +++ b/src/feeds/tests/models/test_article.py @@ -0,0 +1,29 @@ +import datetime as dt + +from django.utils import timezone + +import pytest + +from feeds.models import Article +from feeds.tests.factories import ArticleFactory + +pytestmark = pytest.mark.django_db + + +def test_published(): + today = timezone.now() + ArticleFactory.create(title="first", date=today, publish=True) + ArticleFactory.create(title="second", date=today, publish=False) + queryset = Article.objects.published() + assert queryset.count() == 1 + assert queryset.first().title == "first" + + +def test_for_date(): + today = timezone.now() + ArticleFactory.create(title="first", date=today - dt.timedelta(days=1)) + ArticleFactory.create(title="second", date=today) + ArticleFactory.create(title="first", date=today + dt.timedelta(days=1)) + queryset = Article.objects.for_date(today) + assert queryset.count() == 1 + assert queryset.first().title == "second" diff --git a/src/feeds/tests/models/test_feed.py b/src/feeds/tests/models/test_feed.py new file mode 100644 index 0000000..f2341c6 --- /dev/null +++ b/src/feeds/tests/models/test_feed.py @@ -0,0 +1,49 @@ +from django.core.exceptions import ValidationError +from django.utils import timezone + +import pytest + +from feeds.models import Feed +from feeds.tests.factories import FeedFactory + +pytestmark = pytest.mark.django_db + + +def test_schedule_invalid(): + with pytest.raises(ValidationError): + FeedFactory(schedule="0 10, 12 * * *").full_clean() + + +def test_schedule_minutes_valid(): + FeedFactory(schedule="0 10 * * *").full_clean() + FeedFactory(schedule="* 10 * * *").full_clean() + + +def test_schedule_minutes_invalid(): + with pytest.raises(ValidationError): + FeedFactory(schedule="15 10 * * *").full_clean() + + +@pytest.mark.freeze_time +def test_feed_scheduled(freezer): + feed = FeedFactory.create(enabled=True, schedule="0 10 * * *") + freezer.move_to(timezone.now().replace(minute=0, hour=10)) + feeds = Feed.objects.scheduled_for(timezone.now()) + assert feed.pk in [feed.pk for feed in feeds] + + +@pytest.mark.freeze_time +def test_unscheduled_feed_skipped(monkeypatch, freezer): + feed = FeedFactory.create(enabled=True, schedule="0 11 * * *") + freezer.move_to(timezone.now().replace(minute=0, hour=10)) + feeds = Feed.objects.scheduled_for(timezone.now()) + assert feed.pk not in [feed.pk for feed in feeds] + + +@pytest.mark.freeze_time +def test_default_schedule_used(monkeypatch, freezer, settings): + feed = FeedFactory.create(enabled=True, schedule="") + settings.FEEDS_LOAD_SCHEDULE = "0 10 * * *" + freezer.move_to(timezone.now().replace(minute=0, hour=10)) + feeds = Feed.objects.scheduled_for(timezone.now()) + assert feed.pk in [feed.pk for feed in feeds] diff --git a/src/feeds/tests/providers/__init__.py b/src/feeds/tests/providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/providers/html.py b/src/feeds/tests/providers/html.py new file mode 100644 index 0000000..729c7ad --- /dev/null +++ b/src/feeds/tests/providers/html.py @@ -0,0 +1,9 @@ +from faker import Faker +from faker.providers import BaseProvider + +fake = Faker() + + +class Provider(BaseProvider): + def html_text(self): # noqa + return "

%s

%s

" % (fake.sentence(), fake.paragraph()) diff --git a/src/feeds/tests/services/__init__.py b/src/feeds/tests/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/feeds/tests/services/test_load_feed.py b/src/feeds/tests/services/test_load_feed.py new file mode 100644 index 0000000..4f7d0c9 --- /dev/null +++ b/src/feeds/tests/services/test_load_feed.py @@ -0,0 +1,183 @@ +from django.template import Context, Template +from django.utils import timezone + +import pytest + +from feeds.models import Article +from feeds.services import feeds +from feeds.services.feeds import normalize_title +from feeds.tests.factories import AliasFactory, ArticleFactory, FeedFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture() +def feed_template(): + return """ + + + + {{ title }} + {{ url }} + {{ identifier }} + {{ date_str }} + {{ date_str }} + {{ author }} + {{ summary }} + {% for tag in tags %} + + {% endfor %} + + + """ + + +@pytest.fixture() +def feed_context(): + # The context contains values for the feed template and building Articles. + # Dates are included as datetimes and RFC 822 format strings so the context + # can be used to fill out the feed template and populate Article models. + now = timezone.now().replace(microsecond=0) + return { + "title": "Article title.", + "url": "https://www.example.com/entry/", + "date": now, + "date_str": now.strftime("%a, %d %b %Y %H:%M:%S %z"), + "identifier": "Article:Identifier", + "author": "Article Author", + "summary": "Article Summary", + "tags": ["First", "Second"], + } + + +def test_article_added(monkeypatch, feed_template, feed_context): + # feedparser can parse a file, url or string so we can use the get_url() + # function in the feeds module to inject a string instead of the Blog's + # url and get feedparser to parse it and return the entries. + + def mock_return(feed): + template = Template(feed_template) + context = Context(feed_context) + return template.render(context) + + monkeypatch.setattr(feeds, "get_source", mock_return) + feed = FeedFactory.create(enabled=True) + feeds.load_feed(feed) + articles = Article.objects.all() + + # Results contain added Article + assert len(articles) == 1 + + # Verify the rss attributes are set to the values from the feed + # Ensure microseconds on the date the entry was published are set + # to zero, rather than the value when the entry is saved in case + # the date is used to see if an entry has changed when the feed + # is next loaded. + + article = articles[0] + + assert article.title == normalize_title(feed_context["title"]) + assert article.authors.first().name == feed_context["author"] + assert article.url == feed_context["url"] + assert article.date == feed_context["date"] + assert article.date.microsecond == 0 + assert article.summary == feed_context["summary"] + assert article.identifier == feed_context["identifier"] + assert article.source == feed.source + assert article.publish == feed.auto_publish + + # Verify the tags are added to the JSON data field. Only one + # tag is used to keep the feed template and this test, simple. + assert article.data["feed:tags"] == feed_context["tags"] + + # Verify the categories are added to the article + assert article.categories.count() == feed.categories.count() != 0 + assert article.categories.first() == feed.categories.first() + + +def test_article_updated(monkeypatch, feed_template, feed_context): + feed = FeedFactory.create(enabled=True, auto_publish=True) + params = { + k: v + for k, v in feed_context.items() + if not k.endswith("date_str") and k not in ("author", "tags") + } + existing = ArticleFactory.create(feed=feed, publish=feed.auto_publish, **params) + + feed_context["title"] = "Updated title" + updated_title = feed_context["title"] + + feed.auto_publish = False + feed.save() + + def mock_return(feed): + template = Template(feed_template) + context = Context(feed_context) + return template.render(context) + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(feed) + articles = Article.objects.filter(modified__gt=existing.modified) + + # Results contains updated Article + assert len(articles) == 1 + + # The title of the entry was updated. All other attributes + # remain unchanged. + + article = articles[0] + + assert article.modified > existing.modified + assert article.title == updated_title + assert article.url == feed_context["url"] + assert article.date == feed_context["date"] + assert article.date.microsecond == 0 + assert article.summary == feed_context["summary"] + assert article.identifier == feed_context["identifier"] + assert article.publish is True + + +def test_article_author_alias(monkeypatch, feed_template, feed_context): + alias = AliasFactory.create() + feed = alias.feed + author = alias.author + + feed_context["author"] = alias.name + + def mock_return(feed): + template = Template(feed_template) + context = Context(feed_context) + return template.render(context) + + monkeypatch.setattr(feeds, "get_source", mock_return) + + feeds.load_feed(feed) + + article = Article.objects.all().first() + + assert article.authors.first() == author + + +titles = [ + (" Hello World", "Hello World"), # leading space + (" Hello World", "Hello World"), # multiple leading spaces + ("Hello World ", "Hello World"), # trailing space + ("Hello World ", "Hello World"), # multiple trailing spaces + ("Hello World.", "Hello World"), # trailing period + ("Hello World...", "Hello World..."), # trailing ellipsis + ("Hello- World", "Hello - World"), # no space before hyphen + ("Hello -World", "Hello - World"), # no space after hyphen + ("Hello , World", "Hello, World"), # space before comma + ("Hello,World", "Hello, World"), # no space after comma + ("Hello(World)", "Hello (World)"), # no space before bracket + ("(Hello)World", "(Hello) World"), # no space after bracket +] + + +@pytest.mark.parametrize("before, expected", titles) +def test_normalise_title(before, expected): + after = feeds.normalize_title(before) + assert after == expected diff --git a/src/feeds/tests/services/test_load_feed_errors.py b/src/feeds/tests/services/test_load_feed_errors.py new file mode 100644 index 0000000..a2a8225 --- /dev/null +++ b/src/feeds/tests/services/test_load_feed_errors.py @@ -0,0 +1,231 @@ +from django.utils import timezone + +import pytest + +from feeds.models import Article, Author +from feeds.services import feeds +from feeds.tests.factories import FeedFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture() +def feed_template(): + return """ + + + + %(title)s + %(url)s + %(identifier)s + %(date)s + %(date)s + %(author)s + %(summary)s + + + + """ + + +@pytest.fixture() +def feed_context(): + return { + "title": "ARTICLE TITLE.", + "url": "https://www.example.com/entry/", + "date": timezone.now().strftime("%a, %d %b %Y %H:%M:%S %z"), + "identifier": "Article:Identifier", + "author": "Article Author", + "summary": "Article Summary", + "tags": "Tag", + } + + +def test_blank_identifier(monkeypatch, feed_template, feed_context): + """Feed entries with an empty identifier field are skipped""" + + def mock_return(feed): + feed_context["identifier"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_missing_identifier(monkeypatch, feed_template, feed_context): + """Feed entries with no identifier are loaded using the url as an identifier""" + feed_template = feed_template.replace("%(identifier)s", "") + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is True + + +def test_blank_title(monkeypatch, feed_template, feed_context): + """Feed entries with an empty title field are skipped""" + + def mock_return(feed): + feed_context["title"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_missing_title(monkeypatch, feed_template, feed_context): + """Feed entries with no title field are skipped""" + feed_template = feed_template.replace("%(title)s", "") + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_blank_link(monkeypatch, feed_template, feed_context): + """Feed entries with an empty link field are skipped""" + + def mock_return(feed): + feed_context["url"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_missing_link(monkeypatch, feed_template, feed_context): + """Feed entries with no link field are skipped""" + feed_template = feed_template.replace("%(url)s", "") + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_blank_date(monkeypatch, feed_template, feed_context): + """Feed entries with an empty published field are skipped""" + + def mock_return(feed): + feed_context["date"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_missing_date(monkeypatch, feed_template, feed_context): + """Feed entries with no published field are skipped""" + feed_template = feed_template.replace("%(date)s", "") + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.exists() is False + + +def test_blank_author(monkeypatch, feed_template, feed_context): + """Feed entries with an empty author field are loaded""" + + def mock_return(feed): + feed_context["author"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().authors.count() == 0 + + +def test_missing_author(monkeypatch, feed_template, feed_context): + """Feed entries with no author field are loaded""" + feed_template = feed_template.replace( + "%(author)s", "" + ) + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().authors.count() == 0 + + +def test_multiple_authors(monkeypatch, feed_template, feed_context): + """Feeds where multiple Authors exist with the same name are loaded""" + Author.objects.create(name=feed_context["author"]) + Author.objects.create(name=feed_context["author"]) + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().authors.count() == 1 + + +def test_blank_summary(monkeypatch, feed_template, feed_context): + """Feed entries with an empty summary field are loaded""" + + def mock_return(feed): + feed_context["summary"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().summary == "" + + +def test_missing_summary(monkeypatch, feed_template, feed_context): + """Feed entries with no summary field are loaded""" + feed_template = feed_template.replace( + '%(summary)s', "" + ) + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().summary == "" + + +def test_blank_tags(monkeypatch, feed_template, feed_context): + """Feed entries with an empty tag field are loaded""" + + def mock_return(feed): + feed_context["tags"] = "" + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().data["feed:tags"] == [] + + +def test_missing_tags(monkeypatch, feed_template, feed_context): + """Feed entries with no tag field are loaded""" + feed_template = feed_template.replace( + '', "" + ) + + def mock_return(feed): + return feed_template % feed_context + + monkeypatch.setattr(feeds, "get_source", mock_return) + feeds.load_feed(FeedFactory.create(enabled=True)) + assert Article.objects.get().data["feed:tags"] == []