Add RSS related files from the django-lynx (private) repository.

The code was taken from a working project which has been in production
for two and a half years. The code was refactored for this project, the
views, templates, etc. were moved to the demo app, requirements updated,
etc. etc.

THe demo site works and works with celery to automatically load feeds
according to a preset schedule. If you just want to see it in action
create an admin account, add a Source, then a Feed and you can load it
via an admin action.

Next steps are make everything run using either a virtualenv or Docker
and get it all polished for a first release.

.env

@@ -0,0 +1,39 @@
+#
+# Environment variables used for development using a virtualenv
+#
+
+# This assumes you've installed postgresql and rabbitmq on your
+# machine and have added the accounts need to connect to them:
+#
+# sudo -u postgres psql
+# postgres=# create database feeds;
+# postgres=# create user feeds with encrypted password 'feeds';
+# postgres=# grant all privileges on database feeds to feeds;
+#
+# sudo rabbitmqctl add_user feeds feeds
+# sudo rabbitmqctl set_permissions -p / feeds ".*" ".*" ".*"
+# sudo rabbitmqctl add_vhost feeds
+
+# Set environment variables used by the postgresql commands to connect
+# to the database. This is the admin account not the account used by
+# Django to connect to the database. PGPASSWORD is particularly useful
+# to avoid having to enter it for each command. You will need to configure
+# the server to allow username/password (md5) authentication on local
+# (socket) connections and so avoid having to su to the postgres user first.
+#
+# /etc/postgresql/<version>/main/pg_hba.conf
+# local    all    all    md5
+#
+# For more info on environment variables see,
+# https://www.postgresql.org/docs/current/libpq-envars.html
+
+PGHOST=localhost
+PGPORT=5432
+PGUSER=postgres
+PGPASSWORD=postgres
+
+# Environment variables used by Django
+
+CELERY_BROKER_URL=amqp://feeds:feeds@localhost:5672/feeds
+
+DB_HOST=localhost

.env.docker

@@ -0,0 +1,31 @@
+#
+# Environment variables used for running the demo site using docker
+#
+
+# Service: db
+# Rather than set POSTGRES_PASSWORD (required) and leave POSTGRES_USER and
+# so POSTGRES_DB to default to "postgres" we set the variables to the name
+# of the app so it mirrors the configuration for locally installed services
+# which might be shared between sites. However, the value of doing this is
+# that you easily initialize the database with dumps from production where
+# the ownership of the tables is defined.
+
+PGHOST=db
+POSTGRES_DB=feeds
+POSTGRES_USER=feeds
+POSTGRES_PASSWORD=feeds
+
+# Service: broker
+# This, like the db service, simply mirrors the configuration for rabbitmq
+# being installed locally and possibly shared between sites.
+
+RABBITMQ_DEFAULT_USER=feeds
+RABBITMQ_DEFAULT_PASS=feeds
+RABBITMQ_DEFAULT_VHOST=feeds
+
+# Services: web, celery, celery-beat
+# Environment variables used in Django settings
+
+CELERY_BROKER_URL=amqp://feeds:feeds@broker:5672/feeds
+
+DB_HOST=db

.envrc

@@ -0,0 +1,12 @@
+# direnv file
+# When you cd to this directory, direnv will activate the virtualenv
+# and set all the environment variables in the .env file.
+# See https://direnv.net/man/direnv-stdlib.1.html
+
+# Activate the virtualenv
+# This simply replicates what venv/bin/activate does.
+export VIRTUAL_ENV=venv
+PATH_add "$VIRTUAL_ENV/bin"
+
+# Set environment variables from .env
+dotenv

.gitignore

@@ -9,6 +9,9 @@ __pycache__/
 # Ignore dot files except those required for the project
 .*
 !.editorconfig
+!.envrc
+!.env
+!.env.docker
 !.flake8
 !.gitattributes
 !.github

Dockerfile

@@ -0,0 +1,36 @@
+#
+# Dockerfile: configuration for building the images for the web app,
+# celery-beat and celery-worker.
+#
+
+# Base python image
+FROM python:3.8.16-buster
+# Add the site to the python path
+ENV PYTHONPATH /code
+# Send all output on stdout and stderr straight to the container logs
+ENV PYTHONUNBUFFERED 1
+# Set the locale
+ENV LC_ALL C.UTF-8
+# Terminals support 256 colours
+ENV TERM xterm-256color
+
+# Mount the site directory
+
+RUN mkdir /code
+WORKDIR /code
+
+# Install dependencies
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends
+
+RUN apt-get install -y --no-install-recommends \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install requirements
+
+COPY requirements/dev.txt /tmp/requirements.txt
+RUN pip install --upgrade setuptools pip wheel\
+    && pip install pip-tools==6.10.0\
+    && pip install -r /tmp/requirements.txt

MANIFEST.in

@@ -11,11 +11,9 @@ include CHANGELOG.rst
 exclude setup.cfg
 exclude pyproject.toml
 
-recursive-include src/app_project/locale *
-recursive-include src/app_project/templates *
-recursive-include src/app_project/static *
+recursive-include src/feeds/ *
 
-recursive-exclude src/app_project/tests *
+recursive-exclude src/feeds/tests *
 
 recursive-exclude * .cache
 recursive-exclude * __pycache__

Makefile

@@ -16,8 +16,8 @@ PYTHON = python3.10
 site_python := /usr/bin/env $(PYTHON)
 
 root_dir := $(realpath .)
-app_dir = $(root_dir)/src/app_project
-demo_dir = $(root_dir)/project
+app_dir = $(root_dir)/src/feeds
+demo_dir = $(root_dir)/demo
 
 venv_name = venv
 venv_dir = $(root_dir)/$(venv_name)
@@ -133,7 +133,7 @@ clean: clean-venv clean-build clean-tests clean-mypy clean-coverage clean-docs
 $(venv_dir):
 	$(site_python) -m venv $(venv_dir)
 	$(pip) install --upgrade pip setuptools wheel
-	$(pip) install pip-tools==6.10.0
+	$(pip) install pip-tools
 
 requirements/dev.txt: requirements/dev.in requirements/docs.in requirements/tests.in
 	$(pip-compile) requirements/dev.in
@@ -189,19 +189,19 @@ translations:
 
 .PHONY: flake8
 flake8:
-	$(flake8) $(site_dir)
+	$(flake8) $(app_dir)
 
 .PHONY: isort
 isort:
-	$(isort) --check $(site_dir)
+	$(isort) --check $(app_dir)
 
 .PHONY: black
 black:
-	$(black) --check $(site_dir)
+	$(black) --check $(app_dir)
 
 .PHONY: mypy
 mypy:
-	$(mypy) $(site_dir)
+	$(mypy) $(app_dir)
 
 .PHONY: checks
 checks: flake8 black isort mypy
@@ -212,7 +212,7 @@ checks: flake8 black isort mypy
 
 .PHONY: coverage
 coverage:
-	$(pytest) --cov=app_project --cov-config=setup.cfg --cov-report html
+	$(pytest) --cov=feeds --cov-config=setup.cfg --cov-report html
 
 .PHONY: test
 test:

README.md

@@ -1,64 +1,26 @@
-# Django App Project
+# Django Feeds
 
-[![Build Status](https://img.shields.io/github/actions/workflow/status/StuartMacKay/django-app-template/ci.yml?branch=master)](https://github.com/StuartMacKay/django-app-template/actions/workflows/ci.yml?query=branch%3Amaster)
+[![Build Status](https://img.shields.io/github/actions/workflow/status/StuartMacKay/django-feeds/ci.yml?branch=master)](https://github.com/StuartMacKay/django-feeds/actions/workflows/ci.yml?query=branch%3Amaster)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black)
 
-## Features
+Django Feeds is an aggregator for RSS and Atom feeds.
 
-* Development with [black](https://github.com/psf/black) so everybody gets the code formatting rules they deserve
-* Development with [flake8](https://flake8.pycqa.org/en/latest/) so people using ed get syntax checking
-* Development with [isort](https://pycqa.github.io/isort/) for automatically sorting imports
-* Development with [mypy](https://mypy-lang.org/) for type-hinting to catch errors
-* Testing with [pytest](https://docs.pytest.org/), [FactoryBoy](https://factoryboy.readthedocs.io/en/stable/) and [tox](https://tox.wiki/en/latest/)
-* Manage versions with [bump2version](https://pypi.org/project/bump2version/) - for semantic version numbers
-* Manage dependency versions with [pip-tools](https://github.com/jazzband/pip-tools)
-* Manage dependency upgrades with [pip-upgrade](https://github.com/simion/pip-upgrader)
+## Features
 
-## Quick start
+## Quick Start
 
-First, download and unzip the project files in the directory of your choice.
-Then rename the project to something more useful:
-```shell
-mv django-app-template django-myapp
-```
+## Demo
 
-Change to the project directory and start setting things up:
-```shell
-cd django-myapp
-```
+If you clone or download the [django-feeds](https://github.com/StuartMacKay/django-feeds) 
+repository there is a demonstration application that lets you see how it 
+all works.
 
-First, build the virtualenv and install all the dependencies. This will 
-also build the library:
 ```shell
+git clone [email protected]:StuartMacKay/django-feeds.git
+cd django-feeds
 make install
-```
-
-Now run the demo site:
-```shell
 make demo
-```
-
-
-Run the database migrations:
-```shell
-./manage.py migrate
-```
-
-Run the tests:
-```shell
-make tests
-```
-
-Run the django server:
-```shell
-./manage.py runserver
-```
-
-Open a browser and visit http://localhost:8000 and, voila, we have a working
-site. Well cover the deployment later.
-
-Almost all steps used the project Makefile. That's great if you're running 
-Linux with GNU Make and not much fun if you're not. All is not lost, however. 
-All the Makefile's targets contain only one or two commands, so even if you 
-are running Windows you should still be able to get the site running without 
-too much effort.
+``` 
+It's a standard django project so if you don't have `make` available
+then just look at the [Makefile](Makefile) and run the commands from
+the various targets.

TODO.md

@@ -1,2 +1,21 @@
 # TODO
 
+The code runs and runs well. It has been in production for almost two
+years, so it is rather polished. It was factored out of the 
+[Voices for Independence](https://www.voices.scot) web site with s small
+number of changes and is now being repackaged to make it reusable in other
+projects we have in mind. 
+
+* replace the original bootstrap css classes with semantic names, so it 
+  can be restyled easily.
+
+* move 'business logic' out of the views and out of the querysets into a  
+  services layer - an internal API. That way you're not forced
+  to subclass the views, and you can easily write your own.
+
+* Add more blocks to the templates and create more snippets so again you
+  can easily replace what is provided in the app.
+
+* Also load the feeds using cron, so you are not forced to use celery.
+
+* Add a REST API - at some point.

demo/__init__.py

@@ -0,0 +1 @@
+from .celery import app as celery_app

demo/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class Config(AppConfig):
+    name = "demo"

demo/conf/asgi.py → demo/asgi.py

@@ -11,6 +11,6 @@
 
 from django.core.asgi import get_asgi_application
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.conf.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings")
 
 application = get_asgi_application()

demo/celery.py

@@ -0,0 +1,73 @@
+import logging
+import os
+
+import structlog
+from celery import Celery  # type: ignore
+from celery.schedules import crontab
+from celery.signals import setup_logging  # type: ignore
+from django_structlog.celery.steps import DjangoStructLogInitStep  # type: ignore
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings")
+
+app = Celery()
+
+# Load the configuration
+app.config_from_object("demo.settings")
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+# Configure workers to use structlog
+app.steps["worker"].add(DjangoStructLogInitStep)
+
+# A celery task runs every hour, on the error and checked the schedule for
+# each active feed. If the schedule, also a crontab, matches the current
+# time then the feed is loaded. This two-level approach comes in useful for
+# feeds that are either updated in frequently, so there's no need to cook
+# the planet by checking every hour, or, dealing with throttling by services
+# such as Cloudflare, where too many request s
+
+FEEDS_TASK_SCHEDULE = os.environ.get("FEEDS_TASK_SCHEDULE", "0 * * * *")
+minute, hour, day_of_week, day_of_month, month_of_year = FEEDS_TASK_SCHEDULE.split()
+
+app.conf.beat_schedule = {
+    "load-feeds": {
+        "task": "demo.tasks.load_feeds",
+        "schedule": crontab(
+            minute=minute,
+            hour=hour,
+            day_of_week=day_of_week,
+            day_of_month=day_of_month,
+            month_of_year=month_of_year,
+        ),
+    },
+    "daily-digest": {
+        "task": "feed.tasks.daily_digest",
+        "schedule": crontab(minute="0", hour="2"),  # 2am, each day
+    },
+}
+
+
+@setup_logging.connect
+def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs):  # noqa
+    from django.conf import settings
+
+    config = settings.LOGGING
+    logging.config.dictConfig(config)  # type: ignore
+
+    # noinspection DuplicatedCode
+    structlog.configure(
+        processors=[
+            structlog.stdlib.filter_by_level,
+            structlog.processors.TimeStamper(fmt="iso"),
+            structlog.stdlib.add_logger_name,
+            structlog.stdlib.add_log_level,
+            structlog.stdlib.PositionalArgumentsFormatter(),
+            structlog.processors.StackInfoRenderer(),
+            structlog.processors.format_exc_info,
+            structlog.processors.UnicodeDecoder(),
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        context_class=structlog.threadlocal.wrap_dict(dict),
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,  # type: ignore
+        cache_logger_on_first_use=True,
+    )