Support Django 4.0 and python 3.10; Support import of "new" Twitter archive downloads

.github/workflows/tests.yml

@@ -17,12 +17,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9"]
-        django-version: ["2.2", "3.1", "3.2"]
-        # If we wanted to exclude combinations, we could add them here:
-        # exclude:
-          # - python-version: "3.5"
-          #   django-version: "3.0"
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        django-version: ["3.2", "4.0"]
+        exclude:
+          # Django 4.0 isn't compatible with python 3.6 and 3.7:
+          - python-version: "3.6"
+            django-version: "4.0"
+          - python-version: "3.7"
+            django-version: "4.0"
 
     steps:
       - name: Git clone

CHANGELOG.md

@@ -4,7 +4,22 @@ Changelog (Django Ditto)
 Unreleased
 ----------
 
-- None.
+- None
+
+
+2.0.0 - 2022-02-14
+------------------
+
+- **Backwards incompatible:** Drop support for Django 2.2 and 3.1.
+
+- **Backwards incompatible:** Requires django-taggit >= v2.0.0
+  (It changed how `TaggableManager` sets tags: https://github.com/jazzband/django-taggit/blob/master/CHANGELOG.rst#200-2021-11-14 )
+
+- Add support for Django 4.0 (#223)
+
+- Add support for Python 3.10 (#225)
+
+- Add support for importing the "new" (2019-onwwards) format of downloadable Twitter archive (#229).
 
 
 1.4.2 - 2021-10-22

README.md

@@ -5,7 +5,7 @@ Django Ditto
 [![image](https://coveralls.io/repos/github/philgyford/django-ditto/badge.svg?branch=main)](https://coveralls.io/github/philgyford/django-ditto?branch=main "Test coverage")
 [![image](https://readthedocs.org/projects/django-ditto/badge/?version=stable)](https://django-ditto.readthedocs.io/en/stable/?badge=stable "Documentation status")
 
-A collection of Django apps for copying things from third-party sites and services. Requires Python 3.6 to 3.9, and Django 2.2 to 3.2.
+A collection of Django apps for copying things from third-party sites and services. Requires Python 3.6, 3.7, 3.8, 3.9 and 3.10, and Django 3.2 and 4.0.
 
 [Read the documentation.](http://django-ditto.readthedocs.io/en/latest/)
 

devproject/Pipfile

@@ -9,4 +9,4 @@ name = "pypi"
 [dev-packages]
 
 [requires]
-python_version = "3.8"
+python_version = "3.10"

devproject/devproject/settings.py

@@ -113,6 +113,10 @@ def get_env_variable(var_name, default=None):
             "handlers": ["console"],
             "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
         },
+        "ditto": {
+            "handlers": ["console"],
+            "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
+        },
         "ditto.management_commands": {
             "handlers": ["console"],
             "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),

ditto/__init__.py

@@ -1,5 +1,5 @@
 __title__ = "Django Ditto"
-__version__ = "1.4.2"
+__version__ = "2.0.0"
 __author__ = "Phil Gyford"
 __author_email__ = "[email protected]"
 __license__ = "MIT"

ditto/core/views.py

@@ -7,8 +7,8 @@
 from django.urls import reverse
 from django.http import Http404
 from django.shortcuts import redirect
-from django.utils.encoding import force_str, force_text
-from django.utils.translation import ugettext as _
+from django.utils.encoding import force_str
+from django.utils.translation import gettext as _
 from django.views.generic import ListView, TemplateView
 from django.views.generic import DayArchiveView as DjangoDayArchiveView
 
@@ -574,7 +574,7 @@ def get_variety_counts(self):
                     raise Http404(
                         _("No %(verbose_name_plural)s available")
                         % {
-                            "verbose_name_plural": force_text(
+                            "verbose_name_plural": force_str(
                                 qs.model._meta.verbose_name_plural
                             )
                         }

ditto/flickr/views.py

@@ -1,5 +1,5 @@
 from django.http import Http404
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from django.views.generic import DetailView, ListView
 from django.views.generic.detail import SingleObjectMixin
 

ditto/lastfm/views.py

@@ -1,7 +1,7 @@
 from datetime import timedelta
 
 from django.http import Http404
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from django.views.generic import DetailView, TemplateView
 from django.views.generic.detail import SingleObjectMixin
 

ditto/pinboard/fetch.py

@@ -225,7 +225,7 @@ def _save_bookmark(self, bookmark, fetch_time, account):
             },
         )
 
-        bookmark_obj.tags.set(*bookmark["tags"])
+        bookmark_obj.tags.set(bookmark["tags"])
 
 
 class AllBookmarksFetcher(BookmarksFetcher):

ditto/pinboard/views.py

@@ -1,5 +1,5 @@
 from django.http import Http404
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from django.views.generic import DetailView, ListView
 from django.views.generic.detail import SingleObjectMixin
 

ditto/twitter/admin.py

@@ -17,7 +17,15 @@ class AccountAdmin(admin.ModelAdmin):
     )
 
     fieldsets = (
-        (None, {"fields": ("user", "is_active",)}),
+        (
+            None,
+            {
+                "fields": (
+                    "user",
+                    "is_active",
+                )
+            },
+        ),
         (
             "API",
             {
@@ -29,7 +37,8 @@ class AccountAdmin(admin.ModelAdmin):
                 ),
                 "description": (
                     "Keys and secrets require creation of an app at "
-                    '<a href="https://apps.twitter.com/">apps.twitter.com</a>'
+                    '<a href="https://developer.twitter.com/portal">'
+                    "developer.twitter.com/portal</a>"
                 ),
             },
         ),
@@ -123,7 +132,13 @@ class MediaAdmin(admin.ModelAdmin):
         ),
         (
             "Data",
-            {"classes": ("collapse",), "fields": ("time_created", "time_modified",)},
+            {
+                "classes": ("collapse",),
+                "fields": (
+                    "time_created",
+                    "time_modified",
+                ),
+            },
         ),
     )
 
@@ -240,7 +255,12 @@ class TweetAdmin(DittoItemModelAdmin):
             "Data",
             {
                 "classes": ("collapse",),
-                "fields": ("raw", "fetch_time", "time_created", "time_modified",),
+                "fields": (
+                    "raw",
+                    "fetch_time",
+                    "time_created",
+                    "time_modified",
+                ),
             },
         ),
     )
@@ -309,7 +329,17 @@ class UserAdmin(admin.ModelAdmin):
                 )
             },
         ),
-        ("Data", {"fields": ("raw", "fetch_time", "time_created", "time_modified",)}),
+        (
+            "Data",
+            {
+                "fields": (
+                    "raw",
+                    "fetch_time",
+                    "time_created",
+                    "time_modified",
+                )
+            },
+        ),
     )
 
     formfield_overrides = {

ditto/twitter/fetch/savers.py

@@ -59,14 +59,16 @@ def save_user(self, user, fetch_time, download_avatar=True):
             "raw": raw_json,
             "screen_name": user["screen_name"],
             "name": user["name"],
-            "is_private": user["protected"],
             "is_verified": user["verified"],
             "profile_image_url_https": user["profile_image_url_https"],
         }
 
         # When ingesting tweets there are lots of fields the 'user' element
         # doesn't have, compared to the API:
 
+        if "protected" in user:
+            defaults["is_private"] = user["protected"]
+
         if "created_at" in user:
             defaults["created_at"] = self._api_time_to_datetime(user["created_at"])
 
@@ -265,7 +267,7 @@ def save_media(self, tweet):
 
         return media_count
 
-    def save_tweet(self, tweet, fetch_time):
+    def save_tweet(self, tweet, fetch_time, user_data=None):
         """Takes a dict of tweet data from the API and creates or updates a
         Tweet object and its associated User object.
 
@@ -286,15 +288,21 @@ def save_tweet(self, tweet, fetch_time):
                 tweet["created_at"], time_format="%Y-%m-%d %H:%M:%S +0000"
             )
 
-        user = UserSaver().save_user(tweet["user"], fetch_time)
+        if user_data is None:
+            if "user" in tweet:
+                user_data = tweet["user"]
+            else:
+                raise ValueError("No user data found to save tweets with")
+
+        user = UserSaver().save_user(user_data, fetch_time)
 
         if "full_text" in tweet:
             # For new (2016) 'extended' format tweet data.
             # https://dev.twitter.com/overview/api/upcoming-changes-to-tweets
             text = tweet["full_text"]
             # Cuts off any @usernames at the start and a trailing URL at the end:
-            frm = tweet["display_text_range"][0]
-            to = tweet["display_text_range"][1]
+            frm = int(tweet["display_text_range"][0])
+            to = int(tweet["display_text_range"][1])
             title = text[frm:to]
         else:
             # Older 'classic' format tweet data.