From f290204c88413df9d6cc3cffc5f3e60803bde097 Mon Sep 17 00:00:00 2001 From: Adrien Matissart Date: Thu, 6 Feb 2025 16:25:05 +0100 Subject: [PATCH] Integrate Bluesky as alternative to Twitterbot (#2051) --------- Co-authored-by: Gresille & Siffle <39056254+GresilleSiffle@users.noreply.github.com> --- backend/requirements.txt | 3 +- backend/twitterbot/admin.py | 9 +- backend/twitterbot/client.py | 113 +++++++++++++++ .../management/commands/load_tweetinfo.py | 2 +- .../management/commands/run_twitterbot.py | 2 +- .../commands/tweet_top_contributors.py | 7 +- ...fo_atproto_uri_alter_tweetinfo_tweet_id.py | 30 ++++ .../0004_alter_tweetinfo_bot_name_and_more.py | 46 ++++++ backend/twitterbot/models/history.py | 81 +++++++++++ backend/twitterbot/models/tweeted.py | 47 ------- backend/twitterbot/settings.py | 17 ++- .../tests/test_get_twitter_account.py | 28 ++-- backend/twitterbot/tests/test_tournesolbot.py | 14 +- backend/twitterbot/tournesolbot.py | 131 +++++++++++------- backend/twitterbot/twitter_api.py | 38 ----- .../twitterbot/uploader_twitter_account.py | 43 +++--- backend/twitterbot/views/__init__.py | 0 .../roles/django/templates/settings.yaml.j2 | 4 + .../ansible/scripts/deploy-without-secrets.sh | 2 + infra/ansible/scripts/forget-secrets.sh | 2 + infra/ansible/scripts/get-vm-secrets.sh | 6 + 21 files changed, 431 insertions(+), 194 deletions(-) create mode 100644 backend/twitterbot/client.py create mode 100644 backend/twitterbot/migrations/0003_tweetinfo_atproto_uri_alter_tweetinfo_tweet_id.py create mode 100644 backend/twitterbot/migrations/0004_alter_tweetinfo_bot_name_and_more.py create mode 100644 backend/twitterbot/models/history.py delete mode 100644 backend/twitterbot/models/tweeted.py delete mode 100644 backend/twitterbot/twitter_api.py delete mode 100644 backend/twitterbot/views/__init__.py diff --git a/backend/requirements.txt b/backend/requirements.txt index 23efe7246f..5dbd624433 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -60,4 +60,5 @@ requests==2.32.2 tweepy==4.14.0 # dotenv for .env file python-dotenv==1.0.0 - +# AT protocol library for Bluesky +atproto==0.0.58 diff --git a/backend/twitterbot/admin.py b/backend/twitterbot/admin.py index bb02687c28..eccc0a8d49 100644 --- a/backend/twitterbot/admin.py +++ b/backend/twitterbot/admin.py @@ -8,7 +8,7 @@ from django.contrib import admin from django.utils.html import format_html -from .models.tweeted import TweetInfo +from .models.history import TweetInfo @admin.register(TweetInfo) @@ -32,13 +32,12 @@ class TwitterBotAdmin(admin.ModelAdmin): list_filter = ["bot_name"] @staticmethod - @admin.display(description="URL of the tweet") + @admin.display(description="Post URL") def get_twitter_link(obj): """Return the URI of the tweet.""" return format_html( - 'Tweet', - obj.bot_name, - obj.tweet_id, + 'Link', + obj.message_url, ) @staticmethod diff --git a/backend/twitterbot/client.py b/backend/twitterbot/client.py new file mode 100644 index 0000000000..3867ce469a --- /dev/null +++ b/backend/twitterbot/client.py @@ -0,0 +1,113 @@ +from functools import cached_property +from pathlib import Path +from typing import List, Optional + +import requests +from django.conf import settings + +from tournesol.models import Entity + + +class TournesolBotClient: + def __init__(self, account): + + credentials = settings.TWITTERBOT_CREDENTIALS + if account not in credentials: + raise ValueError(f"No credentials found for {account} account!") + + self.account_cred = credentials[account] + self.language = self.account_cred["LANGUAGE"] + + @cached_property + def tweepy_api(self): + # Client for Twitter API v1.1 + # We need this authentication also because to add media it only works with api v1.1 + + import tweepy # pylint:disable=import-outside-toplevel + auth = tweepy.OAuth1UserHandler( + consumer_key=self.account_cred["CONSUMER_KEY"], + consumer_secret=self.account_cred["CONSUMER_SECRET"], + access_token=self.account_cred["ACCESS_TOKEN"], + access_token_secret=self.account_cred["ACCESS_TOKEN_SECRET"], + ) + return tweepy.API(auth) + + @cached_property + def tweepy_client(self): + # Client for Twitter API v2 + + import tweepy # pylint:disable=import-outside-toplevel + return tweepy.Client( + consumer_key=self.account_cred["CONSUMER_KEY"], + consumer_secret=self.account_cred["CONSUMER_SECRET"], + access_token=self.account_cred["ACCESS_TOKEN"], + access_token_secret=self.account_cred["ACCESS_TOKEN_SECRET"], + ) + + @property + def bluesky_handle(self): + return self.account_cred["ATPROTO_HANDLE"] + + @cached_property + def atproto_client(self): + from atproto import Client # pylint:disable=import-outside-toplevel + client = Client() + client.login( + self.account_cred["ATPROTO_HANDLE"], + self.account_cred["ATPROTO_PASSWORD"], + ) + return client + + def create_tweet(self, text: str, media_files: Optional[List[Path]] = None): + if media_files is None: + media_ids = [] + else: + medias = (self.tweepy_api.media_upload(filepath) for filepath in media_files) + media_ids = [media.media_id for media in medias] + + resp = self.tweepy_client.create_tweet( + text=text, + media_ids=media_ids, + ) + return resp.data["id"] + + def create_bluesky_post( + self, + text, + embed_video: Optional[Entity] = None, + image_files: Optional[List[Path]] = None, + image_alts: Optional[List[str]] = None, + ): + from atproto import models # pylint:disable=import-outside-toplevel + if image_files is None: + if embed_video is None: + embed = None + else: + preview_response = requests.get( + f"https://api.tournesol.app/preview/entities/{embed_video.uid}", + timeout=10, + ) + preview_response.raise_for_status() + img_data = preview_response.content + thumb_blob = self.atproto_client.upload_blob(img_data).blob + embed = models.AppBskyEmbedExternal.Main( + external=models.AppBskyEmbedExternal.External( + title=embed_video.metadata.get("name", ""), + description=embed_video.metadata.get("uploader", ""), + uri=f"https://tournesol.app/entities/{embed_video.uid}", + thumb=thumb_blob, + ) + ) + resp = self.atproto_client.send_post( + text=text, + embed=embed, + langs=[self.language], + ) + else: + resp = self.atproto_client.send_images( + text=text, + langs=[self.language], + images=[p.read_bytes() for p in image_files], + image_alts=image_alts, + ) + return resp.uri diff --git a/backend/twitterbot/management/commands/load_tweetinfo.py b/backend/twitterbot/management/commands/load_tweetinfo.py index accf0bb5c8..12aa9eb897 100644 --- a/backend/twitterbot/management/commands/load_tweetinfo.py +++ b/backend/twitterbot/management/commands/load_tweetinfo.py @@ -4,7 +4,7 @@ from tournesol.entities.video import YOUTUBE_UID_NAMESPACE from tournesol.models.entity import Entity -from twitterbot.models.tweeted import TweetInfo +from twitterbot.models.history import TweetInfo class Command(BaseCommand): diff --git a/backend/twitterbot/management/commands/run_twitterbot.py b/backend/twitterbot/management/commands/run_twitterbot.py index e8c956b756..07e4681f13 100644 --- a/backend/twitterbot/management/commands/run_twitterbot.py +++ b/backend/twitterbot/management/commands/run_twitterbot.py @@ -52,4 +52,4 @@ def handle(self, *args, **options): return - tweet_video_recommendation(bot_name, assumeyes=options["assumeyes"]) + tweet_video_recommendation(bot_name, assumeyes=options["assumeyes"], dest=["bluesky"]) diff --git a/backend/twitterbot/management/commands/tweet_top_contributors.py b/backend/twitterbot/management/commands/tweet_top_contributors.py index 5292d877e4..2fc97f3f79 100644 --- a/backend/twitterbot/management/commands/tweet_top_contributors.py +++ b/backend/twitterbot/management/commands/tweet_top_contributors.py @@ -26,5 +26,8 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - - tweet_top_contributor_graph(options["bot_name"], assumeyes=options["assumeyes"]) + tweet_top_contributor_graph( + options["bot_name"], + assumeyes=options["assumeyes"], + dest=["bluesky"], + ) diff --git a/backend/twitterbot/migrations/0003_tweetinfo_atproto_uri_alter_tweetinfo_tweet_id.py b/backend/twitterbot/migrations/0003_tweetinfo_atproto_uri_alter_tweetinfo_tweet_id.py new file mode 100644 index 0000000000..a7abce21e4 --- /dev/null +++ b/backend/twitterbot/migrations/0003_tweetinfo_atproto_uri_alter_tweetinfo_tweet_id.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.11 on 2025-01-27 22:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("twitterbot", "0002_rename_tweetedvideo_tweetinfo"), + ] + + operations = [ + migrations.AddField( + model_name="tweetinfo", + name="atproto_uri", + field=models.CharField( + default=None, + help_text="URI of the post on the AT protocol network", + max_length=255, + null=True, + ), + ), + migrations.AlterField( + model_name="tweetinfo", + name="tweet_id", + field=models.CharField( + default=None, help_text="Tweet ID from Twitter URL", max_length=22, null=True + ), + ), + ] diff --git a/backend/twitterbot/migrations/0004_alter_tweetinfo_bot_name_and_more.py b/backend/twitterbot/migrations/0004_alter_tweetinfo_bot_name_and_more.py new file mode 100644 index 0000000000..851a10ace4 --- /dev/null +++ b/backend/twitterbot/migrations/0004_alter_tweetinfo_bot_name_and_more.py @@ -0,0 +1,46 @@ +# Generated by Django 4.2.18 on 2025-02-06 14:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("tournesol", "0061_comparisoncriteriascore_score_max_and_more"), + ("twitterbot", "0003_tweetinfo_atproto_uri_alter_tweetinfo_tweet_id"), + ] + + operations = [ + migrations.AlterField( + model_name="tweetinfo", + name="bot_name", + field=models.CharField( + blank=True, + choices=[ + ("@TournesolBot", "@TournesolBot"), + ("@TournesolBotFR", "@TournesolBotFR"), + ], + help_text="Name of the bot", + max_length=200, + null=True, + ), + ), + migrations.AlterField( + model_name="tweetinfo", + name="datetime_tweet", + field=models.DateTimeField( + auto_now_add=True, help_text="Time when the video was posted", null=True + ), + ), + migrations.AlterField( + model_name="tweetinfo", + name="video", + field=models.ForeignKey( + help_text="Posted video", + on_delete=django.db.models.deletion.CASCADE, + related_name="tweets", + to="tournesol.entity", + ), + ), + ] diff --git a/backend/twitterbot/models/history.py b/backend/twitterbot/models/history.py new file mode 100644 index 0000000000..92b69bd937 --- /dev/null +++ b/backend/twitterbot/models/history.py @@ -0,0 +1,81 @@ +""" +Models related to the Twitter bot. +""" +import re + +from django.db import models + +from tournesol.models import Entity + +BOT_NAME = [ + ("@TournesolBot", "@TournesolBot"), + ("@TournesolBotFR", "@TournesolBotFR"), +] + + +class TweetInfo(models.Model): + """One tweeted video.""" + + video = models.ForeignKey( + Entity, + on_delete=models.CASCADE, + related_name="tweets", + help_text="Posted video", + ) + + tweet_id = models.CharField( + null=True, + default=None, + max_length=22, + help_text="Tweet ID from Twitter URL", + ) + + atproto_uri = models.CharField( + null=True, + default=None, + max_length=255, + help_text="URI of the post on the AT protocol network", + ) + + datetime_tweet = models.DateTimeField( + auto_now_add=True, + help_text="Time when the video was posted", + null=True, + blank=True, + ) + + bot_name = models.CharField( + null=True, + blank=True, + max_length=200, + help_text="Name of the bot", + choices=BOT_NAME, + ) + + def __str__(self): + return f"{self.video.uid} posted at {self.datetime_tweet}" + + @property + def tweet_url(self): + if not self.tweet_id: + return None + return f"https://twitter.com/{self.bot_name}/status/{self.tweet_id}" + + @property + def bluesky_url(self): + if not self.atproto_uri: + return None + match = re.match( + r"at://(?P.+)/(?P.+)/(?P.+)", + self.atproto_uri, + ) + if not match or match.group("collection") != "app.bsky.feed.post": + return None + return f"https://bsky.app/profile/{match.group('authority')}/post/{match.group('key')}" + + @property + def message_url(self): + bluesky_url = self.bluesky_url + if bluesky_url: + return self.bluesky_url + return self.tweet_url diff --git a/backend/twitterbot/models/tweeted.py b/backend/twitterbot/models/tweeted.py deleted file mode 100644 index 28ed03eb84..0000000000 --- a/backend/twitterbot/models/tweeted.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Models for Tournesol twitter bot already tweeted videos -""" - -from django.db import models - -from tournesol.models import Entity - -BOT_NAME = [ - ("@TournesolBot", "@TournesolBot"), - ("@TournesolBotFR", "@TournesolBotFR"), -] - - -class TweetInfo(models.Model): - """One tweeted video.""" - - video = models.ForeignKey( - Entity, - on_delete=models.CASCADE, - related_name="tweets", - help_text="Tweeted video", - ) - - tweet_id = models.CharField( - null=False, - max_length=22, - help_text="Tweet ID from Twitter URL", - ) - - datetime_tweet = models.DateTimeField( - auto_now_add=True, - help_text="Time when the video was tweeted", - null=True, - blank=True, - ) - - bot_name = models.CharField( - null=True, - blank=True, - max_length=200, - help_text="Name of the twitter bot", - choices=BOT_NAME, - ) - - def __str__(self): - return f"{self.video.uid} tweeted at {self.datetime_tweet}" diff --git a/backend/twitterbot/settings.py b/backend/twitterbot/settings.py index 962882fba6..d76aaea2c1 100644 --- a/backend/twitterbot/settings.py +++ b/backend/twitterbot/settings.py @@ -16,14 +16,12 @@ "en": ( "Today, I recommend '{title}' by {twitter_account}" ", compared {n_comparison} times on #Tournesol\U0001F33B by {n_contributor}" - " contributors, favorite criteria:\n- {crit1}\n- {crit2}\n" - "tournesol.app/entities/yt:{video_id}" + " contributors, favorite criteria:\n- {crit1}\n- {crit2}" ), "fr": ( "Aujourd'hui, je recommande '{title}' de {twitter_account}" ", comparée {n_comparison} fois sur #Tournesol\U0001F33B par {n_contributor}" - " contributeurs, critères favoris:\n- {crit1}\n- {crit2}\n" - "tournesol.app/entities/yt:{video_id}" + " contributeurs, critères favoris:\n- {crit1}\n- {crit2}" ), } @@ -49,6 +47,17 @@ ), } +top_contrib_tweet_image_alt = { + "en": ( + "A bar plot showing the users who contributed the most comparisons on Tournesol " + "in the last month." + ), + "fr": ( + "Un graphique en barres représentant les utilisateur·rice·s ayant effectué le " + "plus de comparaisons sur Tournesol durant le mois dernier." + ), +} + # Name of the Discord channel where the twitterbot will post its tweets. # An empty value won't trigger any post. TWITTERBOT_DISCORD_CHANNEL = "twitter" diff --git a/backend/twitterbot/tests/test_get_twitter_account.py b/backend/twitterbot/tests/test_get_twitter_account.py index e6561351ea..aa5f59a994 100644 --- a/backend/twitterbot/tests/test_get_twitter_account.py +++ b/backend/twitterbot/tests/test_get_twitter_account.py @@ -1,36 +1,30 @@ from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import patch from twitterbot.uploader_twitter_account import ( - get_twitter_account_from_channel_id, + get_twitter_account_from_video_id, get_twitter_handles_from_html, ) -@patch("twitterbot.uploader_twitter_account.requests.get") -def test_get_twitter_account_from_channel_id(mock_requests): - """Test function for get_twitter_account_from_channel_id""" - +@patch("twitterbot.uploader_twitter_account.get_video_channel_html") +def test_get_twitter_account_from_video_id(mock_get_html): # Correct response from youtube mock_resp = Path("./twitterbot/tests/mock_resp.txt").read_text() - mock_requests.return_value = MagicMock(status_code=200, text=mock_resp) - - assert ( - get_twitter_account_from_channel_id("UC0NCbj8CxzeCGIF6sODJ-7A") - == "@le_science4all" - ) + mock_get_html.return_value = mock_resp + assert get_twitter_account_from_video_id("video_id1") == "@le_science4all" # Bad response ("This channel does not exist.") mock_bad_resp = Path("./twitterbot/tests/mock_bad_resp.txt").read_text() - mock_requests.return_value = MagicMock(status_code=200, text=mock_bad_resp) - - assert get_twitter_account_from_channel_id("UC0NCbj8CxzeCGIF6sOZZZZZ") is None + mock_get_html.return_value = mock_bad_resp + assert get_twitter_account_from_video_id("video_id2") is None def test_twitter_account_from_html_with_extra_params(): html = 'q=https%3A%2F%2Ftwitter.com%2FKurz_Gesagt%3Fref_src%3Dtwsrc%255Egoogle%257Ctwcamp%255Eserp%257Ctwgr%255Eauthor"' - assert get_twitter_handles_from_html(html) == ["Kurz_Gesagt"] + assert get_twitter_handles_from_html(html) == {"@kurz_gesagt"} + def test_twitter_account_from_html_with_http(): html = 'q=http%3A%2F%2Ftwitter.com%2Fveritasium"' - assert get_twitter_handles_from_html(html) == ["veritasium"] + assert get_twitter_handles_from_html(html) == {"@veritasium"} diff --git a/backend/twitterbot/tests/test_tournesolbot.py b/backend/twitterbot/tests/test_tournesolbot.py index eab3110731..19b25c4d8f 100644 --- a/backend/twitterbot/tests/test_tournesolbot.py +++ b/backend/twitterbot/tests/test_tournesolbot.py @@ -16,7 +16,7 @@ generate_top_contributor_figure, get_best_criteria, get_video_recommendations, - prepare_tweet, + prepare_text, tweet_top_contributor_graph, ) @@ -174,7 +174,7 @@ def test_prepare_tweet(self, mock_get_twitter_account_from_video_id): mock_get_twitter_account_from_video_id.return_value = "@TournesolApp" - assert prepare_tweet(self.videos[8]) == tweet_text + assert prepare_text(self.videos[8], dest="twitter") == tweet_text # Test automatic shortening of the video title to fit in the tweet self.videos[8].metadata[ @@ -183,14 +183,14 @@ def test_prepare_tweet(self, mock_get_twitter_account_from_video_id): tweet_text_too_long = ( "Aujourd'hui, je recommande 'Tournesol is great! But this title is way to long to fit" - " in one tw...' de @TournesolApp, comparée 77 fois sur #Tournesol🌻 par 28 " + " in one twee…' de @TournesolApp, comparée 77 fois sur #Tournesol🌻 par 28 " "contributeurs, critères favoris:" "\n- Important & actionnable" "\n- Stimulant & suscite la réflexion" "\ntournesol.app/entities/yt:AAAAAAAAAAA" ) - assert prepare_tweet(self.videos[8]) == tweet_text_too_long + assert prepare_text(self.videos[8], dest="twitter") == tweet_text_too_long # Test replacement of special characters in the video title self.videos[8].metadata["name"] = "Tournesol.app is great but mention @twitter are not..." @@ -204,7 +204,7 @@ def test_prepare_tweet(self, mock_get_twitter_account_from_video_id): "\ntournesol.app/entities/yt:AAAAAAAAAAA" ) - assert prepare_tweet(self.videos[8]) == tweet_special_characters + assert prepare_text(self.videos[8], dest="twitter") == tweet_special_characters def test_get_video_recommendations(self): """ @@ -297,8 +297,8 @@ def test_tweet_top_contributor_graph(self, api_mock, client_mock): mocked_api_client = api_mock.return_value mocked_v2_client = client_mock.return_value - tweet_top_contributor_graph("@TournesolBot", assumeyes=True) - tweet_top_contributor_graph("@TournesolBotFR", assumeyes=True) + tweet_top_contributor_graph("@TournesolBot", assumeyes=True, dest=["twitter"]) + tweet_top_contributor_graph("@TournesolBotFR", assumeyes=True, dest=["twitter"]) self.assertEqual( mocked_api_client.media_upload.call_count, 2, mocked_api_client.media_upload.calls diff --git a/backend/twitterbot/tournesolbot.py b/backend/twitterbot/tournesolbot.py index e70fbdad6a..ddde2ddf13 100644 --- a/backend/twitterbot/tournesolbot.py +++ b/backend/twitterbot/tournesolbot.py @@ -3,6 +3,7 @@ import tempfile from datetime import timedelta from pathlib import Path +from typing import Literal import matplotlib.image as mpimg import matplotlib.pyplot as plt @@ -17,11 +18,15 @@ from tournesol.models.poll import DEFAULT_POLL_NAME, Poll from tournesol.utils.contributors import get_top_public_contributors_last_month from twitterbot import settings -from twitterbot.models.tweeted import TweetInfo -from twitterbot.twitter_api import TwitterBot +from twitterbot.client import TournesolBotClient +from twitterbot.models.history import TweetInfo from twitterbot.uploader_twitter_account import get_twitter_account_from_video_id +def get_video_short_url(video: Entity): + return f"tournesol.app/entities/{video.uid}" + + def get_best_criteria(video, nb_criteria): """Get the nb_criteria best-rated criteria""" @@ -40,7 +45,7 @@ def get_best_criteria(video, nb_criteria): return [(crit.criteria, crit.score) for crit in criteria_list] -def prepare_tweet(video: Entity): +def prepare_text(video: Entity, dest: Literal["twitter", "bluesky"]): """Create the tweet text from the video.""" uploader = video.metadata["uploader"] @@ -48,10 +53,15 @@ def prepare_tweet(video: Entity): video_id = video.metadata["video_id"] # Get twitter account - twitter_account = get_twitter_account_from_video_id(video_id) + if dest == "twitter": + channel_handle = get_twitter_account_from_video_id(video_id) + else: + channel_handle = None + # TODO: implement fetch Bluesky handle + # twitter_account = get_bluesky_handle_from_video_id(video_id) - if not twitter_account: - twitter_account = f"'{uploader}'" + if not channel_handle: + channel_handle = f"'{uploader}'" # Get two best criteria and criteria dict name crit1, crit2 = get_best_criteria(video, 2) @@ -59,17 +69,18 @@ def prepare_tweet(video: Entity): CriteriaLocale.objects.filter(language=language).values_list("criteria__name", "label") ) - # Replace "@" by a smaller "@" to avoid false mentions in the tweet - video_title = video.metadata["name"].replace("@", "﹫") + if dest == "twitter": + # Replace "@" by a smaller "@" to avoid false mentions in the tweet + video_title = video.metadata["name"].replace("@", "﹫") - # Replace "." in between words to avoid in the tweet false detection of links - video_title = re.sub(r"\b(?:\.)\b", "․", video_title) + # Replace "." in between words to avoid in the tweet false detection of links + video_title = re.sub(r"\b(?:\.)\b", "․", video_title) # Generate the text of the tweet poll_rating = video.all_poll_ratings.get(poll__name=DEFAULT_POLL_NAME) tweet_text = settings.tweet_text_template[language].format( title=video_title, - twitter_account=twitter_account, + twitter_account=channel_handle, n_comparison=poll_rating.n_comparisons, n_contributor=poll_rating.n_contributors, crit1=crit_dict[crit1[0]], @@ -78,15 +89,15 @@ def prepare_tweet(video: Entity): ) # Check the total length of the tweet and shorten title if the tweet is too long - # 288 is used because the link will be count as 23 characters and not 37 so 274 which leaves - # a margin of error for emoji which are counted as 2 characters - diff = len(tweet_text) - 288 + # 250 is used because the link will be counted as 23 characters, which leaves + # a margin of error for emoji which are counted as 2 characters before reaching + # the limit of 280 characters. + diff = len(tweet_text) - 250 if diff > 0: - video_title = video_title[: -diff - 3] + "..." - + video_title = video_title[: -diff - 1] + "…" tweet_text = settings.tweet_text_template[language].format( title=video_title, - twitter_account=twitter_account, + twitter_account=channel_handle, n_comparison=poll_rating.n_comparisons, n_contributor=poll_rating.n_contributors, crit1=crit_dict[crit1[0]], @@ -94,6 +105,10 @@ def prepare_tweet(video: Entity): video_id=video_id, ) + if dest != "bluesky": + # on Bluesky the URL preview is attached separately as "embed" + tweet_text += f"\n{get_video_short_url(video)}" + return tweet_text @@ -151,52 +166,57 @@ def select_a_video(tweetable_videos): return selected_video -def tweet_video_recommendation(bot_name, assumeyes=False): +def tweet_video_recommendation(bot_name, dest: list[str], assumeyes=False): """Tweet a video recommendation. Args: bot_name (str): The name of the bot. assumeyes (bool): If False, a confirmation will be asked before tweeting it. - + dest (list[str]): List of destinations where to post the message + Accepted values are "twitter" and "bluesky". """ - twitterbot = TwitterBot(bot_name) + bot_client = TournesolBotClient(bot_name) - tweetable_videos = get_video_recommendations(language=twitterbot.language) + tweetable_videos = get_video_recommendations(language=bot_client.language) if not tweetable_videos: print("No video reach the criteria to be tweeted today!!!") return video = select_a_video(tweetable_videos) - tweet_text = prepare_tweet(video) - print("Today's video to tweet will be:") - print(tweet_text) + print("Today's video to post will be:") + print(f"{video} '{video.metadata['name']}' by {video.metadata['uploader']}") if not assumeyes: confirmation = input("\nWould you like to tweet that? (y/n): ") if confirmation not in ["y", "yes"]: return + tweet_id = None + atproto_uri = None # Tweet the video - resp = twitterbot.client.create_tweet(text=tweet_text) - tweet_id = resp.data['id'] + if "twitter" in dest: + tweet_text = prepare_text(video, dest="twitter") + tweet_id = bot_client.create_tweet(text=tweet_text) - # Post the tweet on Discord - discord_channel = settings.TWITTERBOT_DISCORD_CHANNEL - if discord_channel: - write_in_channel( - discord_channel, - f"https://twitter.com/{bot_name}/status/{tweet_id}", - ) + if "bluesky" in dest: + text = prepare_text(video, dest="bluesky") + atproto_uri = bot_client.create_bluesky_post(text=text, embed_video=video) # Add the video to the TweetInfo table - TweetInfo.objects.create( + tweet_info: TweetInfo = TweetInfo.objects.create( video=video, tweet_id=tweet_id, + atproto_uri=atproto_uri, bot_name=bot_name, ) + # Post the tweet on Discord + discord_channel = settings.TWITTERBOT_DISCORD_CHANNEL + if discord_channel: + write_in_channel(discord_channel, message=tweet_info.message_url) + def generate_top_contributor_figure(top_contributors_qs, language="en") -> Path: """Generate a figure with the top contributor of each video.""" @@ -251,7 +271,7 @@ def generate_top_contributor_figure(top_contributors_qs, language="en") -> Path: return figure_path -def tweet_top_contributor_graph(bot_name, assumeyes=False): +def tweet_top_contributor_graph(bot_name, dest: Literal["twitter", "bluesky"], assumeyes=False): """Tweet the top contibutor graph of last month. Args: @@ -260,8 +280,8 @@ def tweet_top_contributor_graph(bot_name, assumeyes=False): """ - twitterbot = TwitterBot(bot_name) - language = twitterbot.language + bot_client = TournesolBotClient(bot_name) + language = bot_client.language top_contributors_qs = get_top_public_contributors_last_month( poll_name=DEFAULT_POLL_NAME, top=10 @@ -280,19 +300,28 @@ def tweet_top_contributor_graph(bot_name, assumeyes=False): if confirmation not in ["y", "yes"]: return - # Upload image - media = twitterbot.api.media_upload(top_contributor_figure) - - # Tweet the graph - resp = twitterbot.client.create_tweet( - text=settings.top_contrib_tweet_text_template[language], - media_ids=[media.media_id], - ) + message_url = None + if "twitter" in dest: + tweet_id = bot_client.create_tweet( + text=settings.top_contrib_tweet_text_template[language], + media_files=[top_contributor_figure] + ) + message_url = f"https://twitter.com/{bot_name}/status/{tweet_id}" - # Post the tweet on Discord - discord_channel = settings.TWITTERBOT_DISCORD_CHANNEL - if discord_channel: - write_in_channel( - discord_channel, - f"https://twitter.com/{bot_name}/status/{resp.data['id']}", + if "bluesky" in dest: + post_uri = bot_client.create_bluesky_post( + text=settings.top_contrib_tweet_text_template[language], + image_files=[top_contributor_figure], + image_alts=[settings.top_contrib_tweet_image_alt[language]] ) + post_id = post_uri.rsplit("/", 1)[-1] + message_url = f'https://bsky.app/profile/{bot_client.bluesky_handle}/post/{post_id}' + + if message_url is not None: + # Post the tweet on Discord + discord_channel = settings.TWITTERBOT_DISCORD_CHANNEL + if discord_channel: + write_in_channel( + discord_channel, + message=message_url, + ) diff --git a/backend/twitterbot/twitter_api.py b/backend/twitterbot/twitter_api.py deleted file mode 100644 index d3788ecdb1..0000000000 --- a/backend/twitterbot/twitter_api.py +++ /dev/null @@ -1,38 +0,0 @@ -import tweepy -from django.conf import settings - - -class TwitterBot: - def __init__(self, account): - - credentials = settings.TWITTERBOT_CREDENTIALS - - if account not in credentials: - raise ValueError(f"No credentials found for {account} account!") - - account_cred = credentials[account] - - self.language = account_cred["LANGUAGE"] - - consumer_key = account_cred["CONSUMER_KEY"] - consumer_secret = account_cred["CONSUMER_SECRET"] - access_token = account_cred["ACCESS_TOKEN"] - access_token_secret = account_cred["ACCESS_TOKEN_SECRET"] - - # v2 - self.client = tweepy.Client( - consumer_key=consumer_key, - consumer_secret=consumer_secret, - access_token=access_token, - access_token_secret=access_token_secret, - ) - - # v1.1 - # We need this authentication also because to add media it only works with api v1.1 - auth = tweepy.OAuth1UserHandler( - consumer_key=consumer_key, - consumer_secret=consumer_secret, - access_token=access_token, - access_token_secret=access_token_secret, - ) - self.api = tweepy.API(auth) diff --git a/backend/twitterbot/uploader_twitter_account.py b/backend/twitterbot/uploader_twitter_account.py index 8f6af4d4c8..b28d94a7a7 100644 --- a/backend/twitterbot/uploader_twitter_account.py +++ b/backend/twitterbot/uploader_twitter_account.py @@ -11,36 +11,39 @@ from tournesol.utils.api_youtube import get_video_metadata +def get_video_channel_html(video_id: str): + metadata = get_video_metadata(video_id, compute_language=False) + channel_id = metadata["channel_id"] + channel_about_url = f"https://www.youtube.com/channel/{channel_id}/about" + resp = requests.get(channel_about_url, headers={"user-agent": "curl/7.68.0"}, timeout=10) + resp.raise_for_status() + return resp.text + + def get_twitter_handles_from_html(html_text): urls = re.findall(r"(https?%3A%2F%2F(?:www\.)?twitter\.com%2F.*?)\"", html_text, re.IGNORECASE) - handles = [] + handles: set[str] = set() for raw_url in urls: url = unquote(raw_url) - handle = urlparse(url).path.strip('/') + handle = urlparse(url).path.strip("/") if handle: - handles.append(handle) + handles.add(f"@{handle.lower()}") return handles -def get_twitter_account_from_channel_id(channel_id): - """Get Twitter account from uploader id""" - - uploader_url = f"https://www.youtube.com/channel/{channel_id}/about" - - resp = requests.get(uploader_url, headers={"user-agent": "curl/7.68.0"}, timeout=10) - twitter_names = get_twitter_handles_from_html(resp.text) - - if len({name.lower() for name in twitter_names}) == 1: - return "@" + twitter_names[0] +def get_twitter_account_from_video_id(video_id: str): + """Get Twitter account from video id""" - print("Error getting the uploader id, Twitter account not found") - return None + channel_html = get_video_channel_html(video_id) + handles = get_twitter_handles_from_html(channel_html) + if len(handles) == 1: + return handles.pop() -def get_twitter_account_from_video_id(video_id): - """Get Twitter account from video id""" + if len(handles) > 1: + print("Error getting the uploader han, Twitter account not found") - metadata = get_video_metadata(video_id, compute_language=False) - channel_id = metadata["channel_id"] + if len(handles) == 0: + print("Twitter account not found: no handle found in html.") - return get_twitter_account_from_channel_id(channel_id) + return None diff --git a/backend/twitterbot/views/__init__.py b/backend/twitterbot/views/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/infra/ansible/roles/django/templates/settings.yaml.j2 b/infra/ansible/roles/django/templates/settings.yaml.j2 index 8a65f1f8e7..425dfdaaa5 100644 --- a/infra/ansible/roles/django/templates/settings.yaml.j2 +++ b/infra/ansible/roles/django/templates/settings.yaml.j2 @@ -69,6 +69,8 @@ TWITTERBOT_CREDENTIALS: "CONSUMER_SECRET": "{{consumer_secret_twitterbot_fr}}", "ACCESS_TOKEN": "{{access_token_twitterbot_fr}}", "ACCESS_TOKEN_SECRET": "{{access_token_secret_twitterbot_fr}}", + "ATPROTO_HANDLE": "tournesolbotfr.tournesol.app", + "ATPROTO_PASSWORD": "{{atproto_password_tournesolbot_fr}}", } "@TournesolBot": { "LANGUAGE": "en", @@ -76,6 +78,8 @@ TWITTERBOT_CREDENTIALS: "CONSUMER_SECRET": "{{consumer_secret_twitterbot_en}}", "ACCESS_TOKEN": "{{access_token_twitterbot_en}}", "ACCESS_TOKEN_SECRET": "{{access_token_secret_twitterbot_en}}", + "ATPROTO_HANDLE": "tournesolbot.tournesol.app", + "ATPROTO_PASSWORD": "{{atproto_password_tournesolbot_en}}", } DISCORD_CHANNEL_WEBHOOKS: { diff --git a/infra/ansible/scripts/deploy-without-secrets.sh b/infra/ansible/scripts/deploy-without-secrets.sh index f73053eb9a..c3da271028 100755 --- a/infra/ansible/scripts/deploy-without-secrets.sh +++ b/infra/ansible/scripts/deploy-without-secrets.sh @@ -57,3 +57,5 @@ ansible-playbook -i inventory.yml -l "$ANSIBLE_HOST" "$SETUP_FILE" \ -e "discord_infra_alert_private_webhook=${DISCORD_INFRA_ALERT_PRIVATE_WEBHOOK:-""}" \ -e "discord_twitter_webhook=${DISCORD_TWITTER_WEBHOOK:-""}" \ -e "plausible_analytics_secret_key=$PLAUSIBLE_ANALYTICS_SECRET_KEY" \ + -e "atproto_password_tournesolbot_fr=$ATPROTO_PASSWORD_BOT_FR" \ + -e "atproto_password_tournesolbot_en=$ATPROTO_PASSWORD_BOT_EN" \ diff --git a/infra/ansible/scripts/forget-secrets.sh b/infra/ansible/scripts/forget-secrets.sh index 8cbc209668..5a7c1aca0a 100755 --- a/infra/ansible/scripts/forget-secrets.sh +++ b/infra/ansible/scripts/forget-secrets.sh @@ -30,3 +30,5 @@ unset DISCORD_INFRA_ALERT_WEBHOOK unset DISCORD_INFRA_ALERT_PRIVATE_WEBHOOK unset DISCORD_TWITTER_WEBHOOK unset PLAUSIBLE_ANALYTICS_SECRET_KEY +unset ATPROTO_PASSWORD_BOT_FR +unset ATPROTO_PASSWORD_BOT_EN diff --git a/infra/ansible/scripts/get-vm-secrets.sh b/infra/ansible/scripts/get-vm-secrets.sh index 287db999d0..3ebbbb2d1a 100755 --- a/infra/ansible/scripts/get-vm-secrets.sh +++ b/infra/ansible/scripts/get-vm-secrets.sh @@ -99,3 +99,9 @@ export DISCORD_TWITTER_WEBHOOK PLAUSIBLE_ANALYTICS_SECRET_KEY="$(ssh "$VM_USER@$VM_ADDR" -- sudo cat /root/plausible_analytics_secret_key)" export PLAUSIBLE_ANALYTICS_SECRET_KEY + +ATPROTO_PASSWORD_BOT_FR=$(get_settings_value .TWITTERBOT_CREDENTIALS.\"@TournesolBotFR\".ATPROTO_PASSWORD) +export ATPROTO_PASSWORD_BOT_FR + +ATPROTO_PASSWORD_BOT_EN=$(get_settings_value .TWITTERBOT_CREDENTIALS.\"@TournesolBot\".ATPROTO_PASSWORD) +export ATPROTO_PASSWORD_BOT_EN