Merge pull request #138 from NicolasConstant/develop

0.20.0 PR

VARIABLES.md

@@ -49,6 +49,8 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
 * `Instance:UnlistedTwitterAccounts` (default: null) to enable unlisted publication for selected twitter accounts, separated by `;` (please limit this to brands and other public profiles). 
 * `Instance:SensitiveTwitterAccounts` (default: null) mark all media from given accounts as sensitive by default, separated by `;`. 
 * `Instance:FailingTwitterUserCleanUpThreshold` (default: 700) set the max allowed errors (due to a banned/deleted/private account) from a Twitter Account retrieval before auto-removal. (by default an account is called every 15 mins)
+* `Instance:FailingFollowerCleanUpThreshold` (default: 30000) set the max allowed errors from a Follower (Fediverse) Account before auto-removal. (often due to account suppression, instance issues, etc)
+* `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
 
 # Docker Compose full example
 

src/BirdsiteLive.ActivityPub/ApDeserializer.cs

@@ -1,4 +1,5 @@
 using System;
+using BirdsiteLive.ActivityPub.Models;
 using Newtonsoft.Json;
 
 namespace BirdsiteLive.ActivityPub
@@ -19,6 +20,8 @@ public static Activity ProcessActivity(string json)
                         if(a.apObject.type == "Follow")
                             return JsonConvert.DeserializeObject<ActivityUndoFollow>(json);
                         break;
+                    case "Delete":
+                        return JsonConvert.DeserializeObject<ActivityDelete>(json);
                     case "Accept":
                         var accept = JsonConvert.DeserializeObject<ActivityAccept>(json);
                         //var acceptType = JsonConvert.DeserializeObject<Activity>(accept.apObject);

src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs

@@ -0,0 +1,10 @@
+using Newtonsoft.Json;
+
+namespace BirdsiteLive.ActivityPub.Models
+{
+    public class ActivityDelete : Activity
+    {
+        [JsonProperty("object")]
+        public object apObject { get; set; }
+    }
+}

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -13,5 +13,8 @@ public class InstanceSettings
         public string SensitiveTwitterAccounts { get; set; }
 
         public int FailingTwitterUserCleanUpThreshold { get; set; }
+        public int FailingFollowerCleanUpThreshold { get; set; } = -1;
+
+        public int UserCacheCapacity { get; set; }
     }
 }

src/BirdsiteLive.Domain/ActivityPubService.cs

@@ -11,7 +11,6 @@
 using BirdsiteLive.Common.Settings;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
-using Org.BouncyCastle.Bcpg;
 
 namespace BirdsiteLive.Domain
 {
@@ -45,6 +44,12 @@ public async Task<Actor> GetUser(string objectId)
             var httpClient = _httpClientFactory.CreateClient();
             httpClient.DefaultRequestHeaders.Add("Accept", "application/activity+json");
             var result = await httpClient.GetAsync(objectId);
+
+            if (result.StatusCode == HttpStatusCode.Gone)
+                throw new FollowerIsGoneException();
+
+            result.EnsureSuccessStatusCode();
+
             var content = await result.Content.ReadAsStringAsync();
 
             var actor = JsonConvert.DeserializeObject<Actor>(content);

src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs

@@ -0,0 +1,51 @@
+using System.Linq;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+
+namespace BirdsiteLive.Domain.BusinessUseCases
+{
+    public interface IProcessDeleteUser
+    {
+        Task ExecuteAsync(Follower follower);
+        Task ExecuteAsync(string followerUsername, string followerDomain);
+    }
+
+    public class ProcessDeleteUser : IProcessDeleteUser
+    {
+        private readonly IFollowersDal _followersDal;
+        private readonly ITwitterUserDal _twitterUserDal;
+
+        #region Ctor
+        public ProcessDeleteUser(IFollowersDal followersDal, ITwitterUserDal twitterUserDal)
+        {
+            _followersDal = followersDal;
+            _twitterUserDal = twitterUserDal;
+        }
+        #endregion
+
+        public async Task ExecuteAsync(string followerUsername, string followerDomain)
+        {
+            // Get Follower and Twitter Users
+            var follower = await _followersDal.GetFollowerAsync(followerUsername, followerDomain);
+            if (follower == null) return;
+
+            await ExecuteAsync(follower);
+        }
+
+        public async Task ExecuteAsync(Follower follower)
+        {
+            // Remove twitter users if no more followers
+            var followings = follower.Followings;
+            foreach (var following in followings)
+            {
+                var followers = await _followersDal.GetFollowersAsync(following);
+                if (followers.Length == 1 && followers.First().Id == follower.Id)
+                    await _twitterUserDal.DeleteTwitterUserAsync(following);
+            }
+
+            // Remove follower from DB
+            await _followersDal.DeleteFollowerAsync(follower.Id);
+        }
+    }
+}

src/BirdsiteLive.Domain/Exceptions/FollowerIsGoneException.cs

@@ -0,0 +1,8 @@
+using System;
+
+namespace BirdsiteLive.Domain
+{
+    public class FollowerIsGoneException : Exception
+    {
+    }
+}

src/BirdsiteLive.Domain/Tools/SigValidationResultExtractor.cs

@@ -0,0 +1,22 @@
+using System.Linq;
+
+namespace BirdsiteLive.Domain.Tools
+{
+    public class SigValidationResultExtractor
+    {
+        public static string GetUserName(SignatureValidationResult result)
+        {
+            return result.User.preferredUsername.ToLowerInvariant().Trim();
+        }
+
+        public static string GetHost(SignatureValidationResult result)
+        {
+            return result.User.url.Replace("https://", string.Empty).Split('/').First();
+        }
+
+        public static string GetSharedInbox(SignatureValidationResult result)
+        {
+            return result.User?.endpoints?.sharedInbox;
+        }
+    }
+}

src/BirdsiteLive.Domain/UserService.cs

@@ -7,6 +7,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Converters;
+using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
 using BirdsiteLive.Cryptography;
@@ -28,10 +29,12 @@ public interface IUserService
         Task<bool> UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityUndoFollow activity, string body);
 
         Task<bool> SendRejectFollowAsync(ActivityFollow activity, string followerHost);
+        Task<bool> DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders, ActivityDelete activity, string body);
     }
 
     public class UserService : IUserService
     {
+        private readonly IProcessDeleteUser _processDeleteUser;
         private readonly IProcessFollowUser _processFollowUser;
         private readonly IProcessUndoFollowUser _processUndoFollowUser;
 
@@ -46,7 +49,7 @@ public class UserService : IUserService
         private readonly IModerationRepository _moderationRepository;
 
         #region Ctor
-        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository)
+        public UserService(InstanceSettings instanceSettings, ICryptoService cryptoService, IActivityPubService activityPubService, IProcessFollowUser processFollowUser, IProcessUndoFollowUser processUndoFollowUser, IStatusExtractor statusExtractor, IExtractionStatisticsHandler statisticsHandler, ITwitterUserService twitterUserService, IModerationRepository moderationRepository, IProcessDeleteUser processDeleteUser)
         {
             _instanceSettings = instanceSettings;
             _cryptoService = cryptoService;
@@ -57,6 +60,7 @@ public UserService(InstanceSettings instanceSettings, ICryptoService cryptoServi
             _statisticsHandler = statisticsHandler;
             _twitterUserService = twitterUserService;
             _moderationRepository = moderationRepository;
+            _processDeleteUser = processDeleteUser;
         }
         #endregion
 
@@ -126,10 +130,10 @@ public async Task<bool> FollowRequestedAsync(string signature, string method, st
             if (!sigValidation.SignatureIsValidated) return false;
 
             // Prepare data
-            var followerUserName = sigValidation.User.preferredUsername.ToLowerInvariant().Trim();
-            var followerHost = sigValidation.User.url.Replace("https://", string.Empty).Split('/').First();
+            var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation);
+            var followerHost = SigValidationResultExtractor.GetHost(sigValidation);
             var followerInbox = sigValidation.User.inbox;
-            var followerSharedInbox = sigValidation.User?.endpoints?.sharedInbox;
+            var followerSharedInbox = SigValidationResultExtractor.GetSharedInbox(sigValidation);
             var twitterUser = activity.apObject.Split('/').Last().Replace("@", string.Empty).ToLowerInvariant().Trim();
 
             // Make sure to only keep routes
@@ -213,7 +217,7 @@ public async Task<bool> SendRejectFollowAsync(ActivityFollow activity, string fo
             return result == HttpStatusCode.Accepted ||
                    result == HttpStatusCode.OK; //TODO: revamp this for better error handling
         }
-
+        
         private string OnlyKeepRoute(string inbox, string host)
         {
             if (string.IsNullOrWhiteSpace(inbox)) 
@@ -258,6 +262,22 @@ public async Task<bool> UndoFollowRequestedAsync(string signature, string method
             return result == HttpStatusCode.Accepted || result == HttpStatusCode.OK; //TODO: revamp this for better error handling
         }
 
+        public async Task<bool> DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders,
+            ActivityDelete activity, string body)
+        {
+            // Validate
+            var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
+            if (!sigValidation.SignatureIsValidated) return false;
+
+            // Remove user and followings
+            var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation);
+            var followerHost = SigValidationResultExtractor.GetHost(sigValidation);
+
+            await _processDeleteUser.ExecuteAsync(followerUserName, followerHost);
+
+            return true;
+        }
+
         private async Task<SignatureValidationResult> ValidateSignature(string actor, string rawSig, string method, string path, string queryString, Dictionary<string, string> requestHeaders, string body)
         {
             //Check Date Validity

src/BirdsiteLive.Moderation/Actions/RemoveFollowerAction.cs

@@ -1,12 +1,6 @@
-using System;
-using System.Linq;
-using System.Threading.Tasks;
-using BirdsiteLive.ActivityPub;
-using BirdsiteLive.ActivityPub.Converters;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
+using System.Threading.Tasks;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Domain;
+using BirdsiteLive.Domain.BusinessUseCases;
 
 namespace BirdsiteLive.Moderation.Actions
 {
@@ -17,16 +11,14 @@ public interface IRemoveFollowerAction
 
     public class RemoveFollowerAction : IRemoveFollowerAction
     {
-        private readonly IFollowersDal _followersDal;
-        private readonly ITwitterUserDal _twitterUserDal;
         private readonly IRejectAllFollowingsAction _rejectAllFollowingsAction;
+        private readonly IProcessDeleteUser _processDeleteUser;
 
         #region Ctor
-        public RemoveFollowerAction(IFollowersDal followersDal, ITwitterUserDal twitterUserDal, IRejectAllFollowingsAction rejectAllFollowingsAction)
+        public RemoveFollowerAction(IRejectAllFollowingsAction rejectAllFollowingsAction, IProcessDeleteUser processDeleteUser)
         {
-            _followersDal = followersDal;
-            _twitterUserDal = twitterUserDal;
             _rejectAllFollowingsAction = rejectAllFollowingsAction;
+            _processDeleteUser = processDeleteUser;
         }
         #endregion
 
@@ -36,16 +28,7 @@ public async Task ProcessAsync(Follower follower)
             await _rejectAllFollowingsAction.ProcessAsync(follower);
 
             // Remove twitter users if no more followers
-            var followings = follower.Followings;
-            foreach (var following in followings)
-            {
-                var followers = await _followersDal.GetFollowersAsync(following);
-                if (followers.Length == 1 && followers.First().Id == follower.Id)
-                    await _twitterUserDal.DeleteTwitterUserAsync(following);
-            }
-
-            // Remove follower from DB
-            await _followersDal.DeleteFollowerAsync(follower.Id);
+            await _processDeleteUser.ExecuteAsync(follower);
         }
     }
 }

src/BirdsiteLive.Pipeline/Processors/RefreshTwitterUserStatusProcessor.cs

@@ -9,6 +9,7 @@
 using BirdsiteLive.Pipeline.Contracts;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
+using BirdsiteLive.Twitter.Models;
 
 namespace BirdsiteLive.Pipeline.Processors
 {
@@ -35,26 +36,61 @@ public async Task<UserWithDataToSync[]> ProcessAsync(SyncTwitterUser[] syncTwitt
 
             foreach (var user in syncTwitterUsers)
             {
-                var userView = _twitterUserService.GetUser(user.Acct);
-                if (userView == null)
+                TwitterUser userView = null;
+
+                try
                 {
-                    await AnalyseFailingUserAsync(user);
+                    userView = _twitterUserService.GetUser(user.Acct);
                 }
-                else if (!userView.Protected)
+                catch (UserNotFoundException)
                 {
-                    user.FetchingErrorCount = 0;
-                    var userWtData = new UserWithDataToSync
-                    {
-                        User = user
-                    };
-                    usersWtData.Add(userWtData);
+                    await ProcessNotFoundUserAsync(user);
+                    continue;
+                }
+                catch (UserHasBeenSuspendedException)
+                {
+                    await ProcessNotFoundUserAsync(user);
+                    continue;
+                }
+                catch (RateLimitExceededException)
+                {
+                    await ProcessRateLimitExceededAsync(user);
+                    continue;
+                }
+                catch (Exception)
+                {
+                    // ignored
+                }
+
+                if (userView == null || userView.Protected)
+                {
+                    await ProcessFailingUserAsync(user);
+                    continue;
                 }
-            }
 
+                user.FetchingErrorCount = 0;
+                var userWtData = new UserWithDataToSync
+                {
+                    User = user
+                };
+                usersWtData.Add(userWtData);
+            }
             return usersWtData.ToArray();
         }
 
-        private async Task AnalyseFailingUserAsync(SyncTwitterUser user)
+        private async Task ProcessRateLimitExceededAsync(SyncTwitterUser user)
+        {
+            var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
+            dbUser.LastSync = DateTime.UtcNow;
+            await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
+        }
+
+        private async Task ProcessNotFoundUserAsync(SyncTwitterUser user)
+        {
+            await _removeTwitterAccountAction.ProcessAsync(user);
+        }
+
+        private async Task ProcessFailingUserAsync(SyncTwitterUser user)
         {
             var dbUser = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
             dbUser.FetchingErrorCount++;
@@ -68,9 +104,6 @@ private async Task AnalyseFailingUserAsync(SyncTwitterUser user)
             {
                 await _twitterUserDal.UpdateTwitterUserAsync(dbUser);
             }
-
-            // Purge
-            _twitterUserService.PurgeUser(user.Acct);
         }
     }
 }