Removed signatures and task cleanup

docker-compose.yml

@@ -198,6 +198,6 @@ services:
             - RUN_ENV=production
             - LOG_LEVEL=INFO
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.scheduled_tasks,redditrepostsleuth.core.celery.admin_tasks
-        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q subreddit_config_updates,update_wiki_from_database -n subreddit_config_update_worker --autoscale=8,1
+        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q batch_delete_searches,subreddit_config_updates,update_wiki_from_database -n subreddit_config_update_worker --autoscale=8,1
 
 

redditrepostsleuth/adminsvc/inbox_monitor.py

@@ -39,7 +39,8 @@ def check_inbox(self):
         print('[Scheduled Job] Checking Inbox Start')
         for msg in self.reddit.inbox.messages(limit=500):
             if msg.author != 'RepostSleuthBot' and msg.subject.lower() in ['false negative', 'false positive']:
-                self._process_user_report(msg)
+                #self._process_user_report(msg)
+                pass
             elif TOP_POST_WATCH_SUBJECT.lower() in msg.subject.lower():
                 self._process_watch_request(msg)
 

redditrepostsleuth/adminsvc/new_activation_monitor.py

@@ -36,7 +36,7 @@ def __init__(
     def check_for_new_invites(self):
         for msg in self.reddit.inbox.messages(limit=1000):
             if 'invitation to moderate' in msg.subject:
-                log.info('Found invitation for %s', msg.subreddit.display_name)
+                log.debug('Found invitation for %s', msg.subreddit.display_name)
                 self.activate_sub(msg)
 
 
@@ -100,7 +100,7 @@ def _create_monitored_sub_in_db(self, msg: Message) -> MonitoredSub:
         with self.uowm.start() as uow:
             existing = uow.monitored_sub.get_by_sub(msg.subreddit.display_name)
             if existing:
-                log.info('Monitored sub %s already exists, skipping activation', msg.subreddit.display_name)
+                log.debug('Monitored sub %s already exists, skipping activation', msg.subreddit.display_name)
                 raise ValueError(f'Monitored Sub already in database: {msg.subreddit.display_name}')
             monitored_sub = MonitoredSub(**{**DEFAULT_CONFIG_VALUES, **{'name': msg.subreddit.display_name}})
             uow.monitored_sub.add(monitored_sub)

redditrepostsleuth/core/celery/admin_tasks.py

@@ -17,6 +17,7 @@
 from redditrepostsleuth.core.exception import UtilApiException, UserNotFound
 from redditrepostsleuth.core.logfilters import ContextFilter
 from redditrepostsleuth.core.logging import log, configure_logger
+from redditrepostsleuth.core.util.helpers import chunk_list
 from redditrepostsleuth.core.util.onlyfans_handling import check_user_for_promoter_links, \
     check_user_comments_for_promoter_links
 
@@ -207,4 +208,6 @@ def check_user_comments_for_only_fans(self, username: str) -> None:
     except IntegrityError:
         pass
     except Exception as e:
-        log.exception('')
+        log.exception('')
+
+

redditrepostsleuth/core/celery/celeryconfig.py

@@ -28,17 +28,18 @@
     'redditrepostsleuth.core.celery.tasks.scheduled_tasks.*': {'queue': 'scheduled_tasks'},
     'redditrepostsleuth.core.celery.admin_tasks.update_proxies_job': {'queue': 'scheduled_tasks'},
     'redditrepostsleuth.core.celery.admin_tasks.check_user_for_only_fans': {'queue': 'onlyfans_check'},
-    'redditrepostsleuth.core.celery.admin_tasks.update_subreddit_config_from_database': {'queue': 'update_wiki_from_database'}
+    'redditrepostsleuth.core.celery.admin_tasks.update_subreddit_config_from_database': {'queue': 'update_wiki_from_database'},
+    'redditrepostsleuth.core.celery.admin_tasks.delete_search_batch': {'queue': 'batch_delete_searches'},
 
 
 }
 
 
 beat_schedule = {
-    'update-proxy-list': {
-        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_proxies_task',
-        'schedule': 3600
-    },
+    # 'update-proxy-list': {
+    #     'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_proxies_task',
+    #     'schedule': 3600
+    # },
     'check-inbox': {
         'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.check_inbox_task',
         'schedule': 300
@@ -60,7 +61,7 @@
         'schedule': 300
     },
     'update-top-reposts': {
-        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_top_reposts_task',
+        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_all_top_reposts_task',
         'schedule': 86400
     },
     'update-top-reposters': {
@@ -71,14 +72,14 @@
         'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_daily_top_reposters_task',
         'schedule': 900
     },
-    'send-reports-to-meme-voting': {
-        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.send_reports_to_meme_voting_task',
-        'schedule': 3600
-    },
-    'check-meme-template-potential-votes': {
-        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.check_meme_template_potential_votes_task',
-        'schedule': 1800
-    },
+    # 'send-reports-to-meme-voting': {
+    #     'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.send_reports_to_meme_voting_task',
+    #     'schedule': 3600
+    # },
+    # 'check-meme-template-potential-votes': {
+    #     'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.check_meme_template_potential_votes_task',
+    #     'schedule': 1800
+    # },
     'monitored-sub-config-update': {
         'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.queue_config_updates_task',
         'schedule': 3600
@@ -91,6 +92,10 @@
         'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_daily_stats',
         'schedule': 86400
     },
+    'search-history-cleanup': {
+        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.queue_search_history_cleanup',
+        'schedule': 3600
+    },
 
 }
 

redditrepostsleuth/core/celery/task_logic/monitored_sub_task_logic.py

@@ -25,10 +25,6 @@ def process_monitored_subreddit_submission(post_id: str, monitored_sub_svc: Moni
         log.warning('Unknown post type for %s - https://redd.it/%s', post.post_id, post.post_id)
         return
 
-
-
-
-
     monitored_sub = uow.monitored_sub.get_by_sub(post.subreddit)
 
     if monitored_sub.adult_promoter_remove_post or monitored_sub.adult_promoter_ban_user or monitored_sub.adult_promoter_notify_mod_mail:

redditrepostsleuth/core/celery/task_logic/scheduled_task_logic.py

@@ -89,7 +89,7 @@ def update_top_reposts(uow: UnitOfWork, post_type_id: int, day_range: int = None
         uow.commit()
 
 def run_update_top_reposts(uow: UnitOfWork) -> None:
-    post_types = [1, 2, 3]
+    post_types = [2, 3]
     day_ranges = [1, 7, 14, 30, None]
     for post_type_id in post_types:
         for days in day_ranges:

redditrepostsleuth/core/celery/tasks/scheduled_tasks.py

@@ -12,6 +12,7 @@
     token_checker, run_update_top_reposters, update_top_reposters, update_monitored_sub_data, run_update_top_reposts
 from redditrepostsleuth.core.db.databasemodels import MonitoredSub, StatsDailyCount
 from redditrepostsleuth.core.logging import configure_logger
+from redditrepostsleuth.core.util.helpers import chunk_list
 from redditrepostsleuth.core.util.reddithelpers import is_sub_mod_praw, get_bot_permissions
 from redditrepostsleuth.core.util.replytemplates import MONITORED_SUB_MOD_REMOVED_CONTENT, \
     MONITORED_SUB_MOD_REMOVED_SUBJECT
@@ -204,14 +205,6 @@ def update_daily_top_reposters_task(self):
     except Exception as e:
         log.exception('Unknown task error')
 
-@celery.task(bind=True, base=SqlAlchemyTask)
-def update_top_reposts_task(self):
-    try:
-        update_top_reposts(self.uowm)
-    except Exception as e:
-        log.exception('Unknown task exception')
-
-
 
 @celery.task(bind=True, base=RedditTask, autoretry_for=(TooManyRequests,), retry_kwards={'max_retries': 3})
 def update_monitored_sub_stats_task(self, sub_name: str) -> None:
@@ -243,4 +236,31 @@ def update_proxies_task(self) -> None:
 @celery.task
 def update_profile_token_task():
     print('Staring token checker')
-    token_checker()
+    token_checker()
+
+@celery.task(bind=True, base=SqlAlchemyTask)
+def delete_search_batch(self, ids: list[int]):
+    try:
+        with self.uowm.start() as uow:
+            log.info('Starting range %s:%s', ids[0], ids[-1])
+            for id in ids:
+                search = uow.repost_search.get_by_id(id)
+                if search:
+                    log.debug('Deleting search %s', search.id)
+                    uow.repost_search.remove(search)
+            uow.commit()
+            log.info('Finished range %s:%s', ids[0], ids[-1])
+    except Exception as e:
+        log.exception('')
+
+@celery.task(bind=True, base=SqlAlchemyTask)
+def queue_search_history_cleanup(self):
+    with self.uowm.start() as uow:
+        searches = uow.repost_search.get_all_ids_older_than_days(120, limit=100000000)
+        if not searches:
+            log.info('No search history to cleanup')
+            return
+        log.info('Queuing Search History Cleanup.  Range: ID Range: %s:%s', searches[0].id, searches[-1].id)
+        ids = [x[0] for x in searches]
+        for chunk in chunk_list(ids, 5000):
+            delete_search_batch.apply_async((chunk,), queue='batch_delete_searches')

redditrepostsleuth/core/db/databasemodels.py

@@ -226,6 +226,8 @@ class RepostSearch(Base):
     searched_at = Column(DateTime, default=func.utc_timestamp(), nullable=False)
 
     post = relationship("Post", back_populates='searches')
+    monitored_sub_checked = relationship("MonitoredSubChecks", back_populates="search")
+    repost = relationship("Repost", back_populates="search")
     post_type = relationship('PostType')
 
     def __repr__(self):

redditrepostsleuth/core/db/repository/repost_search_repo.py

@@ -19,6 +19,25 @@ def add(self, search: RepostSearch):
     def update(self, revision: RepostSearch):
         self.db_session.update(revision)
 
+    def get_all_older_than_days(self, days: int, limit: int = 100) -> list[RepostSearch]:
+        delta = datetime.utcnow() - timedelta(days=days)
+        return self.db_session.query(RepostSearch).filter(RepostSearch.searched_at < delta).limit(limit).all()
+
+    def get_all_ids_older_than_days(self, days: int, limit: int = 100) -> list[RepostSearch]:
+        delta = datetime.utcnow() - timedelta(days=days)
+        return self.db_session.query(RepostSearch.id).filter(RepostSearch.searched_at < delta).order_by(RepostSearch.id).limit(limit).all()
+
+
+    def delete_all_older_than_days(self, days: int, limit: int = 100) -> None:
+        delta = datetime.utcnow() - timedelta(days=days)
+        self.db_session.query(RepostSearch).filter(RepostSearch.searched_at < delta).limit(limit).delete()
+
+    def delete_all_with_lower_id(self, lower_id: int) -> None:
+        self.db_session.query(RepostSearch).filter(RepostSearch.id < lower_id).delete()
+
+    def get_oldest_search(self) -> RepostSearch:
+        return self.db_session.query(RepostSearch).order_by(RepostSearch.id).first()
+
     def get_all(self, limit: int = None):
         return self.db_session.query(RepostSearch).limit(limit).all()
 

redditrepostsleuth/core/services/responsebuilder.py

@@ -163,10 +163,10 @@ def build_default_comment(
             if closest_template:
                 message += f'\n\n{closest_template}'
 
-        if signature:
-            message += f'\n\n{self._get_signature(search_results)} - {REPORT_POST_LINK}'
-        else:
-            message += f'\n\n{REPORT_POST_LINK}'
+        # if signature:
+        #     message += f'\n\n{self._get_signature(search_results)} - {REPORT_POST_LINK}'
+        # else:
+        #     message += f'\n\n{REPORT_POST_LINK}'
 
         # Checking post type is temp until the site supports everything
         if search_link and search_results.checked_post.post_type.name in ['image']:

redditrepostsleuth/core/util/onlyfans_handling.py

@@ -38,7 +38,8 @@
     'beacons.ai',
     'linktr.ee',
     'linkbio.co',
-    'snipfeed.co'
+    'snipfeed.co',
+    'allmylink.me'
 ]
 
 flagged_words = [
@@ -117,19 +118,20 @@ def fetch_from_util_api(url: str) -> Response:
 def get_profile_links(username: str) -> list[str]:
     url = f'{config.util_api}/profile?username={username}'
     response = fetch_from_util_api(url)
+    if response.status_code == 200:
+        profile_links = json.loads(response.text)
+        return profile_links
 
     if response.status_code == 404:
         log.info('Redditor %s no longer exists', username)
         raise UserNotFound(f'Redditor {username} no longer exists')
-
-    if response.status_code != 200:
+    elif response.status_code == 503:
+        log.info('No token to cehck user with')
+        return []
+    else:
         log.warning('Non 200 return code %s from Util API', response.status_code)
         raise UtilApiException(f'Unexpected status {response.status_code} from util API')
 
-    profile_links = json.loads(response.text)
-    return profile_links
-
-links = []
 
 def check_user_for_promoter_links(username: str) -> Optional[LinkCheckResult]:
 

redditrepostsleuth/core/util/replytemplates.py

@@ -75,7 +75,8 @@
 
 TOP_POST_REPORT_MSG = 'Looks like a repost. I\'ve seen this {post_type} {match_count} {times_word}. First seen {oldest_shortlink}' \
 
-IMAGE_SEARCH_SETTINGS = '**Scope:** {search_scope} | **Meme Filter:** {meme_filter_used} | **Target:** {effective_target_match_percent}% | **Check Title:** {check_title} | **Max Age:** {max_age}'
+IMAGE_SEARCH_SETTINGS = '**Scope:** {search_scope} | **Target Percent:** {effective_target_match_percent}% | **Max Age:** {max_age}'
+#IMAGE_SEARCH_SETTINGS = '**Scope:** {search_scope} | **Meme Filter:** {meme_filter_used} | **Target:** {effective_target_match_percent}% | **Check Title:** {check_title} | **Max Age:** {max_age}'
 GENERIC_SEARCH_SETTINGS = '**Scope:** {search_scope} | **Check Title:** {check_title} | **Max Age:** {max_days_old}'
 REPORT_RESPONSE = 'Thank you for your report. \n\nIt has been documented and will be reviewed further'
 

redditrepostsleuth/summonssvc/summons_monitor.py

@@ -87,6 +87,9 @@ def handle_summons(summons: Summons) -> None:
 def monitor_for_mentions(reddit: Reddit, uowm: UnitOfWorkManager):
 
     for comment in reddit.inbox.mentions():
+        if not comment.author:
+            log.info('Skipping comment without author')
+            continue
 
         if comment.created_utc < datetime.utcnow().timestamp() - 86400:
             log.debug('Skipping old mention. Created at %s', datetime.fromtimestamp(comment.created_utc))