Merge pull request #379 from barrycarey/dev

Dev merge
barrycarey · May 27, 2024 · bc37475 · bc37475
2 parents d6a4a42 + dd2f360
commit bc37475
Show file tree

Hide file tree

Showing 49 changed files with 1,525 additions and 667 deletions.
diff --git a/docker-compose-infra.yml b/docker-compose-infra.yml
@@ -31,7 +31,7 @@ services:
         - '6379:6379'
       volumes:
         - /config/redis:/data
-      command: redis-server --save 60 1
+      command: redis-server --save 550 1 --maxmemory 40gb
 
     influxdb:
       image: influxdb:latest

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -122,6 +122,22 @@ services:
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.monitored_sub_tasks
         entrypoint: celery -A redditrepostsleuth.core.celery worker -Q submonitor -n submonitor_worker --autoscale=6,2
 
+    reddit_actions_worker:
+        container_name: reddit-actions-worker
+        restart: unless-stopped
+        user: '1001'
+        build:
+            context: .
+            dockerfile: docker/WorkerDockerFile
+        env_file:
+            - .env
+        environment:
+            - RUN_ENV=production
+            - db_user=sub_monitor
+            - LOG_LEVEL=INFO
+            - CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.reddit_action_tasks
+        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q reddit_actions -n reddit_actions --autoscale=3,2
+
     ingest_worker:
         restart: unless-stopped
         container_name: ingest-worker
@@ -134,9 +150,9 @@ services:
         environment:
             - RUN_ENV=production
             - db_user=ingest
-            - LOG_LEVEL=ERROR
+            - LOG_LEVEL=INFO
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.ingest_tasks
-        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q post_ingest -n ingest_worker --autoscale=3,16
+        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q post_ingest -n ingest_worker --autoscale=16,1
 
     link_repost_worker:
         container_name: link-repost-worker
@@ -181,9 +197,9 @@ services:
             - .env
         environment:
             - RUN_ENV=production
-            - LOG_LEVEL=INFO
+            - LOG_LEVEL=WARNING
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.admin_tasks
-        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q onlyfans_check -n onlyfans_worker --autoscale=8,5
+        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q onlyfans_check -n onlyfans_worker --autoscale=10,5
 
     subreddit_config_update_worker:
         container_name: subreddit-config-update-worker
@@ -198,6 +214,6 @@ services:
             - RUN_ENV=production
             - LOG_LEVEL=INFO
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.scheduled_tasks,redditrepostsleuth.core.celery.admin_tasks
-        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q subreddit_config_updates,update_wiki_from_database -n subreddit_config_update_worker --autoscale=8,1
+        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q batch_delete_searches,subreddit_config_updates,update_wiki_from_database -n subreddit_config_update_worker --autoscale=8,1
 
 
diff --git a/docs/dev_docs/modifying_monitored_sub.md b/docs/dev_docs/modifying_monitored_sub.md
@@ -0,0 +1,5 @@
+
+### Adding or Removing Config Values
+* Add / Remove config values in core/db/databasemodels.py
+* Add/Remove in core/util/default_bot_config.py
+* Update sub_monitor_exposed_config_options in the config json
diff --git a/redditrepostsleuth/adminsvc/inbox_monitor.py b/redditrepostsleuth/adminsvc/inbox_monitor.py
@@ -39,7 +39,8 @@ def check_inbox(self):
         print('[Scheduled Job] Checking Inbox Start')
         for msg in self.reddit.inbox.messages(limit=500):
             if msg.author != 'RepostSleuthBot' and msg.subject.lower() in ['false negative', 'false positive']:
-                self._process_user_report(msg)
+                #self._process_user_report(msg)
+                pass
             elif TOP_POST_WATCH_SUBJECT.lower() in msg.subject.lower():
                 self._process_watch_request(msg)
 

diff --git a/redditrepostsleuth/adminsvc/new_activation_monitor.py b/redditrepostsleuth/adminsvc/new_activation_monitor.py
@@ -7,6 +7,7 @@
 from praw.models import Subreddit, Message
 from prawcore import TooManyRequests
 
+from redditrepostsleuth.core.celery.tasks.reddit_action_tasks import send_modmail_task
 from redditrepostsleuth.core.config import Config
 from redditrepostsleuth.core.db.databasemodels import MonitoredSub
 from redditrepostsleuth.core.db.db_utils import get_db_engine
@@ -36,7 +37,7 @@ def __init__(
     def check_for_new_invites(self):
         for msg in self.reddit.inbox.messages(limit=1000):
             if 'invitation to moderate' in msg.subject:
-                log.info('Found invitation for %s', msg.subreddit.display_name)
+                log.debug('Found invitation for %s', msg.subreddit.display_name)
                 self.activate_sub(msg)
 
 
@@ -78,11 +79,13 @@ def _notify_added(self, subreddit: Subreddit) -> NoReturn:
             log.info('Sending success PM to %s', subreddit.display_name)
             wiki_url = f'https://www.reddit.com/r/{subreddit.display_name}/wiki/repost_sleuth_config'
             try:
-                self.response_handler.send_mod_mail(
-                    subreddit.display_name,
-                    MONITORED_SUB_ADDED.format(wiki_config=wiki_url),
-                    'Repost Sleuth Activated',
-                    source='activation'
+                send_modmail_task.apply_async(
+                    (
+                        subreddit.display_name,
+                        MONITORED_SUB_ADDED.format(wiki_config=wiki_url),
+                        'Repost Sleuth Activated',
+                    ),
+                    {'source': 'activation'}
                 )
                 monitored_sub.activation_notification_sent = True
             except RedditAPIException as e:
@@ -100,7 +103,7 @@ def _create_monitored_sub_in_db(self, msg: Message) -> MonitoredSub:
         with self.uowm.start() as uow:
             existing = uow.monitored_sub.get_by_sub(msg.subreddit.display_name)
             if existing:
-                log.info('Monitored sub %s already exists, skipping activation', msg.subreddit.display_name)
+                log.debug('Monitored sub %s already exists, skipping activation', msg.subreddit.display_name)
                 raise ValueError(f'Monitored Sub already in database: {msg.subreddit.display_name}')
             monitored_sub = MonitoredSub(**{**DEFAULT_CONFIG_VALUES, **{'name': msg.subreddit.display_name}})
             uow.monitored_sub.add(monitored_sub)

diff --git a/redditrepostsleuth/core/celery/admin_tasks.py b/redditrepostsleuth/core/celery/admin_tasks.py
@@ -17,7 +17,9 @@
 from redditrepostsleuth.core.exception import UtilApiException, UserNotFound
 from redditrepostsleuth.core.logfilters import ContextFilter
 from redditrepostsleuth.core.logging import log, configure_logger
-from redditrepostsleuth.core.util.onlyfans_handling import check_user_for_promoter_links
+from redditrepostsleuth.core.util.helpers import chunk_list
+from redditrepostsleuth.core.util.onlyfans_handling import check_user_for_promoter_links, \
+    check_user_comments_for_promoter_links
 
 log = configure_logger(
     name='redditrepostsleuth',
@@ -124,27 +126,28 @@ def update_subreddit_config_from_database(self, monitored_sub: MonitoredSub, use
     )
 
 
-@celery.task(bind=True, base=AdminTask, autoretry_for=(UtilApiException,ConnectionError,TooManyRequests), retry_kwards={'max_retries': 3})
+@celery.task(bind=True, base=AdminTask, autoretry_for=(UtilApiException,ConnectionError,TooManyRequests), retry_kwards={'max_retries': 3, 'retry_backoff': True})
 def check_user_for_only_fans(self, username: str) -> None:
     skip_names = ['[deleted]', 'AutoModerator']
 
     if username in skip_names:
         log.info('Skipping name %s', username)
         return
+
     try:
         with self.uowm.start() as uow:
             user = uow.user_review.get_by_username(username)
 
             if user:
                 delta = datetime.utcnow() - user.last_checked
                 if delta.days < 30:
-                    log.debug('Skipping existing user %s, last check was %s days ago', username, delta.days)
+                    log.info('Skipping existing user %s, last check was %s days ago', username, delta.days)
                     return
                 user.content_links_found = False
                 user.notes = None
                 user.last_checked = func.utc_timestamp()
 
-            log.debug('Checking user %s', username)
+            log.info('Checking user %s', username)
             if not user:
                 user = UserReview(username=username)
             try:
@@ -164,4 +167,47 @@ def check_user_for_only_fans(self, username: str) -> None:
     except IntegrityError:
         pass
     except Exception as e:
-        log.exception('')
+        log.exception('')
+
+
+@celery.task(bind=True, base=AdminTask, autoretry_for=(UtilApiException,ConnectionError,TooManyRequests), retry_kwards={'max_retries': 3})
+def check_user_comments_for_only_fans(self, username: str) -> None:
+    """
+    This should be run after the profile check so we don't do any timeframe checking
+    :param self:
+    :param username:
+    :return:
+    """
+    skip_names = ['[deleted]', 'AutoModerator']
+
+    if username in skip_names:
+        log.info('Skipping name %s', username)
+        return
+
+    try:
+        with self.uowm.start() as uow:
+            user = uow.user_review.get_by_username(username)
+
+            if not user:
+                log.error('User not found: %s', username)
+
+            try:
+                result = check_user_comments_for_promoter_links(username)
+            except UserNotFound as e:
+                log.warning(e)
+                return
+
+            if result:
+                log.info('Promoter found: %s - %s', username, str(result))
+                user.content_links_found = True
+                user.notes = str(result)
+            uow.user_review.add(user)
+            uow.commit()
+    except (UtilApiException, ConnectionError, TooManyRequests) as e:
+        raise e
+    except IntegrityError:
+        pass
+    except Exception as e:
+        log.exception('')
+
+
diff --git a/redditrepostsleuth/core/celery/basetasks.py b/redditrepostsleuth/core/celery/basetasks.py
@@ -5,7 +5,6 @@
 from redditrepostsleuth.core.db.uow.unitofworkmanager import UnitOfWorkManager
 from redditrepostsleuth.core.notification.notification_service import NotificationService
 from redditrepostsleuth.core.services.eventlogging import EventLogging
-from redditrepostsleuth.core.services.reddit_manager import RedditManager
 from redditrepostsleuth.core.services.response_handler import ResponseHandler
 from redditrepostsleuth.core.services.subreddit_config_updater import SubredditConfigUpdater
 from redditrepostsleuth.core.util.helpers import get_reddit_instance

diff --git a/redditrepostsleuth/core/celery/celeryconfig.py b/redditrepostsleuth/core/celery/celeryconfig.py
@@ -28,17 +28,19 @@
     'redditrepostsleuth.core.celery.tasks.scheduled_tasks.*': {'queue': 'scheduled_tasks'},
     'redditrepostsleuth.core.celery.admin_tasks.update_proxies_job': {'queue': 'scheduled_tasks'},
     'redditrepostsleuth.core.celery.admin_tasks.check_user_for_only_fans': {'queue': 'onlyfans_check'},
-    'redditrepostsleuth.core.celery.admin_tasks.update_subreddit_config_from_database': {'queue': 'update_wiki_from_database'}
+    'redditrepostsleuth.core.celery.admin_tasks.update_subreddit_config_from_database': {'queue': 'update_wiki_from_database'},
+    'redditrepostsleuth.core.celery.admin_tasks.delete_search_batch': {'queue': 'batch_delete_searches'},
+    'redditrepostsleuth.core.celery.tasks.reddit_action_tasks.*': {'queue': 'reddit_actions'},
 
 
 }
 
 
 beat_schedule = {
-    'update-proxy-list': {
-        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_proxies_task',
-        'schedule': 3600
-    },
+    # 'update-proxy-list': {
+    #     'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_proxies_task',
+    #     'schedule': 3600
+    # },
     'check-inbox': {
         'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.check_inbox_task',
         'schedule': 300
@@ -60,7 +62,7 @@
         'schedule': 300
     },
     'update-top-reposts': {
-        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_top_reposts_task',
+        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_all_top_reposts_task',
         'schedule': 86400
     },
     'update-top-reposters': {
@@ -71,26 +73,30 @@
         'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_daily_top_reposters_task',
         'schedule': 900
     },
-    'send-reports-to-meme-voting': {
-        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.send_reports_to_meme_voting_task',
-        'schedule': 3600
-    },
-    'check-meme-template-potential-votes': {
-        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.check_meme_template_potential_votes_task',
-        'schedule': 1800
-    },
+    # 'send-reports-to-meme-voting': {
+    #     'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.send_reports_to_meme_voting_task',
+    #     'schedule': 3600
+    # },
+    # 'check-meme-template-potential-votes': {
+    #     'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.check_meme_template_potential_votes_task',
+    #     'schedule': 1800
+    # },
     'monitored-sub-config-update': {
         'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.queue_config_updates_task',
         'schedule': 3600
     },
-    'update-profile-token': {
-        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_profile_token_task',
-        'schedule': 120
-    },
+    # 'update-profile-token': {
+    #     'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_profile_token_task',
+    #     'schedule': 120
+    # },
     'update-daily-stats': {
         'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.update_daily_stats',
         'schedule': 86400
     },
+    'search-history-cleanup': {
+        'task': 'redditrepostsleuth.core.celery.tasks.scheduled_tasks.queue_search_history_cleanup',
+        'schedule': 3600
+    },
 
 }