Implement statistics feature

.github/workflows/build-image-bot.yml

@@ -0,0 +1,76 @@
+name: Build and Push Bot Docker Image
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'
+    paths:
+      - 'Dockerfile_bot'
+      - 'src/bot/**'
+      - 'requirements-bot.txt'
+  pull_request:
+    paths:
+      - 'Dockerfile_bot'
+      - 'src/bot/**'
+      - 'requirements-bot.txt'
+
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+
+    steps:
+
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0 
+
+    - name: Get the latest Git tag
+      id: get_tag
+      run: |
+        TAG=$(git describe --tags --abbrev=0)
+        echo "Latest tag is $TAG"
+        echo "LATEST_TAG=$TAG" >> $GITHUB_ENV
+
+    - name: Set short git commit SHA
+      id: get_sha
+      run: |
+        SHORT_SHA=$(git rev-parse --short ${{ github.sha }})
+        echo "Short commit SHA is $SHORT_SHA"
+        echo "SHORT_SHA=$SHORT_SHA" >> $GITHUB_ENV
+
+    - name: Confirm git commit SHA and tag variables
+      run: |
+        if [ -z "${{ env.SHORT_SHA }}" ] || [ -z "${{ env.LATEST_TAG }}" ]; then
+          echo "Error: One of SHORT_SHA, LATEST_TAG is empty"
+          exit 1
+        fi
+
+    # Setup Docker Buildx
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    # Login to Docker Hub
+    - name: Login to DockerHub
+      if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    # Build and push bot image
+    - name: Build and push bot Docker image
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        file: ./Dockerfile_bot
+        push: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) }}
+        tags: |
+          olegeech/mvcr-application-checker:bot-latest
+          olegeech/mvcr-application-checker:bot-${{ env.LATEST_TAG }}
+        build-args: |
+          BASE_VERSION=${{ env.LATEST_TAG }}
+          GIT_COMMIT=${{ env.SHORT_SHA }}

.github/workflows/build-images.yml → .github/workflows/build-image-fetcher.yml

@@ -1,4 +1,4 @@
-name: Build and Push Docker Images
+name: Build and Push Fetcher Docker Image
 
 on:
   push:
@@ -7,12 +7,16 @@ on:
     tags:
       - '*'
     paths:
-      - 'Dockerfile_bot'
       - 'Dockerfile_fetcher'
-      - 'src/**'
-      - 'requirements-bot.txt'
+      - 'src/fetcher/**'
+      - 'requirements-fetcher.txt'
+  pull_request:
+    paths:
+      - 'Dockerfile_fetcher'
+      - 'src/fetcher/**'
       - 'requirements-fetcher.txt'
 
+
 jobs:
   docker:
     runs-on: ubuntu-latest
@@ -51,32 +55,19 @@ jobs:
 
     # Login to Docker Hub
     - name: Login to DockerHub
+      if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
       uses: docker/login-action@v1
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-    # Build and push bot image
-    - name: Build and push bot Docker image
-      uses: docker/build-push-action@v2
-      with:
-        context: .
-        file: ./Dockerfile_bot
-        push: true
-        tags: |
-          olegeech/mvcr-application-checker:bot-latest
-          olegeech/mvcr-application-checker:bot-${{ env.LATEST_TAG }}
-        build-args: |
-          BASE_VERSION=${{ env.LATEST_TAG }}
-          GIT_COMMIT=${{ env.SHORT_SHA }}
-
     # Build and push fetcher image
     - name: Build and push fetcher Docker image
       uses: docker/build-push-action@v2
       with:
         context: .
         file: ./Dockerfile_fetcher
-        push: true
+        push: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) }}
         tags: |
           olegeech/mvcr-application-checker:fetcher-latest
           olegeech/mvcr-application-checker:fetcher-${{ env.LATEST_TAG }}

.vscode/settings.json

@@ -2,5 +2,6 @@
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"
   },
-  "python.formatting.provider": "none"
+  "python.testing.unittestEnabled": false,
+  "python.testing.pytestEnabled": true
 }

bot.sample.env

@@ -16,10 +16,14 @@ RABBIT_USER="bunny_admin"
 RABBIT_PASSWORD="password"
 
 # Time in seconds before an application request can be requeued
-REQUEUE_THRESHOLD_SECONDS = 3600
+REQUEUE_THRESHOLD_SECONDS=3600
 
 # Application monitor config
 REFRESH_PERIOD=3600
 SCHEDULER_PERIOD=300
 NOT_FOUND_MAX_DAYS=30
-NOT_FOUND_REFRESH_PERIOD=86400
+NOT_FOUND_REFRESH_PERIOD=86400
+
+# Statistics config
+STATISTICS_PERIOD_DAYS=60
+STATISTICS_MIN_TRESHOLD_DAYS=1

src/bot/__main__.py

@@ -8,7 +8,7 @@
 from bot.handlers import start_command, help_command, unknown_text, unknown_command, status_command
 from bot.handlers import unsubscribe_command, subscribe_command, admin_stats_command, fetcher_stats_command
 from bot.handlers import force_refresh_command, subscribe_button, lang_command, set_language_startup, set_language_cmd
-from bot.handlers import status_button, unsubscribe_button, force_refresh_button
+from bot.handlers import stats_command, status_button, unsubscribe_button, force_refresh_button
 from bot.handlers import (
     application_dialog_number,
     application_dialog_year,
@@ -93,6 +93,7 @@ async def main():
     bot.add_handler(CommandHandler("fetcher_stats", fetcher_stats_command, has_args=False))
     bot.add_handler(CommandHandler("lang", lang_command, has_args=False))
     bot.add_handler(CallbackQueryHandler(set_language_cmd, pattern="set_lang_cmd_*"))
+    bot.add_handler(CommandHandler("stats", stats_command))
     bot.add_handler(CommandHandler("help", help_command, has_args=False))
     # Define conversatinal handler for user-friendly application dialog
     conv_handler = ConversationHandler(

src/bot/database.py

@@ -96,8 +96,15 @@ async def insert_application(
         return True
 
     async def update_application_status(
-        self, chat_id, application_number, application_type, application_year,
-        current_status, is_resolved, application_state, has_changed
+        self,
+        chat_id,
+        application_number,
+        application_type,
+        application_year,
+        current_status,
+        is_resolved,
+        application_state,
+        has_changed,
     ):
         """Update status, is_resolved, changed_at, and state for a specific application"""
 
@@ -118,9 +125,7 @@ async def update_application_status(
         changed_at_clause = ", changed_at = CURRENT_TIMESTAMP" if has_changed else ""
         query = base_query.format(changed_at_clause=changed_at_clause)
 
-        params = (
-            current_status, is_resolved, application_state, chat_id, application_number, application_type, application_year
-        )
+        params = (current_status, is_resolved, application_state, chat_id, application_number, application_type, application_year)
         async with self.pool.acquire() as conn:
             try:
                 await conn.execute(query, *params)
@@ -274,7 +279,6 @@ async def fetch_applications_needing_update(self, refresh_period, not_found_refr
                 logger.error(f"Error while fetching applications needing update from DB: {e}")
                 return []
 
-
     async def fetch_applications_to_expire(self, not_found_max_age):
         """Fetch applications in NOT_FOUND state exceeding the max age"""
         not_found_seconds = not_found_max_age.total_seconds()
@@ -553,7 +557,7 @@ async def count_all_subscriptions(self, active_only=False):
         query = "SELECT COUNT(*) FROM Applications"
         if active_only:
             query += " WHERE is_resolved = FALSE"
-        
+
         async with self.pool.acquire() as conn:
             try:
                 count = await conn.fetchval(query)
@@ -562,6 +566,52 @@ async def count_all_subscriptions(self, active_only=False):
                 logger.error(f"Error while fetching total {'active ' if active_only else ''}subscriptions count. Error: {e}")
                 return None
 
+    async def fetch_application_states_within_period(self, start_date, end_date):
+        """Fetch application states within a certain period"""
+        query = """
+            SELECT application_state
+            FROM Applications
+            WHERE created_at BETWEEN $1 AND $2
+        """
+        async with self.pool.acquire() as conn:
+            try:
+                rows = await conn.fetch(query, start_date, end_date)
+                return [dict(row) for row in rows]
+            except Exception as e:
+                logger.error(f"Error fetching applications within period from DB: {e}")
+                return []
+
+    async def fetch_processing_times_within_period(self, start_date, end_date):
+        """Calculates time spent in IN_PROGRESS state for each type of applications within the period"""
+        query = """
+            SELECT application_type, EXTRACT(EPOCH FROM (changed_at - created_at)) AS processing_time
+            FROM Applications
+            WHERE application_state IN ('APPROVED', 'DENIED')
+              AND changed_at IS NOT NULL
+              AND created_at BETWEEN $1 AND $2
+        """
+        async with self.pool.acquire() as conn:
+            try:
+                rows = await conn.fetch(query, start_date, end_date)
+                return [dict(row) for row in rows]
+            except Exception as e:
+                logger.error(f"Error fetching processing times from DB: {e}")
+                return []
+
+    async def fetch_status_change_hours_within_period(self, start_date, end_date):
+        """Get hours when application change occurs within the period"""
+        query = """
+            SELECT EXTRACT(HOUR FROM changed_at AT TIME ZONE 'Europe/Prague') AS hour
+            FROM Applications
+            WHERE changed_at IS NOT NULL AND changed_at BETWEEN $1 AND $2
+        """
+        async with self.pool.acquire() as conn:
+            try:
+                rows = await conn.fetch(query, start_date, end_date)
+                return [row["hour"] for row in rows]
+            except Exception as e:
+                logger.error(f"Error fetching status change times from DB: {e}")
+                return []
 
     async def close(self):
         logger.info("Shutting down DB connection")

src/bot/handlers.py

@@ -5,14 +5,15 @@
 
 from telegram import Update, InlineKeyboardMarkup, InlineKeyboardButton, BotCommand, BotCommandScopeChat, ForceReply
 from telegram.ext import ContextTypes, ConversationHandler
-from bot.loader import loader, ADMIN_CHAT_IDS, REFRESH_PERIOD, FULL_VERSION
+from bot.loader import loader, ADMIN_CHAT_IDS, REFRESH_PERIOD, FULL_VERSION, STATISTICS_PERIOD_DAYS
 from bot.texts import button_texts, message_texts, commands_description
 from bot.utils import generate_oam_full_string
+from bot.statistics import Statistics
 
 SUBSCRIPTIONS_LIMIT = 5
 BUTTON_WAIT_SECONDS = 1
 FORCE_FETCH_LIMIT_SECONDS = 86400
-COMMANDS_LIST = ["status", "subscribe", "unsubscribe", "force_refresh", "lang", "start", "help", "reminder"]
+COMMANDS_LIST = ["status", "subscribe", "unsubscribe", "force_refresh", "lang", "start", "help", "reminder", "stats"]
 ADMIN_COMMANDS = ["admin_stats", "fetcher_stats", "admin_broadcast"]
 DEFAULT_LANGUAGE = "EN"
 LANGUAGE_LIST = ["EN 🏴󠁧󠁢󠁥󠁮󠁧󠁿|🇺🇸", "RU 🇷🇺", "CZ 🇨🇿", "UA 🇺🇦"]
@@ -31,6 +32,7 @@
 # Get instances of database and rabbitmq (lazy init)
 db = loader.db
 rabbit = loader.rabbit
+stats = Statistics(db)
 
 def get_allowed_years():
     """Compute allowed years to select"""
@@ -1013,6 +1015,70 @@ async def add_reminder(update: Update, context: ContextTypes.DEFAULT_TYPE):
     return ConversationHandler.END
 
 
+# Handler for /stats command
+async def stats_command(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    """Provides application statistics to the user."""
+    logger.info(f"💻 Received /stats command from {user_info(update)}")
+    chat_id = update.effective_chat.id
+    lang = await _get_user_language(update, context)
+
+    # Parse arguments for period_days (e.g. /stats 30)
+    args = context.args
+    if args and args[0].isdigit():
+        period_days = int(args[0])
+        # enforce maximum and minimum limits [ 1 - 360]
+        period_days = max(1, min(period_days, 360))
+    else:
+        period_days = STATISTICS_PERIOD_DAYS
+
+    # Fetch statistics
+    try:
+        general_stats = await stats.get_general_stats(period_days)
+        overall_avg, avg_times_by_category = await stats.calculate_average_processing_times(period_days)
+        common_hour = await stats.get_common_update_time(period_days)
+        predictions = await stats.predict_user_application_time(chat_id, period_days)
+    except Exception as e:
+        logger.error(f"Error fetching statistics: {e}")
+        await update.message.reply_text(message_texts[lang]["error_fetching_stats"])
+        return
+
+    # Prepare the response message
+    message_lines = [
+        message_texts[lang]['stats_header'].format(days=period_days),
+        message_texts[lang]['stats_total_applications'].format(count=general_stats['total_applications']),
+        message_texts[lang]['stats_approved'].format(count=general_stats['approved']),
+        message_texts[lang]['stats_denied'].format(count=general_stats['denied']),
+    ]
+
+    if overall_avg:
+        message_lines.append(
+            message_texts[lang]['average_processing_time'].format(days=round(overall_avg / 86400, 2))
+        )
+
+    if avg_times_by_category:
+        message_lines.append(message_texts[lang]['average_by_category'])
+        for app_type, avg_time in avg_times_by_category.items():
+            message_lines.append(f"- {app_type}: {round(avg_time / 86400, 2)} days")
+
+    if common_hour is not None:
+        message_lines.append(message_texts[lang]['most_common_update_time'].format(hour=common_hour))
+
+    if predictions:
+        message_lines.append(message_texts[lang]['your_estimated_times'])
+        for prediction in predictions:
+            message_lines.append(
+                message_texts[lang]['predicted_approval_time'].format(
+                    application_number=prediction['application_number'],
+                    days_remaining=round(prediction['days_remaining'], 2)
+                )
+            )
+    else:
+        message_lines.append(message_texts[lang]['no_predictions'])
+
+    message = '\n'.join(message_lines)
+    await update.message.reply_text(message)
+
+
 # Handler for /lang command
 async def lang_command(update: Update, context: ContextTypes.DEFAULT_TYPE):
     """Handler for /lang command to set language preference"""

src/bot/loader.py

@@ -33,6 +33,9 @@
 SCHEDULER_PERIOD = int(os.getenv("SCHEDULER_PERIOD", 300))
 NOT_FOUND_MAX_DAYS = int(os.getenv("NOT_FOUND_MAX_DAYS", 30))
 NOT_FOUND_REFRESH_PERIOD = int(os.getenv("NOT_FOUND_REFRESH_PERIOD", 86400))
+# Statistics module config
+STATISTICS_PERIOD_DAYS = int(os.getenv("STATISTICS_PERIOD_DAYS", 60))
+STATISTICS_MIN_TRESHOLD_DAYS = int(os.getenv("STATISTICS_MIN_TRESHOLD_DAYS", 1))
 # Run mode for tests
 RUN_MODE = os.getenv("RUN_MODE", "PROD")