From 6995e85012cca6eca44f956cac260553cd7a2ae0 Mon Sep 17 00:00:00 2001
From: Seth <seth@powerloom.io>
Date: Wed, 31 Jul 2024 13:33:32 -0400
Subject: [PATCH 01/13] chore: update config for telegram reporting

---
 docker-compose-dev.yaml |  2 ++
 docker-compose.yaml     |  2 ++
 env.example             |  2 ++
 snapshotter_autofill.sh | 18 ++++++++++++++++--
 4 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml
index b04708e..03c60c1 100755
--- a/docker-compose-dev.yaml
+++ b/docker-compose-dev.yaml
@@ -55,6 +55,8 @@ services:
       - POWERLOOM_REPORTING_URL=$POWERLOOM_REPORTING_URL
       - WEB3_STORAGE_TOKEN=$WEB3_STORAGE_TOKEN
       - NAMESPACE=$NAMESPACE
+      - TELEGRAM_REPORTING_URL=$TELEGRAM_REPORTING_URL
+      - TELEGRAM_CHAT_ID=$TELEGRAM_CHAT_ID
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8002/health"]
       interval: 10s
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 55f1b5a..36b04f7 100755
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -53,6 +53,8 @@ services:
       - POWERLOOM_REPORTING_URL=$POWERLOOM_REPORTING_URL
       - WEB3_STORAGE_TOKEN=$WEB3_STORAGE_TOKEN
       - NAMESPACE=$NAMESPACE
+      - TELEGRAM_REPORTING_URL=$TELEGRAM_REPORTING_URL
+      - TELEGRAM_CHAT_ID=$TELEGRAM_CHAT_ID
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8002/health"]
       interval: 10s
diff --git a/env.example b/env.example
index 572acd5..2ab017d 100755
--- a/env.example
+++ b/env.example
@@ -21,3 +21,5 @@ IPFS_API_SECRET=
 SLACK_REPORTING_URL=
 WEB3_STORAGE_TOKEN=
 DASHBOARD_ENABLED=
+TELEGRAM_REPORTING_URL=
+TELEGRAM_CHAT_ID=
diff --git a/snapshotter_autofill.sh b/snapshotter_autofill.sh
index 8b44d7f..faff705 100755
--- a/snapshotter_autofill.sh
+++ b/snapshotter_autofill.sh
@@ -50,7 +50,15 @@ if [ "$SLACK_REPORTING_URL" ]; then
 fi
 
 if [ "$POWERLOOM_REPORTING_URL" ]; then
-    echo "Found SLACK_REPORTING_URL ${POWERLOOM_REPORTING_URL}";
+    echo "Found POWERLOOM_REPORTING_URL ${POWERLOOM_REPORTING_URL}";
+fi
+
+if [ "$TELEGRAM_REPORTING_URL" ]; then
+    echo "Found TELEGRAM_REPORTING_URL ${TELEGRAM_REPORTING_URL}";
+fi
+
+if [ "$TELEGRAM_CHAT_ID" ]; then
+    echo "Found TELEGRAM_CHAT_ID ${TELEGRAM_CHAT_ID}";
 fi
 
 if [ "$WEB3_STORAGE_TOKEN" ]; then
@@ -74,7 +82,8 @@ export relayer_host="${RELAYER_HOST:-https://relayer-nms-testnet-public.powerloo
 export local_collector_port="${LOCAL_COLLECTOR_PORT:-50051}"
 export slack_reporting_url="${SLACK_REPORTING_URL:-}"
 export powerloom_reporting_url="${POWERLOOM_REPORTING_URL:-}"
-
+export telegram_reporting_url="${TELEGRAM_REPORTING_URL:-}"
+export telegram_chat_id="${TELEGRAM_CHAT_ID:-}"
 
 
 # If IPFS_URL is empty, clear IPFS API key and secret
@@ -92,6 +101,8 @@ echo "Using slack reporting url: ${slack_reporting_url}"
 echo "Using powerloom reporting url: ${powerloom_reporting_url}"
 echo "Using web3 storage token: ${web3_storage_token}"
 echo "Using relayer host: ${relayer_host}"
+echo "Using telegram reporting url: ${telegram_reporting_url}"
+echo "Using telegram chat id: ${telegram_chat_id}
 
 sed -i'.backup' "s#relevant-namespace#$namespace#" config/settings.json
 
@@ -122,4 +133,7 @@ sed -i'.backup' "s#https://relayer-url#$relayer_host#" config/settings.json
 
 sed -i'.backup' "s#local-collector-port#$local_collector_port#" config/settings.json
 
+sed -i'.backup' "s#https://telegram-reporting-url#$telegram_reporting_url#" config/settings.json
+sed -i'.backup' "s#telegram-chat-id#$telegram_chat_id#" config/settings.json
+
 echo 'settings has been populated!'

From ce36d4e4cd96a85f468c4ef8d63f2b577b8cf8d8 Mon Sep 17 00:00:00 2001
From: Seth <seth@elklabs.org>
Date: Wed, 6 Mar 2024 23:11:56 -0500
Subject: [PATCH 02/13] chore: update models for tg health monitoring

---
 snapshotter/utils/models/data_models.py    | 27 ++++++++++++++++++++++
 snapshotter/utils/models/settings_model.py |  3 +++
 2 files changed, 30 insertions(+)

diff --git a/snapshotter/utils/models/data_models.py b/snapshotter/utils/models/data_models.py
index 50d681a..0ea6e5d 100644
--- a/snapshotter/utils/models/data_models.py
+++ b/snapshotter/utils/models/data_models.py
@@ -134,6 +134,7 @@ class SnapshotterStatus(BaseModel):
     totalSuccessfulSubmissions: int = 0
     totalIncorrectSubmissions: int = 0
     totalMissedSubmissions: int = 0
+    consecutiveMissedSubmissions: int = 0
     projects: List[ProjectStatus]
 
 
@@ -186,3 +187,29 @@ class UnfinalizedSnapshot(BaseModel):
 class TaskStatusRequest(BaseModel):
     task_type: str
     wallet_address: str
+
+
+class SnapshotterReportData(BaseModel):
+    snapshotterIssue: SnapshotterIssue
+    snapshotterStatus: SnapshotterStatus
+
+
+class TelegramSnapshotterReportMessage(BaseModel):
+    chatId: int
+    slotId: int
+    issue: SnapshotterIssue
+    status: SnapshotterStatus
+
+
+class EpochProcessingIssue(BaseModel):
+    instanceID: str
+    issueType: str
+    timeOfReporting: str
+    extra: Optional[str] = ''
+
+
+class TelegramEpochProcessingReportMessage(BaseModel):
+    chatId: int
+    slotId: int
+    issue: EpochProcessingIssue
+
diff --git a/snapshotter/utils/models/settings_model.py b/snapshotter/utils/models/settings_model.py
index 163545b..e496db4 100644
--- a/snapshotter/utils/models/settings_model.py
+++ b/snapshotter/utils/models/settings_model.py
@@ -47,6 +47,9 @@ class Timeouts(BaseModel):
 class ReportingConfig(BaseModel):
     slack_url: str
     service_url: str
+    telegram_url: str
+    telegram_chat_id: str
+    failure_report_frequency: int
 
 
 class Logs(BaseModel):

From 1d139e3e4dce5de327e6f0d07af58547b3dad473 Mon Sep 17 00:00:00 2001
From: Seth <seth@elklabs.org>
Date: Wed, 6 Mar 2024 23:12:55 -0500
Subject: [PATCH 03/13] chore: add telegram reporting to callback helpers

---
 .../tests/test_tg_reporting_service.py        |  63 +++++++++
 snapshotter/utils/callback_helpers.py         | 120 ++++++++++++++++--
 2 files changed, 174 insertions(+), 9 deletions(-)
 create mode 100644 snapshotter/tests/test_tg_reporting_service.py

diff --git a/snapshotter/tests/test_tg_reporting_service.py b/snapshotter/tests/test_tg_reporting_service.py
new file mode 100644
index 0000000..36d1dc4
--- /dev/null
+++ b/snapshotter/tests/test_tg_reporting_service.py
@@ -0,0 +1,63 @@
+import time
+import json
+import asyncio
+from httpx import AsyncClient
+from httpx import AsyncHTTPTransport
+from httpx import Limits
+from httpx import Timeout
+from snapshotter.utils.callback_helpers import send_failure_notifications_async
+from snapshotter.utils.models.data_models import SnapshotterReportData 
+from snapshotter.utils.models.data_models import SnapshotterIssue
+from snapshotter.utils.models.data_models import SnapshotterReportState
+from snapshotter.utils.models.data_models import SnapshotterStatus
+from snapshotter.utils.default_logger import logger
+from snapshotter.settings.config import settings
+
+
+
+# ensure telegram__url / telegram_chat_id are set in config/settings.json
+# telegram_url endpoint needs to be active
+async def test_tg_reporting_call():
+
+    project_id = 'test_project_id'
+    epoch_id = 0
+    
+    async_client = AsyncClient(
+        timeout=Timeout(timeout=5.0),
+        follow_redirects=False,
+        transport=AsyncHTTPTransport(
+            limits=Limits(
+                max_connections=200,
+                max_keepalive_connections=50,
+                keepalive_expiry=None,
+            ),
+        ),
+    )
+
+    notification_message = SnapshotterReportData(
+        snapshotterIssue=SnapshotterIssue(
+            instanceID=settings.instance_id,
+            issueType=SnapshotterReportState.MISSED_SNAPSHOT.value,
+            projectID=project_id,
+            epochId=epoch_id,
+            timeOfReporting=str(time.time()),
+            extra=json.dumps({'issueDetails': f'Error : TEST ERROR MESSAGE'}),
+        ),
+        snapshotterStatus=SnapshotterStatus(
+            projects=[],
+        ),
+    )
+
+    await send_failure_notifications_async(
+        client=async_client, message=notification_message,
+    )
+
+    # wait for the callback to complete
+    await asyncio.sleep(5)
+
+
+if __name__ == '__main__':
+    try:
+        asyncio.get_event_loop().run_until_complete(test_tg_reporting_call())
+    except Exception as e:
+        logger.opt(exception=True).error('exception: {}', e)
\ No newline at end of file
diff --git a/snapshotter/utils/callback_helpers.py b/snapshotter/utils/callback_helpers.py
index 1050421..2df15ec 100644
--- a/snapshotter/utils/callback_helpers.py
+++ b/snapshotter/utils/callback_helpers.py
@@ -11,7 +11,12 @@
 from pydantic import BaseModel
 
 from snapshotter.settings.config import settings
+from snapshotter.settings.config import project_types
 from snapshotter.utils.default_logger import logger
+from snapshotter.utils.models.data_models import TelegramEpochProcessingReportMessage
+from snapshotter.utils.models.data_models import TelegramSnapshotterReportMessage
+from snapshotter.utils.models.data_models import SnapshotterReportData
+from snapshotter.utils.models.data_models import EpochProcessingIssue
 from snapshotter.utils.models.message_models import EpochBase
 from snapshotter.utils.models.message_models import SnapshotProcessMessage
 from snapshotter.utils.rpc import RpcHelper
@@ -66,22 +71,23 @@ def sync_notification_callback_result_handler(f: functools.partial):
         logger.debug('Callback or notification result:{}', result)
 
 
-async def send_failure_notifications_async(client: AsyncClient, message: BaseModel):
+async def send_failure_notifications_async(client: AsyncClient, message: SnapshotterReportData):
     """
     Sends failure notifications to the configured reporting services.
 
     Args:
         client (AsyncClient): The async HTTP client to use for sending notifications.
-        message (BaseModel): The message to send as notification.
+        message (SnapshotterReportData): The message to send as notification.
 
     Returns:
         None
     """
+    
     if settings.reporting.service_url:
         f = asyncio.ensure_future(
             client.post(
                 url=urljoin(settings.reporting.service_url, '/reportIssue'),
-                json=message.dict(),
+                json=message.snapshotterIssue.dict(),
             ),
         )
         f.add_done_callback(misc_notification_callback_result_handler)
@@ -90,19 +96,41 @@ async def send_failure_notifications_async(client: AsyncClient, message: BaseMod
         f = asyncio.ensure_future(
             client.post(
                 url=settings.reporting.slack_url,
-                json=message.dict(),
+                json=message.snapshotterIssue.dict(),
             ),
         )
         f.add_done_callback(misc_notification_callback_result_handler)
 
+    if settings.reporting.telegram_url and settings.reporting.telegram_chat_id:
+        reporting_message = TelegramSnapshotterReportMessage(
+            chatId=settings.reporting.telegram_chat_id,
+            slotId=settings.slot_id,
+            issue=message.snapshotterIssue,
+            status=message.snapshotterStatus,
+        )
+        report_frequency = settings.reporting.failure_report_frequency
+        consecutive_misses = reporting_message.status.consecutiveMissedSubmissions
+        # Need to account for # of projects to avoid skipping reports
+        if (
+            consecutive_misses <= 1 or 
+            (consecutive_misses // len(project_types)) % report_frequency == 0
+        ):
+            f = asyncio.ensure_future(
+                client.post(
+                    url=urljoin(settings.reporting.telegram_url, '/reportSnapshotIssue'),
+                    json=reporting_message.dict(),
+                ),
+            )
+            f.add_done_callback(misc_notification_callback_result_handler)
+
 
-def send_failure_notifications_sync(client: SyncClient, message: BaseModel):
+def send_failure_notifications_sync(client: SyncClient, message: SnapshotterReportData):
     """
-    Sends failure notifications synchronously to the reporting service and/or Slack.
+    Sends failure notifications synchronously to to the configured reporting services.
 
     Args:
         client (SyncClient): The HTTP client to use for sending notifications.
-        message (BaseModel): The message to send as notification.
+        message (SnapshotterReportData): The message to send as notification.
 
     Returns:
         None
@@ -111,7 +139,7 @@ def send_failure_notifications_sync(client: SyncClient, message: BaseModel):
         f = functools.partial(
             client.post,
             url=urljoin(settings.reporting.service_url, '/reportIssue'),
-            json=message.dict(),
+            json=message.snapshotterIssue.dict(),
         )
         sync_notification_callback_result_handler(f)
 
@@ -119,8 +147,82 @@ def send_failure_notifications_sync(client: SyncClient, message: BaseModel):
         f = functools.partial(
             client.post,
             url=settings.reporting.slack_url,
-            json=message.dict(),
+            json=message.snapshotterIssue.dict(),
+        )
+        sync_notification_callback_result_handler(f)
+
+    if settings.reporting.telegram_url and settings.reporting.telegram_chat_id:
+        reporting_message = TelegramSnapshotterReportMessage(
+            chatId=settings.reporting.telegram_chat_id,
+            slotId=settings.slot_id,
+            issue=message.snapshotterIssue,
+            status=message.snapshotterStatus,
+        )
+        report_frequency = settings.reporting.failure_report_frequency
+        consecutive_misses = reporting_message.status.consecutiveMissedSubmissions
+
+        if (
+            consecutive_misses <= 1 or 
+            (consecutive_misses // len(project_types)) % report_frequency == 0
+        ):
+            f = functools.partial(
+                client.post,
+                url=urljoin(settings.reporting.telegram_url, '/reportSnapshotIssue'),
+                json=reporting_message.dict(),
+            )
+            sync_notification_callback_result_handler(f)
+
+
+async def send_epoch_processing_failure_notification_async(client: AsyncClient, message: EpochProcessingIssue):
+    """
+    Sends epoch processing failure notifications synchronously to the telegarm reporting service.
+
+    Args:
+        client (SyncClient): The HTTP client to use for sending notifications.
+        message (EpochProcessingIssue): The message to send as notification.
+
+    Returns:
+        None
+    """
+    if settings.reporting.telegram_url and settings.reporting.telegram_chat_id:
+        reporting_message = TelegramEpochProcessingReportMessage(
+            chatId=settings.reporting.telegram_chat_id,
+            slotId=settings.slot_id,
+            issue=message,
         )
+
+        f = asyncio.ensure_future(
+                client.post(
+                    url=urljoin(settings.reporting.telegram_url, '/reportEpochProcessingIssue'),
+                    json=reporting_message.dict(),
+                ),
+            )
+        f.add_done_callback(misc_notification_callback_result_handler)
+
+
+def send_epoch_processing_failure_notification_sync(client: SyncClient, message: EpochProcessingIssue):
+    """
+    Sends epoch processing failure notifications synchronously to the telegarm reporting service.
+
+    Args:
+        client (SyncClient): The HTTP client to use for sending notifications.
+        message (EpochProcessingIssue): The message to send as notification.
+
+    Returns:
+        None
+    """
+    if settings.reporting.telegram_url and settings.reporting.telegram_chat_id:
+        reporting_message = TelegramEpochProcessingReportMessage(
+            chatId=settings.reporting.telegram_chat_id,
+            slotId=settings.slot_id,
+            issue=message,
+        )
+
+        f = functools.partial(
+                client.post,
+                url=urljoin(settings.reporting.telegram_url, '/reportEpochProcessingIssue'),
+                json=reporting_message.dict(),
+            )
         sync_notification_callback_result_handler(f)
 
 

From b677e4511f0b9f7b1ae18b0012927dc8a3357de8 Mon Sep 17 00:00:00 2001
From: Seth <seth@elklabs.org>
Date: Wed, 6 Mar 2024 23:16:19 -0500
Subject: [PATCH 04/13] add: snapshot/epoch telegram reporting to workers

---
 snapshotter/system_event_detector.py | 32 +++++++++++++++++++
 snapshotter/utils/generic_worker.py  | 46 +++++++++++++++++++---------
 snapshotter/utils/snapshot_worker.py | 21 ++++++++-----
 3 files changed, 78 insertions(+), 21 deletions(-)

diff --git a/snapshotter/system_event_detector.py b/snapshotter/system_event_detector.py
index c7323c3..3982a55 100644
--- a/snapshotter/system_event_detector.py
+++ b/snapshotter/system_event_detector.py
@@ -1,3 +1,4 @@
+import json
 import asyncio
 import multiprocessing
 import resource
@@ -12,12 +13,15 @@
 import sys
 from snapshotter.processor_distributor import ProcessorDistributor
 from snapshotter.settings.config import settings
+from snapshotter.utils.callback_helpers import send_epoch_processing_failure_notification_sync
 from snapshotter.utils.default_logger import logger
 from snapshotter.utils.exceptions import GenericExitOnSignal
 from snapshotter.utils.file_utils import read_json_file
 from snapshotter.utils.models.data_models import DailyTaskCompletedEvent
 from snapshotter.utils.models.data_models import DayStartedEvent
 from snapshotter.utils.models.data_models import EpochReleasedEvent
+from snapshotter.utils.models.data_models import EpochProcessingIssue
+from snapshotter.utils.models.data_models import SnapshotterReportState
 from snapshotter.utils.rpc import get_event_sig_and_abi
 from snapshotter.utils.rpc import RpcHelper
 from urllib.parse import urljoin
@@ -76,6 +80,7 @@ def __init__(self, name, **kwargs):
             abi=self.contract_abi,
         )
         self._last_reporting_service_ping = 0
+        self._last_reporting_message_sent = 0
 
         # event EpochReleased(uint256 indexed epochId, uint256 begin, uint256 end, uint256 timestamp);
         # event DayStartedEvent(uint256 dayId, uint256 timestamp);
@@ -249,6 +254,19 @@ async def _detect_events(self):
                     settings.rpc.polling_interval,
                 )
 
+                if int(time.time()) - self._last_reporting_message_sent >= 600:
+                    self._last_reporting_message_sent = int(time.time())
+                    notification_message = EpochProcessingIssue(
+                        instanceID=settings.instance_id,
+                        issueType=SnapshotterReportState.UNHEALTHY_EPOCH_PROCESSING.value,
+                        timeOfReporting=str(time.time()),
+                        extra=json.dumps({'issueDetails': f'Error : {e}'})
+                    )
+                    send_epoch_processing_failure_notification_sync(
+                        client=self._httpx_client, 
+                        message=notification_message
+                    )
+
                 await asyncio.sleep(settings.rpc.polling_interval)
                 continue
 
@@ -284,6 +302,20 @@ async def _detect_events(self):
                     e,
                     settings.rpc.polling_interval,
                 )
+
+                if int(time.time()) - self._last_reporting_message_sent >= 600:
+                    self._last_reporting_message_sent = int(time.time())
+                    notification_message = EpochProcessingIssue(
+                        instanceID=settings.instance_id,
+                        issueType=SnapshotterReportState.UNHEALTHY_EPOCH_PROCESSING.value,
+                        timeOfReporting=str(time.time()),
+                        extra=json.dumps({'issueDetails': f'Error : {e}'})
+                    )
+                    send_epoch_processing_failure_notification_sync(
+                        client=self._httpx_client, 
+                        message=notification_message
+                    )
+
                 await asyncio.sleep(settings.rpc.polling_interval)
                 continue
 
diff --git a/snapshotter/utils/generic_worker.py b/snapshotter/utils/generic_worker.py
index f9563a3..e989553 100644
--- a/snapshotter/utils/generic_worker.py
+++ b/snapshotter/utils/generic_worker.py
@@ -37,7 +37,9 @@
 from snapshotter.utils.default_logger import logger
 from snapshotter.utils.file_utils import read_json_file
 from snapshotter.utils.models.data_models import SnapshotterIssue
+from snapshotter.utils.models.data_models import SnapshotterReportData
 from snapshotter.utils.models.data_models import SnapshotterReportState
+from snapshotter.utils.models.data_models import SnapshotterStatus
 from snapshotter.utils.models.message_models import SnapshotProcessMessage
 from snapshotter.utils.models.message_models import SnapshotSubmittedMessage
 from snapshotter.utils.models.message_models import SnapshotSubmittedMessageLite
@@ -127,6 +129,7 @@ def __init__(self):
         self.protocol_state_contract_address = settings.protocol_state.address
         self.initialized = False
         self.logger = logger.bind(module='GenericAsyncWorker')
+        self._status = SnapshotterStatus(projects=[])
 
     def _notification_callback_result_handler(self, fut: asyncio.Future):
         """
@@ -373,13 +376,18 @@ async def _commit_payload(
                 'Exception uploading snapshot to IPFS for epoch {}: {}, Error: {},'
                 'sending failure notifications', epoch, snapshot, e,
             )
-            notification_message = SnapshotterIssue(
-                instanceID=settings.instance_id,
-                issueType=SnapshotterReportState.MISSED_SNAPSHOT.value,
-                projectID=project_id,
-                epochId=str(epoch.epochId),
-                timeOfReporting=str(time.time()),
-                extra=json.dumps({'issueDetails': f'Error : {e}'}),
+            self._status.totalMissedSubmissions += 1
+            self._status.consecutiveMissedSubmissions += 1
+            notification_message = SnapshotterReportData(
+                snapshotterIssue=SnapshotterIssue(
+                    instanceID=settings.instance_id,
+                    issueType=SnapshotterReportState.MISSED_SNAPSHOT.value,
+                    projectID=project_id,
+                    epochId=str(epoch.epochId),
+                    timeOfReporting=str(time.time()),
+                    extra=json.dumps({'issueDetails': f'Error : {e}'}),
+                ),
+                snapshotterStatus=self._status,
             )
             await send_failure_notifications_async(
                 client=self._client, message=notification_message,
@@ -393,17 +401,27 @@ async def _commit_payload(
                     'Exception submitting snapshot to collector for epoch {}: {}, Error: {},'
                     'sending failure notifications', epoch, snapshot, e,
                 )
-                notification_message = SnapshotterIssue(
-                    instanceID=settings.instance_id,
-                    issueType=SnapshotterReportState.MISSED_SNAPSHOT.value,
-                    projectID=project_id,
-                    epochId=str(epoch.epochId),
-                    timeOfReporting=str(time.time()),
-                    extra=json.dumps({'issueDetails': f'Error : {e}'}),
+                self._status.totalMissedSubmissions += 1
+                self._status.consecutiveMissedSubmissions += 1
+
+                notification_message = SnapshotterReportData(
+                    snapshotterIssue=SnapshotterIssue(
+                        instanceID=settings.instance_id,
+                        issueType=SnapshotterReportState.MISSED_SNAPSHOT.value,
+                        projectID=project_id,
+                        epochId=str(epoch.epochId),
+                        timeOfReporting=str(time.time()),
+                        extra=json.dumps({'issueDetails': f'Error : {e}'}),
+                    ),
+                    snapshotterStatus=self._status,
                 )
                 await send_failure_notifications_async(
                     client=self._client, message=notification_message,
                 )
+            else:
+                # reset consecutive missed snapshots counter
+                self._status.consecutiveMissedSubmissions = 0
+                self._status.totalSuccessfulSubmissions += 1
 
         # upload to web3 storage
         if storage_flag:
diff --git a/snapshotter/utils/snapshot_worker.py b/snapshotter/utils/snapshot_worker.py
index 1675049..8cffa43 100644
--- a/snapshotter/utils/snapshot_worker.py
+++ b/snapshotter/utils/snapshot_worker.py
@@ -14,6 +14,7 @@
 from snapshotter.utils.generic_worker import GenericAsyncWorker
 from snapshotter.utils.models.data_models import SnapshotterIssue
 from snapshotter.utils.models.data_models import SnapshotterReportState
+from snapshotter.utils.models.data_models import SnapshotterReportData
 from snapshotter.utils.models.message_models import SnapshotProcessMessage
 
 
@@ -97,13 +98,19 @@ async def _process(self, msg_obj: SnapshotProcessMessage, task_type: str, eth_pr
                 'sending failure notifications', msg_obj, e,
             )
 
-            notification_message = SnapshotterIssue(
-                instanceID=settings.instance_id,
-                issueType=SnapshotterReportState.MISSED_SNAPSHOT.value,
-                projectID=f'{task_type}:{settings.namespace}',
-                epochId=str(msg_obj.epochId),
-                timeOfReporting=str(time.time()),
-                extra=json.dumps({'issueDetails': f'Error : {e}'}),
+            self._status.totalMissedSubmissions += 1
+            self._status.consecutiveMissedSubmissions += 1
+
+            notification_message = SnapshotterReportData(
+                snapshotterIssue=SnapshotterIssue(
+                    instanceID=settings.instance_id,
+                    issueType=SnapshotterReportState.MISSED_SNAPSHOT.value,
+                    projectID=f'{task_type}:{settings.namespace}',
+                    epochId=str(msg_obj.epochId),
+                    timeOfReporting=str(time.time()),
+                    extra=json.dumps({'issueDetails': f'Error : {e}'}),
+                ),
+                snapshotterStatus=self._status,
             )
 
             await send_failure_notifications_async(

From 224ad9929375cab386d0492d916989b65b00076f Mon Sep 17 00:00:00 2001
From: Seth <seth@elklabs.org>
Date: Thu, 7 Mar 2024 18:04:56 -0500
Subject: [PATCH 05/13] add: reporting on get_eth_price_usd failure

---
 snapshotter/processor_distributor.py | 40 ++++++++++++++++++----------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/snapshotter/processor_distributor.py b/snapshotter/processor_distributor.py
index 5b59149..cb67ada 100644
--- a/snapshotter/processor_distributor.py
+++ b/snapshotter/processor_distributor.py
@@ -1,5 +1,6 @@
 import asyncio
 import json
+import time
 from collections import defaultdict
 from typing import Union
 
@@ -9,12 +10,10 @@
 from httpx import Limits
 from httpx import Timeout
 from web3 import Web3
-import time
 
-from snapshotter.utils.models.data_models import SnapshotterReportState
-from snapshotter.utils.models.data_models import SnapshotterIssue
 from snapshotter.settings.config import projects_config
 from snapshotter.settings.config import settings
+from snapshotter.utils.callback_helpers import send_failure_notifications_async
 from snapshotter.utils.data_utils import get_snapshot_submision_window
 from snapshotter.utils.data_utils import get_source_chain_epoch_size
 from snapshotter.utils.data_utils import get_source_chain_id
@@ -24,13 +23,17 @@
 from snapshotter.utils.models.data_models import DayStartedEvent
 from snapshotter.utils.models.data_models import EpochReleasedEvent
 from snapshotter.utils.models.data_models import SnapshotFinalizedEvent
+from snapshotter.utils.models.data_models import SnapshotterIssue
+from snapshotter.utils.models.data_models import SnapshotterReportData
+from snapshotter.utils.models.data_models import SnapshotterReportState
+from snapshotter.utils.models.data_models import SnapshotterStatus
 from snapshotter.utils.models.data_models import SnapshottersUpdatedEvent
 from snapshotter.utils.models.message_models import EpochBase
 from snapshotter.utils.models.message_models import SnapshotProcessMessage
 from snapshotter.utils.rpc import RpcHelper
-from snapshotter.utils.snapshot_worker import SnapshotAsyncWorker
 from snapshotter.utils.snapshot_utils import get_eth_price_usd
-from snapshotter.utils.callback_helpers import send_failure_notifications_async
+from snapshotter.utils.snapshot_worker import SnapshotAsyncWorker
+
 
 class ProcessorDistributor:
     _anchor_rpc_helper: RpcHelper
@@ -143,7 +146,9 @@ async def init(self):
                 self._slots_per_day = slots_per_day
 
             try:
-                snapshotter_address = self._protocol_state_contract.functions.slotSnapshotterMapping(settings.slot_id).call()
+                snapshotter_address = self._protocol_state_contract.functions.slotSnapshotterMapping(
+                    settings.slot_id,
+                ).call()
                 if snapshotter_address != to_checksum_address(settings.instance_id):
                     self._logger.error('Signer Account is not the one configured in slot, exiting!')
                     exit(0)
@@ -236,13 +241,20 @@ async def _epoch_release_processor(self, message: EpochReleasedEvent):
                 'Exception in getting eth price: {}',
                 e,
             )
-            notification_message = SnapshotterIssue(
-                instanceID=settings.instance_id,
-                issueType=SnapshotterReportState.MISSED_SNAPSHOT.value,
-                projectID='ETH_PRICE_LOAD',
-                epochId=str(message.epochId),
-                timeOfReporting=str(time.time()),
-                extra=json.dumps({'issueDetails': f'Error : {e}'}),
+            notification_message = SnapshotterReportData(
+                snapshotterIssue=SnapshotterIssue(
+                    instanceID=settings.instance_id,
+                    issueType=SnapshotterReportState.MISSED_SNAPSHOT.value,
+                    projectID='ETH_PRICE_LOAD',
+                    epochId=str(message.epochId),
+                    timeOfReporting=str(time.time()),
+                    extra=json.dumps({'issueDetails': f'Error : {e}'}),
+                ),
+                snapshotterStatus=SnapshotterStatus(
+                    totalMissedSubmissions=-1,
+                    consecutiveMissedSubmissions=-1,
+                    projects=[],
+                ),
             )
             await send_failure_notifications_async(
                 client=self._client, message=notification_message,
@@ -253,7 +265,7 @@ async def _epoch_release_processor(self, message: EpochReleasedEvent):
             # release for snapshotting
             asyncio.ensure_future(
                 self._distribute_callbacks_snapshotting(
-                    project_type, epoch, eth_price_dict
+                    project_type, epoch, eth_price_dict,
                 ),
             )
 

From 646ba638c75894c5061949c5fd759b0778a9ea15 Mon Sep 17 00:00:00 2001
From: Seth-Schmidt <seth@powerloom.io>
Date: Wed, 3 Apr 2024 00:41:10 -0400
Subject: [PATCH 06/13] chore: update README with telegram instructions

---
 README.md | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index fba63b7..0f9fd27 100755
--- a/README.md
+++ b/README.md
@@ -192,7 +192,7 @@ NOTE: It is recommended to run `build.sh` in a screen or tmux session so that th
    git clone https://github.com/PowerLoom/snapshotter-lite-v2.git powerloom-pre-mainnet
    ```
     This will clone the repository into a directory named `powerloom`.
-  
+
 3. Change your working directory to the `powerloom-pre-mainnet` directory:
    ```bash
    cd powerloom-pre-mainnet
@@ -233,7 +233,7 @@ If you want to run the Snapshotter Lite Node without Docker, you need to make su
    git clone https://github.com/PowerLoom/snapshotter-lite-v2.git powerloom-pre-mainnet
    ```
     This will clone the repository into a directory named `powerloom`.
-  
+
 2. Change your working directory to the `powerloom-pre-mainnet` directory:
    ```bash
    cd powerloom-pre-mainnet
@@ -248,9 +248,17 @@ If you want to run the Snapshotter Lite Node without Docker, you need to make su
 
 5. Your node should start in background and you should start seeing logs in your terminal.
 6. To stop the node, you can run `pkill -f snapshotter` in a new terminal window.
-  
+
 ## Monitoring and Debugging
 
+### Monitoring
+
+To enable Telegram reporting for snapshotter issues:
+1. Search for `@PowerloomReportingBot` in the Telegram App and start a conversation.
+2. Start the bot by typing the `/start` command in the chat. You will receive a response containing your `Chat ID` for the bot.
+3. Enter the `Chat ID` when prompted on node startup.
+4. You will now receive an error report whenever your node fails to process an epoch or snapshot.
+
 ### Debugging
 Usually the easiest way to fix node related issues is to restart the node. If you're facing issues with the node, you can try going through the logs present in the `logs` directory. If you're unable to find the issue, you can reach out to us on [Discord](https://powerloom.io/discord) and we will be happy to help you out.
 

From 3c83b64b6c82109a0d159eb0cc9f3ed179295005 Mon Sep 17 00:00:00 2001
From: Seth-Schmidt <seth@powerloom.io>
Date: Wed, 3 Apr 2024 19:40:14 -0400
Subject: [PATCH 07/13] fix: import order causing logging issues

---
 snapshotter/processor_distributor.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/snapshotter/processor_distributor.py b/snapshotter/processor_distributor.py
index cb67ada..cf90251 100644
--- a/snapshotter/processor_distributor.py
+++ b/snapshotter/processor_distributor.py
@@ -1,6 +1,5 @@
 import asyncio
 import json
-import time
 from collections import defaultdict
 from typing import Union
 
@@ -10,10 +9,12 @@
 from httpx import Limits
 from httpx import Timeout
 from web3 import Web3
+import time
 
+from snapshotter.utils.models.data_models import SnapshotterReportData
+from snapshotter.utils.models.data_models import SnapshotterReportState
 from snapshotter.settings.config import projects_config
 from snapshotter.settings.config import settings
-from snapshotter.utils.callback_helpers import send_failure_notifications_async
 from snapshotter.utils.data_utils import get_snapshot_submision_window
 from snapshotter.utils.data_utils import get_source_chain_epoch_size
 from snapshotter.utils.data_utils import get_source_chain_id
@@ -23,16 +24,15 @@
 from snapshotter.utils.models.data_models import DayStartedEvent
 from snapshotter.utils.models.data_models import EpochReleasedEvent
 from snapshotter.utils.models.data_models import SnapshotFinalizedEvent
-from snapshotter.utils.models.data_models import SnapshotterIssue
-from snapshotter.utils.models.data_models import SnapshotterReportData
-from snapshotter.utils.models.data_models import SnapshotterReportState
-from snapshotter.utils.models.data_models import SnapshotterStatus
 from snapshotter.utils.models.data_models import SnapshottersUpdatedEvent
 from snapshotter.utils.models.message_models import EpochBase
 from snapshotter.utils.models.message_models import SnapshotProcessMessage
 from snapshotter.utils.rpc import RpcHelper
-from snapshotter.utils.snapshot_utils import get_eth_price_usd
 from snapshotter.utils.snapshot_worker import SnapshotAsyncWorker
+from snapshotter.utils.snapshot_utils import get_eth_price_usd
+from snapshotter.utils.callback_helpers import send_failure_notifications_async
+from snapshotter.utils.models.data_models import SnapshotterIssue
+from snapshotter.utils.models.data_models import SnapshotterStatus
 
 
 class ProcessorDistributor:

From 51a8967a189760cf2b1cf2acc6ea1032d780e815 Mon Sep 17 00:00:00 2001
From: Seth-Schmidt <seth@powerloom.io>
Date: Wed, 24 Apr 2024 12:33:29 -0400
Subject: [PATCH 08/13] fix: share worker submission state with processor
 distributor

---
 snapshotter/processor_distributor.py |  8 +++-----
 snapshotter/utils/generic_worker.py  | 20 ++++++++++----------
 snapshotter/utils/snapshot_worker.py |  6 +++---
 3 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/snapshotter/processor_distributor.py b/snapshotter/processor_distributor.py
index cf90251..498a61a 100644
--- a/snapshotter/processor_distributor.py
+++ b/snapshotter/processor_distributor.py
@@ -241,6 +241,8 @@ async def _epoch_release_processor(self, message: EpochReleasedEvent):
                 'Exception in getting eth price: {}',
                 e,
             )
+            self.snapshot_worker.status.totalMissedSubmissions += 1
+            self.snapshot_worker.status.consecutiveMissedSubmissions += 1
             notification_message = SnapshotterReportData(
                 snapshotterIssue=SnapshotterIssue(
                     instanceID=settings.instance_id,
@@ -250,11 +252,7 @@ async def _epoch_release_processor(self, message: EpochReleasedEvent):
                     timeOfReporting=str(time.time()),
                     extra=json.dumps({'issueDetails': f'Error : {e}'}),
                 ),
-                snapshotterStatus=SnapshotterStatus(
-                    totalMissedSubmissions=-1,
-                    consecutiveMissedSubmissions=-1,
-                    projects=[],
-                ),
+                snapshotterStatus=self.snapshot_worker.status,
             )
             await send_failure_notifications_async(
                 client=self._client, message=notification_message,
diff --git a/snapshotter/utils/generic_worker.py b/snapshotter/utils/generic_worker.py
index e989553..005fa3f 100644
--- a/snapshotter/utils/generic_worker.py
+++ b/snapshotter/utils/generic_worker.py
@@ -129,8 +129,8 @@ def __init__(self):
         self.protocol_state_contract_address = settings.protocol_state.address
         self.initialized = False
         self.logger = logger.bind(module='GenericAsyncWorker')
-        self._status = SnapshotterStatus(projects=[])
-
+        self.status = SnapshotterStatus(projects=[])
+    
     def _notification_callback_result_handler(self, fut: asyncio.Future):
         """
         Handles the result of a callback or notification.
@@ -376,8 +376,8 @@ async def _commit_payload(
                 'Exception uploading snapshot to IPFS for epoch {}: {}, Error: {},'
                 'sending failure notifications', epoch, snapshot, e,
             )
-            self._status.totalMissedSubmissions += 1
-            self._status.consecutiveMissedSubmissions += 1
+            self.status.totalMissedSubmissions += 1
+            self.status.consecutiveMissedSubmissions += 1
             notification_message = SnapshotterReportData(
                 snapshotterIssue=SnapshotterIssue(
                     instanceID=settings.instance_id,
@@ -387,7 +387,7 @@ async def _commit_payload(
                     timeOfReporting=str(time.time()),
                     extra=json.dumps({'issueDetails': f'Error : {e}'}),
                 ),
-                snapshotterStatus=self._status,
+                snapshotterStatus=self.status,
             )
             await send_failure_notifications_async(
                 client=self._client, message=notification_message,
@@ -401,8 +401,8 @@ async def _commit_payload(
                     'Exception submitting snapshot to collector for epoch {}: {}, Error: {},'
                     'sending failure notifications', epoch, snapshot, e,
                 )
-                self._status.totalMissedSubmissions += 1
-                self._status.consecutiveMissedSubmissions += 1
+                self.status.totalMissedSubmissions += 1
+                self.status.consecutiveMissedSubmissions += 1
 
                 notification_message = SnapshotterReportData(
                     snapshotterIssue=SnapshotterIssue(
@@ -413,15 +413,15 @@ async def _commit_payload(
                         timeOfReporting=str(time.time()),
                         extra=json.dumps({'issueDetails': f'Error : {e}'}),
                     ),
-                    snapshotterStatus=self._status,
+                    snapshotterStatus=self.status,
                 )
                 await send_failure_notifications_async(
                     client=self._client, message=notification_message,
                 )
             else:
                 # reset consecutive missed snapshots counter
-                self._status.consecutiveMissedSubmissions = 0
-                self._status.totalSuccessfulSubmissions += 1
+                self.status.consecutiveMissedSubmissions = 0
+                self.status.totalSuccessfulSubmissions += 1
 
         # upload to web3 storage
         if storage_flag:
diff --git a/snapshotter/utils/snapshot_worker.py b/snapshotter/utils/snapshot_worker.py
index 8cffa43..4a506d7 100644
--- a/snapshotter/utils/snapshot_worker.py
+++ b/snapshotter/utils/snapshot_worker.py
@@ -98,8 +98,8 @@ async def _process(self, msg_obj: SnapshotProcessMessage, task_type: str, eth_pr
                 'sending failure notifications', msg_obj, e,
             )
 
-            self._status.totalMissedSubmissions += 1
-            self._status.consecutiveMissedSubmissions += 1
+            self.status.totalMissedSubmissions += 1
+            self.status.consecutiveMissedSubmissions += 1
 
             notification_message = SnapshotterReportData(
                 snapshotterIssue=SnapshotterIssue(
@@ -110,7 +110,7 @@ async def _process(self, msg_obj: SnapshotProcessMessage, task_type: str, eth_pr
                     timeOfReporting=str(time.time()),
                     extra=json.dumps({'issueDetails': f'Error : {e}'}),
                 ),
-                snapshotterStatus=self._status,
+                snapshotterStatus=self.status,
             )
 
             await send_failure_notifications_async(

From 49e6d1d24bb11450b79479c2181ae872cab838e1 Mon Sep 17 00:00:00 2001
From: Seth-Schmidt <seth@powerloom.io>
Date: Wed, 24 Apr 2024 14:52:02 -0400
Subject: [PATCH 09/13] chore: remove unnecessary report throttling

---
 snapshotter/utils/callback_helpers.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/snapshotter/utils/callback_helpers.py b/snapshotter/utils/callback_helpers.py
index 2df15ec..def5f57 100644
--- a/snapshotter/utils/callback_helpers.py
+++ b/snapshotter/utils/callback_helpers.py
@@ -11,7 +11,6 @@
 from pydantic import BaseModel
 
 from snapshotter.settings.config import settings
-from snapshotter.settings.config import project_types
 from snapshotter.utils.default_logger import logger
 from snapshotter.utils.models.data_models import TelegramEpochProcessingReportMessage
 from snapshotter.utils.models.data_models import TelegramSnapshotterReportMessage
@@ -110,10 +109,9 @@ async def send_failure_notifications_async(client: AsyncClient, message: Snapsho
         )
         report_frequency = settings.reporting.failure_report_frequency
         consecutive_misses = reporting_message.status.consecutiveMissedSubmissions
-        # Need to account for # of projects to avoid skipping reports
         if (
             consecutive_misses <= 1 or 
-            (consecutive_misses // len(project_types)) % report_frequency == 0
+            consecutive_misses % report_frequency == 0
         ):
             f = asyncio.ensure_future(
                 client.post(
@@ -163,7 +161,7 @@ def send_failure_notifications_sync(client: SyncClient, message: SnapshotterRepo
 
         if (
             consecutive_misses <= 1 or 
-            (consecutive_misses // len(project_types)) % report_frequency == 0
+            consecutive_misses % report_frequency == 0
         ):
             f = functools.partial(
                 client.post,

From 8f5dbd235f836256080f50aa88bdfb4428f5d7af Mon Sep 17 00:00:00 2001
From: Seth-Schmidt <seth@powerloom.io>
Date: Wed, 24 Apr 2024 22:53:37 -0400
Subject: [PATCH 10/13] chore: always send reports to tg host, limiting is
 handled by host

---
 snapshotter/utils/callback_helpers.py | 41 ++++++++++-----------------
 1 file changed, 15 insertions(+), 26 deletions(-)

diff --git a/snapshotter/utils/callback_helpers.py b/snapshotter/utils/callback_helpers.py
index def5f57..c08ce05 100644
--- a/snapshotter/utils/callback_helpers.py
+++ b/snapshotter/utils/callback_helpers.py
@@ -107,19 +107,14 @@ async def send_failure_notifications_async(client: AsyncClient, message: Snapsho
             issue=message.snapshotterIssue,
             status=message.snapshotterStatus,
         )
-        report_frequency = settings.reporting.failure_report_frequency
-        consecutive_misses = reporting_message.status.consecutiveMissedSubmissions
-        if (
-            consecutive_misses <= 1 or 
-            consecutive_misses % report_frequency == 0
-        ):
-            f = asyncio.ensure_future(
-                client.post(
-                    url=urljoin(settings.reporting.telegram_url, '/reportSnapshotIssue'),
-                    json=reporting_message.dict(),
-                ),
-            )
-            f.add_done_callback(misc_notification_callback_result_handler)
+        
+        f = asyncio.ensure_future(
+            client.post(
+                url=urljoin(settings.reporting.telegram_url, '/reportSnapshotIssue'),
+                json=reporting_message.dict(),
+            ),
+        )
+        f.add_done_callback(misc_notification_callback_result_handler)
 
 
 def send_failure_notifications_sync(client: SyncClient, message: SnapshotterReportData):
@@ -156,19 +151,13 @@ def send_failure_notifications_sync(client: SyncClient, message: SnapshotterRepo
             issue=message.snapshotterIssue,
             status=message.snapshotterStatus,
         )
-        report_frequency = settings.reporting.failure_report_frequency
-        consecutive_misses = reporting_message.status.consecutiveMissedSubmissions
-
-        if (
-            consecutive_misses <= 1 or 
-            consecutive_misses % report_frequency == 0
-        ):
-            f = functools.partial(
-                client.post,
-                url=urljoin(settings.reporting.telegram_url, '/reportSnapshotIssue'),
-                json=reporting_message.dict(),
-            )
-            sync_notification_callback_result_handler(f)
+
+        f = functools.partial(
+            client.post,
+            url=urljoin(settings.reporting.telegram_url, '/reportSnapshotIssue'),
+            json=reporting_message.dict(),
+        )
+        sync_notification_callback_result_handler(f)
 
 
 async def send_epoch_processing_failure_notification_async(client: AsyncClient, message: EpochProcessingIssue):

From 62a84ce0289b344b42241d4f19379fbf8c5bc340 Mon Sep 17 00:00:00 2001
From: Seth <seth@powerloom.io>
Date: Wed, 31 Jul 2024 15:06:33 -0400
Subject: [PATCH 11/13] fix: node setup config for telegram reporting

---
 build-dev.sh            | 11 +++++++++--
 build.sh                |  7 +++++++
 env.example             |  4 ++--
 snapshotter_autofill.sh |  2 +-
 4 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/build-dev.sh b/build-dev.sh
index c78fd15..3442afc 100755
--- a/build-dev.sh
+++ b/build-dev.sh
@@ -34,6 +34,13 @@ if [ ! -f .env ]; then
         sed -i'.backup' "s#<slot-id>#$SLOT_ID#" .env
     fi
 
+    # ask user for TELEGRAM_CHAT_ID and replace it in .env
+    if [ -z "$TELEGRAM_CHAT_ID" ]; then
+        echo "Enter Your TELEGRAM_CHAT_ID (Optional, leave blank to skip.): ";
+        read TELEGRAM_CHAT_ID;
+        sed -i'.backup' "s#<telegram-chat-id>#$TELEGRAM_CHAT_ID#" .env
+    fi
+
 fi
 
 source .env
@@ -124,7 +131,7 @@ git submodule update --init --recursive
 #     read response;
 #     if [ "$response" == "y" ]; then
 #         rm -rf ./snapshotter-lite-local-collector
-        
+
 #     fi
 # fi
 rm -rf snapshotter-lite-local-collector
@@ -149,4 +156,4 @@ else
     else
         docker-compose -f docker-compose-dev.yaml up --no-deps -V --abort-on-container-exit
     fi
-fi
\ No newline at end of file
+fi
diff --git a/build.sh b/build.sh
index da9e158..f633441 100755
--- a/build.sh
+++ b/build.sh
@@ -34,6 +34,13 @@ if [ ! -f .env ]; then
         sed -i'.backup' "s#<slot-id>#$SLOT_ID#" .env
     fi
 
+    # ask user for TELEGRAM_CHAT_ID and replace it in .env
+    if [ -z "$TELEGRAM_CHAT_ID" ]; then
+        echo "Enter Your TELEGRAM_CHAT_ID (Optional, leave blank to skip.): ";
+        read TELEGRAM_CHAT_ID;
+        sed -i'.backup' "s#<telegram-chat-id>#$TELEGRAM_CHAT_ID#" .env
+    fi
+
 fi
 
 source .env
diff --git a/env.example b/env.example
index 2ab017d..4c5e586 100755
--- a/env.example
+++ b/env.example
@@ -21,5 +21,5 @@ IPFS_API_SECRET=
 SLACK_REPORTING_URL=
 WEB3_STORAGE_TOKEN=
 DASHBOARD_ENABLED=
-TELEGRAM_REPORTING_URL=
-TELEGRAM_CHAT_ID=
+TELEGRAM_REPORTING_URL=https://tg-testing.powerloom.io/
+TELEGRAM_CHAT_ID=<telegram-chat-id>
diff --git a/snapshotter_autofill.sh b/snapshotter_autofill.sh
index faff705..5646f26 100755
--- a/snapshotter_autofill.sh
+++ b/snapshotter_autofill.sh
@@ -102,7 +102,7 @@ echo "Using powerloom reporting url: ${powerloom_reporting_url}"
 echo "Using web3 storage token: ${web3_storage_token}"
 echo "Using relayer host: ${relayer_host}"
 echo "Using telegram reporting url: ${telegram_reporting_url}"
-echo "Using telegram chat id: ${telegram_chat_id}
+echo "Using telegram chat id: ${telegram_chat_id}"
 
 sed -i'.backup' "s#relevant-namespace#$namespace#" config/settings.json
 

From e4cc74f3745bc7e16ad7e5a370b90ed9b42155b8 Mon Sep 17 00:00:00 2001
From: Seth <seth@powerloom.io>
Date: Thu, 8 Aug 2024 19:37:17 -0400
Subject: [PATCH 12/13] fix: notify on failure to send simulation submission

---
 snapshotter/system_event_detector.py | 10 ++++++++++
 snapshotter/utils/generic_worker.py  | 26 +++++++++++++++++++++++++-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/snapshotter/system_event_detector.py b/snapshotter/system_event_detector.py
index 3982a55..2f335ed 100644
--- a/snapshotter/system_event_detector.py
+++ b/snapshotter/system_event_detector.py
@@ -137,6 +137,16 @@ async def _init_check_and_report(self):
                 '❌ Dummy Event processing failed! Error: {}', e,
             )
             self._logger.info("Please check your config and if issue persists please reach out to the team!")
+            notification_message = EpochProcessingIssue(
+                instanceID=settings.instance_id,
+                issueType=SnapshotterReportState.UNHEALTHY_EPOCH_PROCESSING.value,
+                timeOfReporting=str(time.time()),
+                extra=json.dumps({'issueDetails': f'Error : {e}'})
+            )
+            send_epoch_processing_failure_notification_sync(
+                client=self._httpx_client, 
+                message=notification_message
+            )
             sys.exit(1)
 
     async def get_events(self, from_block: int, to_block: int):
diff --git a/snapshotter/utils/generic_worker.py b/snapshotter/utils/generic_worker.py
index 005fa3f..2a9ff77 100644
--- a/snapshotter/utils/generic_worker.py
+++ b/snapshotter/utils/generic_worker.py
@@ -19,6 +19,7 @@
 from grpclib.client import Channel
 from httpx import AsyncClient
 from httpx import AsyncHTTPTransport
+from httpx import Client
 from httpx import Limits
 from httpx import Timeout
 from ipfs_cid import cid_sha256_hash
@@ -34,6 +35,7 @@
 from snapshotter.settings.config import settings
 from snapshotter.utils.callback_helpers import misc_notification_callback_result_handler
 from snapshotter.utils.callback_helpers import send_failure_notifications_async
+from snapshotter.utils.callback_helpers import send_failure_notifications_sync
 from snapshotter.utils.default_logger import logger
 from snapshotter.utils.file_utils import read_json_file
 from snapshotter.utils.models.data_models import SnapshotterIssue
@@ -130,7 +132,7 @@ def __init__(self):
         self.initialized = False
         self.logger = logger.bind(module='GenericAsyncWorker')
         self.status = SnapshotterStatus(projects=[])
-    
+
     def _notification_callback_result_handler(self, fut: asyncio.Future):
         """
         Handles the result of a callback or notification.
@@ -248,6 +250,28 @@ async def _submit_to_snap_api_and_check(self, project_id: str, epoch: SnapshotPr
                 '❌ Event processing failed: {}', epoch,
             )
             self.logger.info('Please check your config and if issue persists please reach out to the team!')
+            self.status.totalMissedSubmissions += 1
+            self.status.consecutiveMissedSubmissions += 1
+            notification_message = SnapshotterReportData(
+                snapshotterIssue=SnapshotterIssue(
+                    instanceID=settings.instance_id,
+                    issueType=SnapshotterReportState.MISSED_SNAPSHOT.value,
+                    projectID=project_id,
+                    epochId=str(epoch.epochId),
+                    timeOfReporting=str(time.time()),
+                    extra=json.dumps({
+                        'issueDetails': f'Error : {e}',
+                    }),
+                ),
+                snapshotterStatus=self.status,
+            )
+            sync_client = Client(
+                timeout=Timeout(timeout=5.0),
+                follow_redirects=False,
+            )
+            send_failure_notifications_sync(
+                client=sync_client, message=notification_message,
+            )
             sys.exit(1)
 
     @asynccontextmanager

From 3f50726162380664b8fcd81e00a2e45faab41cb7 Mon Sep 17 00:00:00 2001
From: Swaroop Hegde <email@swaroophegde.com>
Date: Mon, 2 Sep 2024 16:29:30 +0530
Subject: [PATCH 13/13] added link to the bot

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0f9fd27..2673bac 100755
--- a/README.md
+++ b/README.md
@@ -254,7 +254,7 @@ If you want to run the Snapshotter Lite Node without Docker, you need to make su
 ### Monitoring
 
 To enable Telegram reporting for snapshotter issues:
-1. Search for `@PowerloomReportingBot` in the Telegram App and start a conversation.
+1. Open the conversation with [@PowerloomReportingBot](https://t.me/PowerloomReportingBot) in the Telegram App and start a conversation.
 2. Start the bot by typing the `/start` command in the chat. You will receive a response containing your `Chat ID` for the bot.
 3. Enter the `Chat ID` when prompted on node startup.
 4. You will now receive an error report whenever your node fails to process an epoch or snapshot.