EventNotify extension added

scrapylib/eventnotify.py

@@ -0,0 +1,33 @@
+import boto
+from scrapy.exceptions import NotConfigured
+
+
+# signal
+event_signal = object()
+
+
+class EventNotify(object):
+
+    def __init__(self, settings):
+        self._ses_credentials = (settings.get('EVENTNOTIFY_SESID'),
+                settings.get('EVENTNOTIFY_SESKEY'))
+        self._from = settings.get('EVENTNOTIFY_FROM', '')
+        self._to = settings.getlist('EVENTNOTIFY_TO', [])
+        self._subject = settings.get('EVENTNOTIFY_SUBJECT', '')
+        self._body = settings.get('EVENTNOTIFY_BODY', '')
+        if not (all(self._ses_credentials) and self._to and self._from):
+            raise NotConfigured
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        settings = crawler.settings
+        if not settings.getbool('EVENTNOTIFY_ENABLED'):
+            raise NotConfigured
+        o = cls(settings)
+        crawler.signals.connect(o.notify, signal=event_signal)
+        return o
+
+    def notify(self, **kwargs):
+        ses = boto.connect_ses(*self._ses_credentials)
+        ses.send_email(self._from, self._subject.format(**kwargs),
+                self._body.format(**kwargs), self._to)

tests/test_eventnotify.py

@@ -0,0 +1,62 @@
+import unittest
+
+from scrapy.utils.test import get_crawler
+from scrapy.exceptions import NotConfigured
+
+from scrapylib.eventnotify import EventNotify, event_signal
+
+
+class EventNotifyTestCase(unittest.TestCase):
+
+    ext_cls = EventNotify
+
+    def _mock_crawler(self, settings=None):
+        class MockedDownloader(object):
+            slots = {}
+
+        class MockedEngine(object):
+            downloader = MockedDownloader()
+            fake_spider_closed_result = None
+
+            def close_spider(self, spider, reason):
+                self.fake_spider_closed_result = (spider, reason)
+
+        crawler = get_crawler(settings)
+        crawler.engine = MockedEngine()
+        return crawler
+
+    def test_enabled(self):
+        settings = {'EVENTNOTIFY_ENABLED': True}
+        crawler = self._mock_crawler(settings)
+        self.assertRaises(NotConfigured, self.ext_cls.from_crawler, crawler)
+        settings['EVENTNOTIFY_SESID'] = 'id_123'
+        settings['EVENTNOTIFY_SESKEY'] = 'key_123'
+        crawler = self._mock_crawler(settings)
+        self.assertRaises(NotConfigured, self.ext_cls.from_crawler, crawler)
+        settings['EVENTNOTIFY_FROM'] = '[email protected]'
+        settings['EVENTNOTIFY_TO'] = '[email protected]'
+        crawler = self._mock_crawler(settings)
+        self.ext_cls.from_crawler(crawler)
+
+    def test_event_signal(self):
+        settings = {
+                'EVENTNOTIFY_TO': '[email protected]',
+                'EVENTNOTIFY_FROM': '[email protected]',
+                'EVENTNOTIFY_SESID': 'id_123',
+                'EVENTNOTIFY_SESKEY': 'key_123',
+                'EVENTNOTIFY_ENABLED': True,
+                }
+
+        class MockedEventNotify(self.ext_cls):
+
+            def __init__(self, *args, **kwargs):
+                super(MockedEventNotify, self).__init__(*args, **kwargs)
+                self.last_event = None
+
+            def notify(self, **kwargs):
+                self.last_event = kwargs
+
+        crawler = self._mock_crawler(settings)
+        ext = MockedEventNotify.from_crawler(crawler)
+        crawler.signals.send_catch_log(signal=event_signal, message='scrapy')
+        self.assertEqual('scrapy', ext.last_event.get('message'))