Merge pull request #15 from johandahlberg/preping_for_aeacus

This enables the siswrap service to run the aeacus-family scripts

config/app.config

@@ -5,6 +5,8 @@ sender: [email protected]
 receiver: [email protected]
 report_bin: /opt/arteria/arteria-siswrap-env/deps/sisyphus/quickReport.pl
 qc_bin: /opt/arteria/arteria-siswrap-env/deps/sisyphus/qcValidateRun.pl
+aeacus_stats: /opt/arteria/arteria-siswrap-env/deps/sisyphus/aeacus-stats.pl
+aeacus_reports: /opt/arteria/arteria-siswrap-env/deps/sisyphus/aeacus-reports.pl
 version_bin: /opt/arteria/arteria-siswrap-env/deps/sisyphus/version.pl
 runfolder_root: /data/testarteria1/mon1
 perl: /usr/bin/perl

siswrap/__init__.py

@@ -1,4 +1,4 @@
 """Runs tools from Sisyphus suite
 """
 
-__version__ = '1.0.2'
+__version__ = '1.1.0'

siswrap/app.py

@@ -1,17 +1,21 @@
+from tornado.web import URLSpec as url
+
 from arteria.web.app import AppService
 from siswrap.handlers import RunHandler, StatusHandler
 from siswrap.wrapper_services import ProcessService
 
 
+def routes(**kwargs):
+    return [
+        url(r"/api/1.0/(?:qc|report|aeacusstats|aeacusreports)/run/([\w_-]+)",
+            RunHandler, name="run", kwargs=kwargs),
+        url(r"/api/1.0/(?:qc|report|aeacusstats|aeacusreports)/status/(\d*)",
+            StatusHandler, name="status", kwargs=kwargs)]
+
 def start():
     app_svc = AppService.create(__package__)
     process_svc = ProcessService(app_svc.config_svc)
 
     # Setup the routing. Help will be automatically available at /api, and will
     # be based on the doc strings of the get/post/put/delete methods
-    args = dict(process_svc=process_svc, config_svc=app_svc.config_svc)
-    routes = [
-        (r"/api/1.0/(?:qc|report)/run/([\w_-]+)", RunHandler, args),
-        (r"/api/1.0/(?:qc|report)/status/(\d*)", StatusHandler, args)
-    ]
-    app_svc.start(routes)
+    app_svc.start(routes(process_svc=process_svc, config_svc=app_svc.config_svc))

siswrap/wrapper_services.py

@@ -3,6 +3,7 @@
 import subprocess
 import shutil
 import time
+import re
 from subprocess import check_output
 import logging
 from arteria.web.state import State
@@ -58,9 +59,10 @@ def none_process(pid):
                            msg="No such process exists")
 
 
-class ExecString(object):
+class ExecStringWithEmailConfig(object):
     """ Object for storing the string that will be executed. Content is semi
-        standardised, as the called Perl scripts almost looks the same. It will
+        standardised, as the called Perl scripts almost looks the same. With this type
+         of configuration it provides information for the sending emails. It will
         vary somewhat depending on which wrapper is creating the object.
 
          Args:
@@ -80,6 +82,27 @@ def __init__(self, wrapper, conf_svc, runfolder):
                      "-mail", conf["receiver"], "-sender", conf["sender"]]
 
 
+class ExecStringBasic(object):
+    """ Object for storing the string that will be executed. Content is semi
+        standardised, as the called Perl scripts almost looks the same. It will
+        vary somewhat depending on which wrapper is creating the object.
+
+         Args:
+            wrapper: the object creating ExecString
+            conf_svc: the ConfigurationService serving our config lookups
+            runfolder: which runfolder to use
+
+         Returns:
+            Sets its property 'text' to the string that will be executed
+            in a subprocess.
+     """
+    def __init__(self, wrapper, conf_svc, runfolder):
+        self.text = None
+        bin_lookup = wrapper.binary_conf_lookup
+        conf = conf_svc.get_app_config()
+        self.text = [conf["perl"], conf[bin_lookup], "-runfolder", runfolder]
+
+
 class Wrapper(object):
     """ Our main wrapper for the Sisyphus scripts (QuickReport and
         QualityControl at the moment).
@@ -98,6 +121,8 @@ class Wrapper(object):
 
     QC_TYPE = "qc"
     REPORT_TYPE = "report"
+    AEACUS_STATS_TYPE = "aeacusstats"
+    AEACUS_REPORTS_TYPE = "aeacusreports"
 
     def __init__(self, params, configuration_svc, logger=None):
         self.conf_svc = configuration_svc
@@ -115,7 +140,6 @@ def __init__(self, params, configuration_svc, logger=None):
             path = runpath + "/sisyphus.yml"
             self.write_new_config_file(path, params["sisyphus_config"])
 
-
     def __get_attr__(self, attr):
         return getattr(self.info, attr)
 
@@ -146,7 +170,6 @@ def write_new_config_file(path, content):
             logger.error("Error writing new config file {0}: {1}".
                               format(path, err))
 
-
     def sisyphus_version(self):
         """
         Use Sisyphus own script to check which version is used.
@@ -161,6 +184,9 @@ def sisyphus_version(self):
     def stop(self):
         pass
 
+    def get_exec_string(self):
+        return ExecStringWithEmailConfig(self, self.conf_svc, self.info.runfolder).text
+
     def run(self):
         """  Creates an execution string that will be unique depending on
              what kind of object did the call to the method. Spawns a subprocess
@@ -174,8 +200,7 @@ def run(self):
                 proc = subprocess.Popen(["/bin/sleep", "1m"])
                 exec_string = "/bin/sleep 1m"
             else:
-                exec_string = ExecString(self, self.conf_svc,
-                                         self.info.runfolder).text
+                exec_string = self.get_exec_string()
                 proc = subprocess.Popen(exec_string, stdout=subprocess.PIPE,
                                         stderr=subprocess.PIPE)
 
@@ -190,11 +215,18 @@ def run(self):
     def url_to_type(url):
         """ Helper method to see which wrapper object might belong to which URL.
         """
-        # TODO: Take out the correct part of the URL instead.
-        if Wrapper.QC_TYPE in url:
+
+        url_pattern = "\/api/\d.\d\/(\w+)\/"
+        first_match = re.search(url_pattern, url).group(1)
+
+        if first_match == Wrapper.QC_TYPE:
             return Wrapper.QC_TYPE
-        elif Wrapper.REPORT_TYPE in url:
+        elif first_match == Wrapper.REPORT_TYPE:
             return Wrapper.REPORT_TYPE
+        elif first_match == Wrapper.AEACUS_STATS_TYPE:
+            return Wrapper.AEACUS_STATS_TYPE
+        elif first_match == Wrapper.AEACUS_REPORTS_TYPE:
+            return Wrapper.AEACUS_REPORTS_TYPE
         else:
             raise RuntimeError("Unknown wrapper runner requested: {0}".
                                format(url))
@@ -208,11 +240,62 @@ def new_wrapper(wrapper_type, runfolder, configuration_svc):
             return QCWrapper(runfolder, configuration_svc)
         elif wrapper_type == Wrapper.REPORT_TYPE:
             return ReportWrapper(runfolder, configuration_svc)
+        elif wrapper_type == Wrapper.AEACUS_STATS_TYPE:
+            return AeacusStatsWrapper(runfolder, configuration_svc)
+        elif wrapper_type == Wrapper.AEACUS_REPORTS_TYPE:
+            return AeacusReportsWrapper(runfolder, configuration_svc)
         else:
             raise RuntimeError("Unknown wrapper runner requested: {0}".
                                format(wrapper_type))
 
 
+class AeacusBaseWrapper(Wrapper):
+    """
+    Base wrapper for the Aeacus commands
+    """
+    def get_exec_string(self):
+        """
+        Overrides the `Wrapper.get_exec_string` implemenation with an ExecString that does not have
+        an email configuration since those values cannot be passed to the aeacus commands.
+        :return:
+        """
+        return ExecStringBasic(self, self.conf_svc, self.info.runfolder).text
+
+
+class AeacusStatsWrapper(AeacusBaseWrapper):
+    """ Wrapper around the aeacus-stats perl script. Inherits behaviour from its
+        base class AeacusBaseWrapper.
+
+        Args:
+            params: Dict of parameters to the wrapper. Must contain the name of
+                    the runfolder to use (not full path). Can contain a YAML
+                    object containing the Sisyphus config to use.
+            configuration_svc: the ConfigurationService for our conf lookups
+            logger: the Logger object in charge of logging output
+    """
+    def __init__(self, params, configuration_svc, logger=None):
+        super(AeacusStatsWrapper, self).__init__(params, configuration_svc, logger)
+        self.binary_conf_lookup = "aeacus_stats"
+        self.type_txt = Wrapper.AEACUS_STATS_TYPE
+
+
+class AeacusReportsWrapper(AeacusBaseWrapper):
+    """ Wrapper around the aeacus-reports perl script. Inherits behaviour from its
+        base class AeacusBaseWrapper.
+
+        Args:
+            params: Dict of parameters to the wrapper. Must contain the name of
+                    the runfolder to use (not full path). Can contain a YAML
+                    object containing the Sisyphus config to use.
+            configuration_svc: the ConfigurationService for our conf lookups
+            logger: the Logger object in charge of logging output
+    """
+
+    def __init__(self, params, configuration_svc, logger=None):
+        super(AeacusReportsWrapper, self).__init__(params, configuration_svc, logger)
+        self.binary_conf_lookup = "aeacus_reports"
+        self.type_txt = Wrapper.AEACUS_REPORTS_TYPE
+
 class ReportWrapper(Wrapper):
     """ Wrapper around the QuickReport perl script. Inherits behaviour from its
         base class Wrapper.

tests/unit/siswrap_handlers_tests.py

@@ -7,7 +7,7 @@
 from siswrap.handlers import *
 from siswrap.wrapper_services import *
 from siswrap_test_helpers import *
-
+from siswrap.app import routes
 
 # Some unit tests for siswrap.handlers
 
@@ -18,11 +18,8 @@
 def app():
     config_svc = ConfigurationService(app_config_path="./config/app.config")
     process_svc = ProcessService(config_svc)
-    args = dict(process_svc=process_svc, config_svc=config_svc)
-    app = tornado.web.Application([
-            (r"/api/1.0/(?:qc|report)/run/([\w_-]+)", RunHandler, args),
-            (r"/api/1.0/(?:qc|report)/status/(\d*)", StatusHandler, args)
-        ], debug=True)
+    #args = dict(process_svc=process_svc, config_svc=config_svc)
+    app = tornado.web.Application(routes(process_svc=process_svc, config_svc=config_svc), debug=True)
     return app
 
 @pytest.fixture
@@ -79,6 +76,26 @@ def test_post_report_job(self, http_client, http_server, base_url, stub_isdir, s
         resp = yield http_client.fetch(base_url + API_URL + "/report/run/123",
                                        method="POST", body=json(payload))
 
+    @pytest.mark.gen_test
+    def test_post_aeacus_report_job(self, http_client, http_server, base_url, stub_isdir, stub_sisyphus_version, stub_new_sisyphus_conf):
+        payload = {"runfolder": "foo", "sisyphus_config": TestHelpers.SISYPHUS_CONFIG}
+        resp = yield http_client.fetch(base_url + API_URL + "/aeacusreports/run/123",
+                                       method="POST", body=json(payload))
+
+        assert resp.code == 202
+        payload = jsonpickle.decode(resp.body)
+        assert payload["sisyphus_version"] == "15.3.2"
+        from siswrap import __version__ as version
+        assert payload["service_version"] == version
+        assert payload["runfolder"] == "/data/testarteria1/mon1/foo"
+
+        # Test empty input for sisyphus_conf field; the asserts in my_new_config
+        # should not be run if we sent this in.
+        payload = {"runfolder": "foo", "sisyphus_config": " "}
+        resp = yield http_client.fetch(base_url + API_URL + "/report/run/123",
+                                       method="POST", body=json(payload))
+
+
     @pytest.mark.gen_test
     def test_post_qc_job(self, http_client, http_server, base_url, stub_isdir, stub_sisyphus_version, stub_new_qc_conf):
         payload = {"runfolder": "foo", "qc_config": TestHelpers.QC_CONFIG}

tests/unit/siswrap_wrapper_services_tests.py

@@ -122,7 +122,7 @@ def __init__(self, wrapper, conf, runfolder):
                 else:
                     self.text = ["/bin/uggla"]
 
-        monkeypatch.setattr("siswrap.wrapper_services.ExecString", MockedExecString)
+        monkeypatch.setattr("siswrap.wrapper_services.ExecStringWithEmailConfig", MockedExecString)
         w = Wrapper(Helper.params, Helper.conf)
         w.run()
 
@@ -154,11 +154,14 @@ def test_new_wrapper(self, stub_isdir):
     # Helper method should return the correct wrapper type in text
     # format for different URLs
     def test_url_to_type(self, stub_isdir):
-        test_urls = ["http://arteria1:1111/v1/api/qc/run/8312",
-                     "https://arteria12:3232/v2/api/report/run/3232",
-                     "http://testweb/api/1/qc/status",
-                     "http://testtest/api/v1/report/run"]
-        test_types = ["qc", "report", "qc", "report"]
+        test_urls = ["http://localhost/api/1.0/qc/run/test",
+                     "http://localhost:10900/api/1.0/qc/run/test",
+                     "http://localhost:10900/api/1.0/report/run/test",
+                     "http://localhost:10900/api/1.0/aeacusstats/run/test",
+                     "http://localhost:10900/api/1.0/aeacusreports/run/test",
+                     "http://localhost:10900/api/1.0/aeacusreports/status/31322"
+                     ]
+        test_types = ["qc", "qc", "report", "aeacusstats", "aeacusreports", "aeacusreports"]
 
         for idx, url in enumerate(test_urls):
             assert Wrapper.url_to_type(url) == test_types[idx]
@@ -231,7 +234,7 @@ def mocked_get_config(self):
                             mocked_get_config)
 
         foobar = FooBar()
-        retobj = ExecString(foobar, Helper.conf, Helper.runfolder)
+        retobj = ExecStringWithEmailConfig(foobar, Helper.conf, Helper.runfolder)
         assert foobar.binary_conf_lookup in retobj.text
 
         # 8 elements in execstring: perl, binary, runfolder, runfolderpath,