From dbeaf49b91993300ab5fb0c1a9b3eddf0fccbca9 Mon Sep 17 00:00:00 2001 From: Johan Dahlberg Date: Mon, 18 Apr 2016 14:18:09 +0200 Subject: [PATCH 1/6] Adding code to run aeacus-* --- config/app.config | 2 ++ siswrap/app.py | 4 ++-- siswrap/wrapper_services.py | 45 +++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/config/app.config b/config/app.config index c6c121b..0b1ccda 100644 --- a/config/app.config +++ b/config/app.config @@ -5,6 +5,8 @@ sender: me@example.com receiver: who@example.com report_bin: /opt/arteria/arteria-siswrap-env/deps/sisyphus/quickReport.pl qc_bin: /opt/arteria/arteria-siswrap-env/deps/sisyphus/qcValidateRun.pl +aeacus_stats: /opt/arteria/arteria-siswrap-env/deps/sisyphus/aeacus-stats.pl +aeacus_reports: /opt/arteria/arteria-siswrap-env/deps/sisyphus/aeacus-reports.pl version_bin: /opt/arteria/arteria-siswrap-env/deps/sisyphus/version.pl runfolder_root: /data/testarteria1/mon1 perl: /usr/bin/perl diff --git a/siswrap/app.py b/siswrap/app.py index 2b11457..486fa27 100644 --- a/siswrap/app.py +++ b/siswrap/app.py @@ -11,7 +11,7 @@ def start(): # be based on the doc strings of the get/post/put/delete methods args = dict(process_svc=process_svc, config_svc=app_svc.config_svc) routes = [ - (r"/api/1.0/(?:qc|report)/run/([\w_-]+)", RunHandler, args), - (r"/api/1.0/(?:qc|report)/status/(\d*)", StatusHandler, args) + (r"/api/1.0/(?:qc|report|aeacus-stats|aeacus-reports)/run/([\w_-]+)", RunHandler, args), + (r"/api/1.0/(?:qc|report|aeacus-stats|aeacus-reports)/status/(\d*)", StatusHandler, args) ] app_svc.start(routes) diff --git a/siswrap/wrapper_services.py b/siswrap/wrapper_services.py index b474150..50bdf59 100644 --- a/siswrap/wrapper_services.py +++ b/siswrap/wrapper_services.py @@ -98,6 +98,8 @@ class Wrapper(object): QC_TYPE = "qc" REPORT_TYPE = "report" + AEACUS_STATS_TYPE = "aeacus-stats" + AEACUS_REPORTS_TYPE = "aeacus-reports" def __init__(self, params, configuration_svc, logger=None): self.conf_svc = configuration_svc @@ -195,6 +197,10 @@ def url_to_type(url): return Wrapper.QC_TYPE elif Wrapper.REPORT_TYPE in url: return Wrapper.REPORT_TYPE + elif Wrapper.AEACUS_STATS_TYPE in url: + return Wrapper.AEACUS_STATS_TYPE + elif Wrapper.AEACUS_REPORTS_TYPE in url: + return Wrapper.AEACUS_REPORTS_TYPE else: raise RuntimeError("Unknown wrapper runner requested: {0}". format(url)) @@ -208,11 +214,50 @@ def new_wrapper(wrapper_type, runfolder, configuration_svc): return QCWrapper(runfolder, configuration_svc) elif wrapper_type == Wrapper.REPORT_TYPE: return ReportWrapper(runfolder, configuration_svc) + elif wrapper_type == Wrapper.AEACUS_STATS_TYPE: + return AeacusStatsWrapper(runfolder, configuration_svc) + elif wrapper_type == Wrapper.AEACUS_REPORTS_TYPE: + return AeacusReportsWrapper(runfolder, configuration_svc) else: raise RuntimeError("Unknown wrapper runner requested: {0}". format(wrapper_type)) +class AeacusStatsWrapper(Wrapper): + """ Wrapper around the aeacus-stats perl script. Inherits behaviour from its + base class Wrapper. + + Args: + params: Dict of parameters to the wrapper. Must contain the name of + the runfolder to use (not full path). Can contain a YAML + object containing the Sisyphus config to use. + configuration_svc: the ConfigurationService for our conf lookups + logger: the Logger object in charge of logging output + """ + + def __init__(self, params, configuration_svc, logger=None): + super(ReportWrapper, self).__init__(params, configuration_svc, logger) + self.binary_conf_lookup = "aeacus_stats" + self.type_txt = Wrapper.AEACUS_STATS_TYPE + + +class AeacusReportsWrapper(Wrapper): + """ Wrapper around the aeacus-reports perl script. Inherits behaviour from its + base class Wrapper. + + Args: + params: Dict of parameters to the wrapper. Must contain the name of + the runfolder to use (not full path). Can contain a YAML + object containing the Sisyphus config to use. + configuration_svc: the ConfigurationService for our conf lookups + logger: the Logger object in charge of logging output + """ + + def __init__(self, params, configuration_svc, logger=None): + super(ReportWrapper, self).__init__(params, configuration_svc, logger) + self.binary_conf_lookup = "aeacus_reports" + self.type_txt = Wrapper.AEACUS_REPORTS_TYPE + class ReportWrapper(Wrapper): """ Wrapper around the QuickReport perl script. Inherits behaviour from its base class Wrapper. From 494681812de5df6d8e14041bc0cab3e0efe69085 Mon Sep 17 00:00:00 2001 From: Johan Dahlberg Date: Mon, 18 Apr 2016 15:03:22 +0200 Subject: [PATCH 2/6] Factor our routes to funcion to use in tests later --- siswrap/app.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/siswrap/app.py b/siswrap/app.py index 486fa27..b428df6 100644 --- a/siswrap/app.py +++ b/siswrap/app.py @@ -1,17 +1,21 @@ +from tornado.web import URLSpec as url + from arteria.web.app import AppService from siswrap.handlers import RunHandler, StatusHandler from siswrap.wrapper_services import ProcessService +def routes(**kwargs): + return [ + url(r"/api/1.0/(?:qc|report|aeacusstats|aeacusreports)/run/([\w_-]+)", + RunHandler, name="run", kwargs=kwargs), + url(r"/api/1.0/(?:qc|report|aeacusstats|aeacusreports)/status/(\d*)", + StatusHandler, name="status", kwargs=kwargs)] + def start(): app_svc = AppService.create(__package__) process_svc = ProcessService(app_svc.config_svc) # Setup the routing. Help will be automatically available at /api, and will # be based on the doc strings of the get/post/put/delete methods - args = dict(process_svc=process_svc, config_svc=app_svc.config_svc) - routes = [ - (r"/api/1.0/(?:qc|report|aeacus-stats|aeacus-reports)/run/([\w_-]+)", RunHandler, args), - (r"/api/1.0/(?:qc|report|aeacus-stats|aeacus-reports)/status/(\d*)", StatusHandler, args) - ] - app_svc.start(routes) + app_svc.start(routes(process_svc=process_svc, config_svc=app_svc.config_svc)) From 830985ba3284dbfeca84d9f4b3965ab1c434eeb0 Mon Sep 17 00:00:00 2001 From: Johan Dahlberg Date: Mon, 18 Apr 2016 15:05:23 +0200 Subject: [PATCH 3/6] Changed class structure, added tests, regex url --- siswrap/wrapper_services.py | 22 +++++++++------- tests/unit/siswrap_handlers_tests.py | 27 ++++++++++++++++---- tests/unit/siswrap_wrapper_services_tests.py | 13 ++++++---- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/siswrap/wrapper_services.py b/siswrap/wrapper_services.py index 50bdf59..fb33581 100644 --- a/siswrap/wrapper_services.py +++ b/siswrap/wrapper_services.py @@ -3,6 +3,7 @@ import subprocess import shutil import time +import re from subprocess import check_output import logging from arteria.web.state import State @@ -98,8 +99,8 @@ class Wrapper(object): QC_TYPE = "qc" REPORT_TYPE = "report" - AEACUS_STATS_TYPE = "aeacus-stats" - AEACUS_REPORTS_TYPE = "aeacus-reports" + AEACUS_STATS_TYPE = "aeacusstats" + AEACUS_REPORTS_TYPE = "aeacusreports" def __init__(self, params, configuration_svc, logger=None): self.conf_svc = configuration_svc @@ -192,14 +193,17 @@ def run(self): def url_to_type(url): """ Helper method to see which wrapper object might belong to which URL. """ - # TODO: Take out the correct part of the URL instead. - if Wrapper.QC_TYPE in url: + + url_pattern = "\/api/\d.\d\/(\w+)\/" + first_match = re.search(url_pattern, url).group(1) + + if first_match == Wrapper.QC_TYPE: return Wrapper.QC_TYPE - elif Wrapper.REPORT_TYPE in url: + elif first_match == Wrapper.REPORT_TYPE: return Wrapper.REPORT_TYPE - elif Wrapper.AEACUS_STATS_TYPE in url: + elif first_match == Wrapper.AEACUS_STATS_TYPE: return Wrapper.AEACUS_STATS_TYPE - elif Wrapper.AEACUS_REPORTS_TYPE in url: + elif first_match == Wrapper.AEACUS_REPORTS_TYPE: return Wrapper.AEACUS_REPORTS_TYPE else: raise RuntimeError("Unknown wrapper runner requested: {0}". @@ -236,7 +240,7 @@ class AeacusStatsWrapper(Wrapper): """ def __init__(self, params, configuration_svc, logger=None): - super(ReportWrapper, self).__init__(params, configuration_svc, logger) + super(AeacusStatsWrapper, self).__init__(params, configuration_svc, logger) self.binary_conf_lookup = "aeacus_stats" self.type_txt = Wrapper.AEACUS_STATS_TYPE @@ -254,7 +258,7 @@ class AeacusReportsWrapper(Wrapper): """ def __init__(self, params, configuration_svc, logger=None): - super(ReportWrapper, self).__init__(params, configuration_svc, logger) + super(AeacusReportsWrapper, self).__init__(params, configuration_svc, logger) self.binary_conf_lookup = "aeacus_reports" self.type_txt = Wrapper.AEACUS_REPORTS_TYPE diff --git a/tests/unit/siswrap_handlers_tests.py b/tests/unit/siswrap_handlers_tests.py index aa1ec46..f6a46a0 100644 --- a/tests/unit/siswrap_handlers_tests.py +++ b/tests/unit/siswrap_handlers_tests.py @@ -7,7 +7,7 @@ from siswrap.handlers import * from siswrap.wrapper_services import * from siswrap_test_helpers import * - +from siswrap.app import routes # Some unit tests for siswrap.handlers @@ -19,10 +19,7 @@ def app(): config_svc = ConfigurationService(app_config_path="./config/app.config") process_svc = ProcessService(config_svc) args = dict(process_svc=process_svc, config_svc=config_svc) - app = tornado.web.Application([ - (r"/api/1.0/(?:qc|report)/run/([\w_-]+)", RunHandler, args), - (r"/api/1.0/(?:qc|report)/status/(\d*)", StatusHandler, args) - ], debug=True) + app = tornado.web.Application(routes(args), debug=True) return app @pytest.fixture @@ -79,6 +76,26 @@ def test_post_report_job(self, http_client, http_server, base_url, stub_isdir, s resp = yield http_client.fetch(base_url + API_URL + "/report/run/123", method="POST", body=json(payload)) + @pytest.mark.gen_test + def test_post_aeacus_report_job(self, http_client, http_server, base_url, stub_isdir, stub_sisyphus_version, stub_new_sisyphus_conf): + payload = {"runfolder": "foo", "sisyphus_config": TestHelpers.SISYPHUS_CONFIG} + resp = yield http_client.fetch(base_url + API_URL + "/report/aeacusreport/run/123", + method="POST", body=json(payload)) + + assert resp.code == 202 + payload = jsonpickle.decode(resp.body) + assert payload["sisyphus_version"] == "15.3.2" + from siswrap import __version__ as version + assert payload["service_version"] == version + assert payload["runfolder"] == "/data/testarteria1/mon1/foo" + + # Test empty input for sisyphus_conf field; the asserts in my_new_config + # should not be run if we sent this in. + payload = {"runfolder": "foo", "sisyphus_config": " "} + resp = yield http_client.fetch(base_url + API_URL + "/report/run/123", + method="POST", body=json(payload)) + + @pytest.mark.gen_test def test_post_qc_job(self, http_client, http_server, base_url, stub_isdir, stub_sisyphus_version, stub_new_qc_conf): payload = {"runfolder": "foo", "qc_config": TestHelpers.QC_CONFIG} diff --git a/tests/unit/siswrap_wrapper_services_tests.py b/tests/unit/siswrap_wrapper_services_tests.py index b0382ec..504b5c8 100644 --- a/tests/unit/siswrap_wrapper_services_tests.py +++ b/tests/unit/siswrap_wrapper_services_tests.py @@ -154,11 +154,14 @@ def test_new_wrapper(self, stub_isdir): # Helper method should return the correct wrapper type in text # format for different URLs def test_url_to_type(self, stub_isdir): - test_urls = ["http://arteria1:1111/v1/api/qc/run/8312", - "https://arteria12:3232/v2/api/report/run/3232", - "http://testweb/api/1/qc/status", - "http://testtest/api/v1/report/run"] - test_types = ["qc", "report", "qc", "report"] + test_urls = ["http://localhost/api/1.0/qc/run/test", + "http://localhost:10900/api/1.0/qc/run/test", + "http://localhost:10900/api/1.0/report/run/test", + "http://localhost:10900/api/1.0/aeacusstats/run/test", + "http://localhost:10900/api/1.0/aeacusreports/run/test", + "http://localhost:10900/api/1.0/aeacusreports/status/31322" + ] + test_types = ["qc", "qc", "report", "aeacusstats", "aeacusreports", "aeacusreports"] for idx, url in enumerate(test_urls): assert Wrapper.url_to_type(url) == test_types[idx] From 3bce1ad216e79573af6ad95f0f1d98c0eb89fae1 Mon Sep 17 00:00:00 2001 From: Johan Dahlberg Date: Tue, 19 Apr 2016 14:17:47 +0200 Subject: [PATCH 4/6] Add another ExecString class --- siswrap/wrapper_services.py | 56 ++++++++++++++++---- tests/unit/siswrap_wrapper_services_tests.py | 4 +- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/siswrap/wrapper_services.py b/siswrap/wrapper_services.py index fb33581..8d3b4f7 100644 --- a/siswrap/wrapper_services.py +++ b/siswrap/wrapper_services.py @@ -59,9 +59,10 @@ def none_process(pid): msg="No such process exists") -class ExecString(object): +class ExecStringWithEmailConfig(object): """ Object for storing the string that will be executed. Content is semi - standardised, as the called Perl scripts almost looks the same. It will + standardised, as the called Perl scripts almost looks the same. With this type + of configuration it provides information for the sending emails. It will vary somewhat depending on which wrapper is creating the object. Args: @@ -81,6 +82,27 @@ def __init__(self, wrapper, conf_svc, runfolder): "-mail", conf["receiver"], "-sender", conf["sender"]] +class ExecStringBasic(object): + """ Object for storing the string that will be executed. Content is semi + standardised, as the called Perl scripts almost looks the same. It will + vary somewhat depending on which wrapper is creating the object. + + Args: + wrapper: the object creating ExecString + conf_svc: the ConfigurationService serving our config lookups + runfolder: which runfolder to use + + Returns: + Sets its property 'text' to the string that will be executed + in a subprocess. + """ + def __init__(self, wrapper, conf_svc, runfolder): + self.text = None + bin_lookup = wrapper.binary_conf_lookup + conf = conf_svc.get_app_config() + self.text = [conf["perl"], conf[bin_lookup], "-runfolder", runfolder] + + class Wrapper(object): """ Our main wrapper for the Sisyphus scripts (QuickReport and QualityControl at the moment). @@ -118,7 +140,6 @@ def __init__(self, params, configuration_svc, logger=None): path = runpath + "/sisyphus.yml" self.write_new_config_file(path, params["sisyphus_config"]) - def __get_attr__(self, attr): return getattr(self.info, attr) @@ -149,7 +170,6 @@ def write_new_config_file(path, content): logger.error("Error writing new config file {0}: {1}". format(path, err)) - def sisyphus_version(self): """ Use Sisyphus own script to check which version is used. @@ -164,6 +184,9 @@ def sisyphus_version(self): def stop(self): pass + def get_exec_string(self): + return ExecStringWithEmailConfig(self, self.conf_svc, self.info.runfolder).text + def run(self): """ Creates an execution string that will be unique depending on what kind of object did the call to the method. Spawns a subprocess @@ -177,8 +200,7 @@ def run(self): proc = subprocess.Popen(["/bin/sleep", "1m"]) exec_string = "/bin/sleep 1m" else: - exec_string = ExecString(self, self.conf_svc, - self.info.runfolder).text + exec_string = self.get_exec_string() proc = subprocess.Popen(exec_string, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -227,9 +249,22 @@ def new_wrapper(wrapper_type, runfolder, configuration_svc): format(wrapper_type)) -class AeacusStatsWrapper(Wrapper): +class AeacusBaseWrapper(Wrapper): + """ + Base wrapper for the Aeacus commands + """ + def get_exec_string(self): + """ + Overrides the `Wrapper.get_exec_string` implemenation with an ExecString that does not have + an email configuration since those values cannot be passed to the aeacus commands. + :return: + """ + return ExecStringBasic(self, self.conf_svc, self.info.runfolder).text + + +class AeacusStatsWrapper(AeacusBaseWrapper): """ Wrapper around the aeacus-stats perl script. Inherits behaviour from its - base class Wrapper. + base class AeacusBaseWrapper. Args: params: Dict of parameters to the wrapper. Must contain the name of @@ -238,16 +273,15 @@ class AeacusStatsWrapper(Wrapper): configuration_svc: the ConfigurationService for our conf lookups logger: the Logger object in charge of logging output """ - def __init__(self, params, configuration_svc, logger=None): super(AeacusStatsWrapper, self).__init__(params, configuration_svc, logger) self.binary_conf_lookup = "aeacus_stats" self.type_txt = Wrapper.AEACUS_STATS_TYPE -class AeacusReportsWrapper(Wrapper): +class AeacusReportsWrapper(AeacusBaseWrapper): """ Wrapper around the aeacus-reports perl script. Inherits behaviour from its - base class Wrapper. + base class AeacusBaseWrapper. Args: params: Dict of parameters to the wrapper. Must contain the name of diff --git a/tests/unit/siswrap_wrapper_services_tests.py b/tests/unit/siswrap_wrapper_services_tests.py index 504b5c8..41ec389 100644 --- a/tests/unit/siswrap_wrapper_services_tests.py +++ b/tests/unit/siswrap_wrapper_services_tests.py @@ -122,7 +122,7 @@ def __init__(self, wrapper, conf, runfolder): else: self.text = ["/bin/uggla"] - monkeypatch.setattr("siswrap.wrapper_services.ExecString", MockedExecString) + monkeypatch.setattr("siswrap.wrapper_services.ExecStringWithEmailConfig", MockedExecString) w = Wrapper(Helper.params, Helper.conf) w.run() @@ -234,7 +234,7 @@ def mocked_get_config(self): mocked_get_config) foobar = FooBar() - retobj = ExecString(foobar, Helper.conf, Helper.runfolder) + retobj = ExecStringWithEmailConfig(foobar, Helper.conf, Helper.runfolder) assert foobar.binary_conf_lookup in retobj.text # 8 elements in execstring: perl, binary, runfolder, runfolderpath, From 70709da899a37a3a612cccf6fcc631d0bcb600f9 Mon Sep 17 00:00:00 2001 From: Johan Dahlberg Date: Tue, 19 Apr 2016 14:53:01 +0200 Subject: [PATCH 5/6] Bump version to 1.1.0 --- siswrap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siswrap/__init__.py b/siswrap/__init__.py index cc606ee..565ae15 100644 --- a/siswrap/__init__.py +++ b/siswrap/__init__.py @@ -1,4 +1,4 @@ """Runs tools from Sisyphus suite """ -__version__ = '1.0.2' +__version__ = '1.1.0' From 9b3978241a0d50075f07a81067f9783ad91d9796 Mon Sep 17 00:00:00 2001 From: Johan Dahlberg Date: Thu, 21 Apr 2016 15:47:37 +0200 Subject: [PATCH 6/6] Fix problems in tests --- tests/unit/siswrap_handlers_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/siswrap_handlers_tests.py b/tests/unit/siswrap_handlers_tests.py index f6a46a0..1d2e98f 100644 --- a/tests/unit/siswrap_handlers_tests.py +++ b/tests/unit/siswrap_handlers_tests.py @@ -18,8 +18,8 @@ def app(): config_svc = ConfigurationService(app_config_path="./config/app.config") process_svc = ProcessService(config_svc) - args = dict(process_svc=process_svc, config_svc=config_svc) - app = tornado.web.Application(routes(args), debug=True) + #args = dict(process_svc=process_svc, config_svc=config_svc) + app = tornado.web.Application(routes(process_svc=process_svc, config_svc=config_svc), debug=True) return app @pytest.fixture @@ -79,7 +79,7 @@ def test_post_report_job(self, http_client, http_server, base_url, stub_isdir, s @pytest.mark.gen_test def test_post_aeacus_report_job(self, http_client, http_server, base_url, stub_isdir, stub_sisyphus_version, stub_new_sisyphus_conf): payload = {"runfolder": "foo", "sisyphus_config": TestHelpers.SISYPHUS_CONFIG} - resp = yield http_client.fetch(base_url + API_URL + "/report/aeacusreport/run/123", + resp = yield http_client.fetch(base_url + API_URL + "/aeacusreports/run/123", method="POST", body=json(payload)) assert resp.code == 202