Merge pull request #8 from johanherman/write-config

Give Sisyphus and QC conf as HTTP input

config/app.config

@@ -1,11 +1,11 @@
 ---
 
 port: 10900
-sender: [email protected]
-receiver: johan.hermansson@gmail.com
+sender: [email protected]
+receiver: who@example.com
 report_bin: /opt/siswrap/deps/sisyphus/quickReport.pl
 qc_bin: /opt/siswrap/deps/sisyphus/qcValidateRun.pl
 version_bin: /opt/siswrap/deps/sisyphus/version.pl
 qc_file: /srv/qc_config/sisyphus_qc.xml
-runfolder_root: /data/testarteria1/mon1/
+runfolder_root: /data/testarteria1/mon1
 perl: /usr/bin/perl

siswrap/handlers.py

@@ -1,9 +1,11 @@
 import jsonpickle
+import json
 import arteria
+import tornado.web
 from arteria.web.handlers import BaseRestHandler
 from arteria.web.state import State
 from wrapper_services import ProcessService, Wrapper, ProcessInfo
-from siswrap import __version__ as version
+from siswrap import __version__ as siswrap_version
 
 
 class BaseSiswrapHandler(BaseRestHandler):
@@ -31,18 +33,14 @@ def write_status(self, proc_info):
         reason = "OK"
 
         # Write output for specific process
-        if type(proc_info) is dict:
-            state = proc_info.get("state")
-
-            if state == State.STARTED:
-                reason = "OK - still processing"
-            elif state == State.DONE:
-                reason = "OK - finished processing"
-            elif state == State.ERROR:
-                reason = "OK - but an error occured while processing"
-            else:
-                http_code = self.HTTP_ERROR
-                reason = "An unexpected error occured"
+        state = proc_info.get("state")
+
+        if state == State.STARTED:
+            reason = "OK - still processing"
+        elif state == State.DONE:
+            reason = "OK - finished processing"
+        elif state == State.ERROR:
+            reason = "OK - but an error occured while processing"
         # Write output for all processes
         else:
             reason = "OK"
@@ -84,13 +82,54 @@ class RunHandler(BaseSiswrapHandler):
     """ Our handler for requesting the launch of a new quick report and
         quality control.
     """
+
+    def setup_wrapper_parameters(self, wrapper_type):
+        """ Setups the input parameters to the wrapper type in question by
+            parsing the HTTP request body.
+
+            Args:
+                wrapper_type: specifies whether or not to look for a qc_config
+                              in the body.
+
+            Returns:
+                A dict with the runfolder, and if given: the Sisyphus config and
+                the QC config.
+        """
+
+        params = {}
+        expect_param = ["runfolder"]
+
+        if wrapper_type == "qc":
+            expect_param = expect_param + ["qc_config"]
+
+        body = self.body_as_object(expect_param)
+
+        params["runfolder"] = body["runfolder"].strip()
+
+        if wrapper_type == "qc":
+            if body["qc_config"].strip():
+                params["qc_config"] = body["qc_config"].strip()
+            else:
+                raise RuntimeError("qc_config can't be empty value!")
+
+        body = json.loads(self.request.body)
+
+        if "sisyphus_config" in body and body["sisyphus_config"].strip():
+            params["sisyphus_config"] = body["sisyphus_config"].strip()
+
+        return params
+
     def post(self, runfolder="/some/runfolder"):
         """ Start running Sisyphus quick report or quality control for specific
             runfolder.
 
             Args:
                 runfolder: Which runfolder to generate a report or quality
-                           control for.
+                           control for. (mandatory)
+                qc_config: Supply a custom QC XML config file that will be copied into
+                           Sisyphus root folder (mandatory for QC actions)
+                sisyphus_config: Supply a custom YAML config file that will overwrite then
+                                 default bundled in Sisyphus. (optional)
 
             Returns:
                 A status code HTTP 202 if the report generation or quality control
@@ -103,18 +142,10 @@ def post(self, runfolder="/some/runfolder"):
         """
         try:
             url = self.request.uri.strip()
-
-            expect_param = ["runfolder"]
-            body = self.body_as_object(expect_param)
-            runfolder = body["runfolder"].strip()
-
-            # Return a new wrapper object depending on what was requested in
-            # the URL, and then ask the process service to start execution.
             wrapper_type = Wrapper.url_to_type(url)
-            wrapper = Wrapper.new_wrapper(wrapper_type, str(runfolder),
-                                          self.config_svc)
-            sisyphus_version = wrapper.sisyphus_version()
-            my_version = version
+            wrapper_params = self.setup_wrapper_parameters(wrapper_type)
+
+            wrapper = Wrapper.new_wrapper(wrapper_type, wrapper_params, self.config_svc)
             result = self.process_svc.run(wrapper)
 
             self.append_status_link(result)
@@ -124,12 +155,12 @@ def post(self, runfolder="/some/runfolder"):
                     "runfolder": result.info.runfolder,
                     "link": result.info.link,
                     "msg": result.info.msg,
-                    "service_version": my_version,
-                    "sisyphus_version": sisyphus_version}
+                    "service_version": siswrap_version,
+                    "sisyphus_version": wrapper.sisyphus_version()}
+
             self.write_accepted(resp)
         except RuntimeError, err:
-            self.write_object("An error ocurred: " + str(err),
-                              http_code=500, reason="An error occurred")
+            raise tornado.web.HTTPError(500, "An error occurred: {0}".format(str(err)))
 
 
 class StatusHandler(BaseSiswrapHandler):
@@ -172,6 +203,6 @@ def get(self, pid):
             else:
                 # If a specific PID wasn't requested then return all
                 # processes of the specific wrapper type
-                self.write_status(self.process_svc.get_all(wrapper_type))
+                self.write_status({"statuses": self.process_svc.get_all(wrapper_type)})
         except RuntimeError, err:
-            self.write_object("An error occurred: " + str(err))
+            raise tornado.web.HTTPError(500, "An error occurred: {0}".format(str(err)))

siswrap/wrapper_services.py

@@ -1,6 +1,8 @@
 import os.path
 import socket
 import subprocess
+import shutil
+import datetime
 from subprocess import check_output
 import logging
 from arteria.web.state import State
@@ -83,30 +85,68 @@ class Wrapper(object):
         QualityControl at the moment).
 
         Args:
-            runfolder: the name of the runfolder to use (not full path)
-            configuration_svc: the ConfigurationService for our config lookups
-            logger: logger object for printouts
+            params: Dict of parameters to the wrapper. Must contain the name of
+                    the runfolder to use (not full path). Can contain a YAML
+                    object containing the Sisyphus config to use, and a XML
+                    object cotaining the QC config to use.
+            configuration_svc: The ConfigurationService for our config lookups
+            logger: Logger object for printouts
 
-        FIXME: Raises
+        Raises:
+            OSError: If the given runfolder doesn't exist.
     """
 
     QC_TYPE = "qc"
     REPORT_TYPE = "report"
 
-    def __init__(self, runfolder, configuration_svc, logger=None):
+    def __init__(self, params, configuration_svc, logger=None):
+        self.conf_svc = configuration_svc
+        self.logger = logger or logging.getLogger(__name__)
+
         conf = configuration_svc.get_app_config()
-        runpath = conf["runfolder_root"] + runfolder
+        runpath = conf["runfolder_root"] + "/" + params["runfolder"]
 
         if not os.path.isdir(runpath):
             raise OSError("No runfolder {0} exists.".format(runpath))
 
         self.info = ProcessInfo(runpath)
-        self.conf_svc = configuration_svc
-        self.logger = logger or logging.getLogger(__name__)
+
+        if "sisyphus_config" in params:
+            path = runpath + "/sisyphus.yml"
+            self.write_new_config_file(path, params["sisyphus_config"])
+
 
     def __get_attr__(self, attr):
         return getattr(self.info, attr)
 
+    @staticmethod
+    def write_new_config_file(path, content):
+        """ Writes new config file (especially used for Sisyphus YAML and QC XML).
+            If the file already exists a backup copy will be created.
+
+            Args:
+                - path: The path to the config file that should be written.
+                - content: The content of the new config file.
+        """
+        try:
+            logger = logging.getLogger(__name__)
+
+            logger.debug("Writing new config file " + path)
+
+            now = datetime.datetime.now().isoformat()
+
+            if os.path.isfile(path):
+                logger.debug("Config file already existed. Making backup copy.")
+                shutil.move(path, path + "." + now)
+
+            with open(path, "w") as f:
+                f.write(content)
+
+        except OSError, err:
+            logger.error("Error writing new config file {0}: {1}".
+                              format(path, err))
+
+
     def sisyphus_version(self):
         """
         Use Sisyphus own script to check which version is used.
@@ -178,48 +218,45 @@ class ReportWrapper(Wrapper):
         base class Wrapper.
 
         Args:
-            runfolder: the runfolder to start processing (not full path)
+            params: Dict of parameters to the wrapper. Must contain the name of
+                    the runfolder to use (not full path). Can contain a YAML
+                    object containing the Sisyphus config to use.
             configuration_svc: the ConfigurationService for our conf lookups
             logger: the Logger object in charge of logging output
     """
 
-    def __init__(self, runfolder, configuration_svc):
-        super(ReportWrapper, self).__init__(runfolder,
-                                            configuration_svc)
+    def __init__(self, params, configuration_svc, logger=None):
+        super(ReportWrapper, self).__init__(params, configuration_svc, logger)
         self.binary_conf_lookup = "report_bin"
         self.type_txt = Wrapper.REPORT_TYPE
 
-
 class QCWrapper(Wrapper):
     """ Wrapper around the QualityControl perl script. Inherits behaviour from
         its base class Wrapper.
 
          Args:
-             runfolder: the runfolder to start processing (not full path)
+            params: Dict of parameters to the wrapper. Must contain the name of
+                    the runfolder to use (not full path). Can contain a YAML
+                    object containing the Sisyphus config to use, and a XML
+                    object containing the QC config to use. If a config is given
+                    then they will be written to the runfolder where Sisyphus
+                    will be able to use them.
              configuration_svc: ConfigurationService serving our conf lookups
              logger: the Logger object in charge of logging output
 
         Raises:
             IOError: if we couldn't copy the QC settings input file
      """
 
-    def __init__(self, runfolder, configuration_svc, logger=None):
-        super(QCWrapper, self).__init__(runfolder, configuration_svc)
+    def __init__(self, params, configuration_svc, logger=None):
+        super(QCWrapper, self).__init__(params, configuration_svc, logger)
         self.binary_conf_lookup = "qc_bin"
         self.type_txt = Wrapper.QC_TYPE
-        self.logger = logger or logging.getLogger(__name__)
+        conf = self.conf_svc.get_app_config()
 
-        try:
-            # Copy QC settings file from central server location (provisioned
-            # from elsewhere) to current runfolder destination
-            import shutil
-            conf = self.conf_svc.get_app_config()
-            src = conf["qc_file"]
-            dst = conf["runfolder_root"] + runfolder + "/sisyphus_qc.xml"
-            shutil.copyfile(src, dst)
-        except IOError, err:
-            self.logger.error("Couldn't copy file {0} to {1}: {2}".
-                              format(src, dst, err))
+        if "qc_config" in params:
+            path = conf["runfolder_root"] + "/" + params["runfolder"] + "/sisyphus_qc.xml"
+            self.write_new_config_file(path, params["qc_config"])
 
 
 class ProcessService(object):
@@ -386,8 +423,8 @@ def get_all(self, wrapper_type):
                 wrapper_type: the object type to check statuses for
 
             Returns:
-                a list of processes and some meta information if they are running, otherwise
-                an empty list
+                a dict of processes and some meta information if they are running, otherwise
+                an empty dict
         """
         # Only update processes of our requested wrapper class
         map(lambda pid: self.poll_process(pid)