allow scripted changes to logged data

pom.xml

@@ -151,7 +151,14 @@
     	<version>1.15</version>
   		<optional>true</optional>
     </dependency>
+
     <dependency>
+       <groupId>org.jenkins-ci.plugins</groupId>
+       <artifactId>script-security</artifactId>
+       <version>1.37</version>
+    </dependency>
+
+      <dependency>
     	<groupId>org.hamcrest</groupId>
     	<artifactId>java-hamcrest</artifactId>
     	<version>2.0.0.0</version>

src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java

@@ -25,6 +25,7 @@
 package jenkins.plugins.logstash;
 
 import java.io.IOException;
+import java.io.OutputStream;
 
 import org.kohsuke.stapler.DataBoundConstructor;
 
@@ -35,8 +36,14 @@
 import hudson.model.BuildListener;
 import hudson.tasks.BuildWrapper;
 import hudson.tasks.BuildWrapperDescriptor;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript;
+
+import javax.annotation.CheckForNull;
+
+import org.kohsuke.stapler.DataBoundSetter;
 
 /**
+ * Logstash note on each output line.
  *
  * This BuildWrapper is not used anymore.
  * We just keep it to be able to convert projects that have the BuildWrapper configured at startup or when posting the xml via the rest api
@@ -48,13 +55,21 @@
 public class LogstashBuildWrapper extends BuildWrapper
 {
 
+  @CheckForNull
+  private SecureGroovyScript secureGroovyScript;
+
   /**
    * Create a new {@link LogstashBuildWrapper}.
    */
   @DataBoundConstructor
   public LogstashBuildWrapper()
   {}
 
+  @DataBoundSetter
+  public void setSecureGroovyScript(@CheckForNull SecureGroovyScript script) {
+    this.secureGroovyScript = script != null ? script.configuringWithNonKeyItem() : null;
+  }
+
   /**
    * {@inheritDoc}
    */
@@ -73,6 +88,21 @@ public DescriptorImpl getDescriptor()
     return (DescriptorImpl)super.getDescriptor();
   }
 
+  @CheckForNull
+  public SecureGroovyScript getSecureGroovyScript() {
+    return secureGroovyScript;
+  }
+
+  // Method to encapsulate calls for unit-testing
+  LogstashWriter getLogStashWriter(AbstractBuild<?, ?> build, OutputStream errorStream) {
+    LogstashScriptProcessor processor = null;
+    if (secureGroovyScript != null) {
+      processor = new LogstashScriptProcessor(secureGroovyScript, errorStream);
+    }
+
+    return new LogstashWriter(build, errorStream, null, build.getCharset(), processor);
+  }
+
   /**
    * Registers {@link LogstashBuildWrapper} as a {@link BuildWrapper}.
    */

src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java

@@ -30,6 +30,8 @@
 import java.io.IOException;
 import java.io.OutputStream;
 
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
 /**
  * Output stream that writes each line to the provided delegate output stream
  * and also sends it to an indexer for logstash to consume.
@@ -54,6 +56,9 @@ LogstashWriter getLogstashWriter()
   }
 
   @Override
+  @SuppressFBWarnings(
+    value="DM_DEFAULT_ENCODING",
+    justification="TODO: not sure how to fix this")
   protected void eol(byte[] b, int len) throws IOException {
     delegate.write(b, 0, len);
     this.flush();
@@ -79,6 +84,7 @@ public void flush() throws IOException {
    */
   @Override
   public void close() throws IOException {
+    logstash.close();
     delegate.close();
     super.close();
   }

src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java

@@ -0,0 +1,50 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2017 Red Hat inc, and individual contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package jenkins.plugins.logstash;
+
+import net.sf.json.JSONObject;
+
+/**
+ * Interface describing processors of persisted payload.
+ *
+ * @author Aleksandar Kostadinov
+ * @since 1.4.0
+ */
+public interface LogstashPayloadProcessor {
+  /**
+   * Modifies a JSON payload compatible with the Logstash schema.
+   *
+   * @param payload the JSON payload that has been constructed so far.
+   * @return The formatted JSON object, can be null to ignore this payload.
+   */
+  JSONObject process(JSONObject payload) throws Exception;
+
+  /**
+   * Finalizes any operations, for example returns cashed lines at end of build.
+   *
+   * @return A formatted JSON object, can be null when it has nothing.
+   */
+  JSONObject finish() throws Exception;
+}

src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java

@@ -0,0 +1,123 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2017 Red Hat inc. and individual contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package jenkins.plugins.logstash;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.util.LinkedHashMap;
+
+import javax.annotation.Nonnull;
+
+import groovy.lang.Binding;
+
+import net.sf.json.JSONObject;
+
+import jenkins.model.Jenkins;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript;
+import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.Whitelisted;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+/**
+ * This class is handling custom groovy script processing of JSON payload.
+ * Each call to process executes the script provided in job configuration.
+ * Script is executed under the same binding each time so that it has ability
+ * to persist data during build execution if desired by script author.
+ * When build is finished, script will receive null as the payload and can
+ * return any cached but non-sent data back for persisting.
+ * The return value of script is the payload to be persisted unless null.
+ *
+ * @author Aleksandar Kostadinov
+ * @since 1.4.0
+ */
+public class LogstashScriptProcessor implements LogstashPayloadProcessor{
+  @Nonnull
+  private final SecureGroovyScript script;
+
+  @Nonnull
+  private final OutputStream consoleOut;
+
+  /** Groovy binding for script execution */
+  @Nonnull
+  private final Binding binding;
+
+  /** Classloader for script execution */
+  @Nonnull
+  private final ClassLoader classLoader;
+
+  public LogstashScriptProcessor(SecureGroovyScript script, OutputStream consoleOut) {
+    this.script = script;
+    this.consoleOut = consoleOut;
+
+    // TODO: should we put variables in the binding like manager, job, etc.?
+    binding = new Binding();
+    binding.setVariable("console", new BuildConsoleWrapper());
+
+    // not sure what the diff is compared to getClass().getClassLoader();
+    final Jenkins jenkins = Jenkins.getInstance();
+    classLoader = jenkins.getPluginManager().uberClassLoader;
+  }
+
+  /**
+   * Helper method to allow logging to build console.
+   */
+  @SuppressFBWarnings(
+    value="DM_DEFAULT_ENCODING",
+    justification="TODO: not sure how to fix this")
+  private void buildLogPrintln(Object o) throws IOException {
+    consoleOut.write(o.toString().getBytes());
+    consoleOut.write("\n".getBytes());
+    consoleOut.flush();
+  }
+
+  /*
+   * good examples in:
+   *  https://github.com/jenkinsci/envinject-plugin/blob/master/src/main/java/org/jenkinsci/plugins/envinject/service/EnvInjectEnvVars.java
+   *  https://github.com/jenkinsci/groovy-postbuild-plugin/pull/11/files
+   */
+  @Override
+  public JSONObject process(JSONObject payload) throws Exception {
+    binding.setVariable("payload", payload);
+    script.evaluate(classLoader, binding);
+    return (JSONObject) binding.getVariable("payload");
+  }
+
+  @Override
+  public JSONObject finish() throws Exception {
+    buildLogPrintln("Tearing down Script Log Processor..");
+    return process(null);
+  }
+
+  /**
+   * Helper to allow access from sandboxed script to output messages to console.
+   */
+  private class BuildConsoleWrapper {
+    @Whitelisted
+    public void println(Object o) throws IOException {
+      buildLogPrintln(o);
+    }
+  }
+}

src/main/java/jenkins/plugins/logstash/LogstashWriter.java

@@ -42,13 +42,18 @@
 import java.util.Date;
 import java.util.List;
 
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+import javax.annotation.CheckForNull;
+
 /**
  * A writer that wraps all Logstash DAOs.  Handles error reporting and per build connection state.
  * Each call to write (one line or multiple lines) sends a Logstash payload to the DAO.
  * If any write fails, writer will not attempt to send any further messages to logstash during this build.
  *
  * @author Rusty Gerard
  * @author Liam Newman
+ * @author Aleksandar Kostadinov
  * @since 1.0.5
  */
 public class LogstashWriter {
@@ -62,11 +67,23 @@ public class LogstashWriter {
   private boolean connectionBroken;
   private Charset charset;
 
+  @CheckForNull
+  private final LogstashPayloadProcessor payloadProcessor;
+
+  public LogstashWriter(Run<?, ?> run, OutputStream error, TaskListener listener) {
+    this(run, error, listener, null);
+  }
+
   public LogstashWriter(Run<?, ?> run, OutputStream error, TaskListener listener, Charset charset) {
+	    this(run, error, listener, charset, null);
+	  }
+
+  public LogstashWriter(Run<?, ?> run, OutputStream error, TaskListener listener, Charset charset, LogstashPayloadProcessor payloadProcessor) {
     this.errorStream = error != null ? error : System.err;
     this.build = run;
     this.listener = listener;
     this.charset = charset;
+    this.payloadProcessor = payloadProcessor;
     this.dao = this.getDaoOrNull();
     if (this.dao == null) {
       this.jenkinsUrl = "";
@@ -165,8 +182,36 @@ String getJenkinsUrl() {
    * Write a list of lines to the indexer as one Logstash payload.
    */
   private void write(List<String> lines) {
+    write(dao.buildPayload(buildData, jenkinsUrl, lines));
+  }
+
+  /**
+   * Write JSONObject payload to the Logstash indexer.
+   * @since 1.0.5
+   */
+  private void write(JSONObject payload) {
     buildData.updateResult();
-    JSONObject payload = dao.buildPayload(buildData, jenkinsUrl, lines);
+    if (payloadProcessor != null) {
+      JSONObject processedPayload = payload;
+      try {
+        processedPayload = payloadProcessor.process(payload);
+      } catch (Exception e) {
+        String msg =  ExceptionUtils.getMessage(e) + "\n" +
+          "[logstash-plugin]: Error in payload processing.\n";
+
+        logErrorMessage(msg);
+      }
+      if (processedPayload != null) { writeRaw(processedPayload); }
+    } else {
+      writeRaw(payload);
+    }
+  }
+
+  /**
+   * Write JSONObject payload to the Logstash indexer.
+   * @since 1.0.5
+   */
+  private void writeRaw(JSONObject payload) {
     try {
       dao.push(payload.toString());
     } catch (IOException e) {
@@ -203,6 +248,9 @@ private LogstashIndexerDao getDaoOrNull() {
   /**
    * Write error message to errorStream and set connectionBroken to true.
    */
+  @SuppressFBWarnings(
+    value="DM_DEFAULT_ENCODING",
+    justification="TODO: not sure how to fix this")
   private void logErrorMessage(String msg) {
     try {
       connectionBroken = true;
@@ -214,4 +262,23 @@ private void logErrorMessage(String msg) {
     }
   }
 
+
+  /**
+   * Signal payload processor that there will be no more lines
+   */
+  public void close() {
+    if (payloadProcessor != null) {
+      JSONObject payload = null;
+      try {
+        // calling finish() is mandatory to avoid memory leaks
+        payload = payloadProcessor.finish();
+      } catch (Exception e) {
+        String msg =  ExceptionUtils.getMessage(e) + "\n" +
+          "[logstash-plugin]: Error with payload processor on finish.\n";
+
+        logErrorMessage(msg);
+      }
+      if (payload != null) writeRaw(payload);
+    }
+  }
 }