diff --git a/pom.xml b/pom.xml index 4bacf6a3..65114a78 100644 --- a/pom.xml +++ b/pom.xml @@ -151,7 +151,14 @@ 1.15 true + + org.jenkins-ci.plugins + script-security + 1.37 + + + org.hamcrest java-hamcrest 2.0.0.0 diff --git a/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java b/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java index 81ec9f41..9873a99c 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java @@ -35,8 +35,17 @@ import hudson.model.BuildListener; import hudson.tasks.BuildWrapper; import hudson.tasks.BuildWrapperDescriptor; +import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript; + +import java.io.IOException; +import java.io.OutputStream; + +import javax.annotation.CheckForNull; + +import org.kohsuke.stapler.DataBoundSetter; /** + * Logstash note on each output line. * * This BuildWrapper is not used anymore. * We just keep it to be able to convert projects that have the BuildWrapper configured at startup or when posting the xml via the rest api @@ -48,6 +57,9 @@ public class LogstashBuildWrapper extends BuildWrapper { + @CheckForNull + private SecureGroovyScript secureGroovyScript; + /** * Create a new {@link LogstashBuildWrapper}. */ @@ -55,6 +67,11 @@ public class LogstashBuildWrapper extends BuildWrapper public LogstashBuildWrapper() {} + @DataBoundSetter + public void setSecureGroovyScript(@CheckForNull SecureGroovyScript script) { + this.secureGroovyScript = script != null ? script.configuringWithNonKeyItem() : null; + } + /** * {@inheritDoc} */ @@ -66,11 +83,26 @@ public Environment setUp(AbstractBuild build, Launcher launcher, BuildListener l { }; } - + @Override - public DescriptorImpl getDescriptor() - { - return (DescriptorImpl)super.getDescriptor(); + public DescriptorImpl getDescriptor() { + return (DescriptorImpl) super.getDescriptor(); + } + + @CheckForNull + public SecureGroovyScript getSecureGroovyScript() { + // FIXME probbably needs to be moved + return secureGroovyScript; + } + + // Method to encapsulate calls for unit-testing + LogstashWriter getLogStashWriter(AbstractBuild build, OutputStream errorStream) { + LogstashScriptProcessor processor = null; + if (secureGroovyScript != null) { + processor = new LogstashScriptProcessor(secureGroovyScript, errorStream); + } + + return new LogstashWriter(build, errorStream, null, build.getCharset(), processor); } /** diff --git a/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java b/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java index ae5922df..27ff38f3 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java @@ -72,7 +72,6 @@ public Descriptor() { load(); } - @Override public String getDisplayName() { return Messages.DisplayName(); diff --git a/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java b/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java index f6ae5de2..412ad0c6 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java @@ -30,6 +30,8 @@ import java.io.IOException; import java.io.OutputStream; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * Output stream that writes each line to the provided delegate output stream * and also sends it to an indexer for logstash to consume. @@ -54,6 +56,9 @@ LogstashWriter getLogstashWriter() } @Override + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") protected void eol(byte[] b, int len) throws IOException { delegate.write(b, 0, len); this.flush(); @@ -79,6 +84,7 @@ public void flush() throws IOException { */ @Override public void close() throws IOException { + logstash.close(); delegate.close(); super.close(); } diff --git a/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java b/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java new file mode 100644 index 00000000..e39f60ea --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java @@ -0,0 +1,50 @@ +/* + * The MIT License + * + * Copyright 2017 Red Hat inc, and individual contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package jenkins.plugins.logstash; + +import net.sf.json.JSONObject; + +/** + * Interface describing processors of persisted payload. + * + * @author Aleksandar Kostadinov + * @since 1.4.0 + */ +public interface LogstashPayloadProcessor { + /** + * Modifies a JSON payload compatible with the Logstash schema. + * + * @param payload the JSON payload that has been constructed so far. + * @return The formatted JSON object, can be null to ignore this payload. + */ + JSONObject process(JSONObject payload) throws Exception; + + /** + * Finalizes any operations, for example returns cashed lines at end of build. + * + * @return A formatted JSON object, can be null when it has nothing. + */ + JSONObject finish() throws Exception; +} diff --git a/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java b/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java new file mode 100644 index 00000000..a0cf545a --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java @@ -0,0 +1,123 @@ +/* + * The MIT License + * + * Copyright 2017 Red Hat inc. and individual contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package jenkins.plugins.logstash; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.LinkedHashMap; + +import javax.annotation.Nonnull; + +import groovy.lang.Binding; + +import net.sf.json.JSONObject; + +import jenkins.model.Jenkins; +import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript; +import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.Whitelisted; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * This class is handling custom groovy script processing of JSON payload. + * Each call to process executes the script provided in job configuration. + * Script is executed under the same binding each time so that it has ability + * to persist data during build execution if desired by script author. + * When build is finished, script will receive null as the payload and can + * return any cached but non-sent data back for persisting. + * The return value of script is the payload to be persisted unless null. + * + * @author Aleksandar Kostadinov + * @since 1.4.0 + */ +public class LogstashScriptProcessor implements LogstashPayloadProcessor{ + @Nonnull + private final SecureGroovyScript script; + + @Nonnull + private final OutputStream consoleOut; + + /** Groovy binding for script execution */ + @Nonnull + private final Binding binding; + + /** Classloader for script execution */ + @Nonnull + private final ClassLoader classLoader; + + public LogstashScriptProcessor(SecureGroovyScript script, OutputStream consoleOut) { + this.script = script; + this.consoleOut = consoleOut; + + // TODO: should we put variables in the binding like manager, job, etc.? + binding = new Binding(); + binding.setVariable("console", new BuildConsoleWrapper()); + + // not sure what the diff is compared to getClass().getClassLoader(); + final Jenkins jenkins = Jenkins.getInstance(); + classLoader = jenkins.getPluginManager().uberClassLoader; + } + + /** + * Helper method to allow logging to build console. + */ + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") + private void buildLogPrintln(Object o) throws IOException { + consoleOut.write(o.toString().getBytes()); + consoleOut.write("\n".getBytes()); + consoleOut.flush(); + } + + /* + * good examples in: + * https://github.com/jenkinsci/envinject-plugin/blob/master/src/main/java/org/jenkinsci/plugins/envinject/service/EnvInjectEnvVars.java + * https://github.com/jenkinsci/groovy-postbuild-plugin/pull/11/files + */ + @Override + public JSONObject process(JSONObject payload) throws Exception { + binding.setVariable("payload", payload); + script.evaluate(classLoader, binding); + return (JSONObject) binding.getVariable("payload"); + } + + @Override + public JSONObject finish() throws Exception { + buildLogPrintln("Tearing down Script Log Processor.."); + return process(null); + } + + /** + * Helper to allow access from sandboxed script to output messages to console. + */ + private class BuildConsoleWrapper { + @Whitelisted + public void println(Object o) throws IOException { + buildLogPrintln(o); + } + } +} diff --git a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java index ba4d26c9..9f4afee1 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java @@ -42,6 +42,10 @@ import java.util.Date; import java.util.List; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +import javax.annotation.CheckForNull; + /** * A writer that wraps all Logstash DAOs. Handles error reporting and per build connection state. * Each call to write (one line or multiple lines) sends a Logstash payload to the DAO. @@ -49,6 +53,7 @@ * * @author Rusty Gerard * @author Liam Newman + * @author Aleksandar Kostadinov * @since 1.0.5 */ public class LogstashWriter { @@ -62,11 +67,23 @@ public class LogstashWriter { private boolean connectionBroken; private Charset charset; + @CheckForNull + private final LogstashPayloadProcessor payloadProcessor; + + public LogstashWriter(Run run, OutputStream error, TaskListener listener) { + this(run, error, listener, null); + } + public LogstashWriter(Run run, OutputStream error, TaskListener listener, Charset charset) { + this(run, error, listener, charset, null); + } + + public LogstashWriter(Run run, OutputStream error, TaskListener listener, Charset charset, LogstashPayloadProcessor payloadProcessor) { this.errorStream = error != null ? error : System.err; this.build = run; this.listener = listener; this.charset = charset; + this.payloadProcessor = payloadProcessor; this.dao = this.getDaoOrNull(); if (this.dao == null) { this.jenkinsUrl = ""; @@ -165,8 +182,36 @@ String getJenkinsUrl() { * Write a list of lines to the indexer as one Logstash payload. */ private void write(List lines) { + write(dao.buildPayload(buildData, jenkinsUrl, lines)); + } + + /** + * Write JSONObject payload to the Logstash indexer. + * @since 1.0.5 + */ + private void write(JSONObject payload) { buildData.updateResult(); - JSONObject payload = dao.buildPayload(buildData, jenkinsUrl, lines); + if (payloadProcessor != null) { + JSONObject processedPayload = payload; + try { + processedPayload = payloadProcessor.process(payload); + } catch (Exception e) { + String msg = ExceptionUtils.getMessage(e) + "\n" + + "[logstash-plugin]: Error in payload processing.\n"; + + logErrorMessage(msg); + } + if (processedPayload != null) { writeRaw(processedPayload); } + } else { + writeRaw(payload); + } + } + + /** + * Write JSONObject payload to the Logstash indexer. + * @since 1.0.5 + */ + private void writeRaw(JSONObject payload) { try { dao.push(payload.toString()); } catch (IOException e) { @@ -203,6 +248,9 @@ private LogstashIndexerDao getDaoOrNull() { /** * Write error message to errorStream and set connectionBroken to true. */ + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") private void logErrorMessage(String msg) { try { connectionBroken = true; @@ -214,4 +262,23 @@ private void logErrorMessage(String msg) { } } + + /** + * Signal payload processor that there will be no more lines + */ + public void close() { + if (payloadProcessor != null) { + JSONObject payload = null; + try { + // calling finish() is mandatory to avoid memory leaks + payload = payloadProcessor.finish(); + } catch (Exception e) { + String msg = ExceptionUtils.getMessage(e) + "\n" + + "[logstash-plugin]: Error with payload processor on finish.\n"; + + logErrorMessage(msg); + } + if (payload != null) writeRaw(payload); + } + } } diff --git a/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java b/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java index 0fea9d4d..40b08362 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java @@ -25,11 +25,14 @@ package jenkins.plugins.logstash.persistence; import java.util.Calendar; +import java.util.Date; import java.util.List; import jenkins.plugins.logstash.LogstashConfiguration; import net.sf.json.JSONObject; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * Abstract data access object for Logstash indexers. * diff --git a/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java b/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java index d96cec29..c091b2db 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/BuildData.java @@ -50,12 +50,15 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import net.sf.json.JSONObject; +import javax.annotation.CheckForNull; import org.apache.commons.lang.StringUtils; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * POJO for mapping build info to JSON. * @@ -93,6 +96,9 @@ public TestData() { this(null); } + @SuppressFBWarnings( + value="URF_UNREAD_FIELD", + justification="TODO: not sure how to fix this") public TestData(Action action) { AbstractTestResultAction testResultAction = null; if (action instanceof AbstractTestResultAction) { @@ -151,7 +157,7 @@ public List getFailedTests() } private String id; - private String result; + @CheckForNull private String result; private String projectName; private String fullProjectName; private String displayName; @@ -256,7 +262,7 @@ private void initData(Run build, Date currentTime) { url = build.getUrl(); buildNum = build.getNumber(); buildDuration = currentTime.getTime() - build.getStartTimeInMillis(); - timestamp = LogstashConfiguration.getInstance().getDateFormatter().format(build.getTimestamp().getTime()); + timestamp = LogstashConfiguration.getInstance().getDateFormatter().format(build.getTime()); updateResult(); } diff --git a/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java b/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java index d96e077c..c2aa9ee4 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java @@ -52,6 +52,7 @@ import jenkins.plugins.logstash.configuration.ElasticSearch; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; /** * Elastic Search Data Access Object. @@ -77,6 +78,9 @@ public ElasticSearchDao(URI uri, String username, String password) { } // Factored for unit testing + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") ElasticSearchDao(HttpClientBuilder factory, URI uri, String username, String password) { if (uri == null) @@ -192,6 +196,9 @@ public void push(String data) throws IOException { } } + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") private String getErrorMessage(CloseableHttpResponse response) { ByteArrayOutputStream byteStream = null; PrintStream stream = null; diff --git a/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java b/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java index 16753078..0a231ba5 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java @@ -33,6 +33,8 @@ import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * RabbitMQ Data Access Object. * @@ -122,6 +124,9 @@ public String getVirtualHost() * @see jenkins.plugins.logstash.persistence.LogstashIndexerDao#push(java.lang.String) */ @Override + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") public void push(String data) throws IOException { Connection connection = null; Channel channel = null; diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/config.jelly b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/config.jelly new file mode 100644 index 00000000..4c7cbf08 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/config.jelly @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help-script.html b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help-script.html new file mode 100644 index 00000000..27404758 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help-script.html @@ -0,0 +1,37 @@ +

+ With this script you can filter, modify and/or group messages that are sent to the + configured logstash backend. The following variables are available during execution: +

    +
  • payload - JSONObject contaning the payload to be persisted.
    • +
  • console - a class to output messages to build log without persisting them. It also works in a sandbox: console.println("my logged message").
    • +
+

+

+ The script will be executed once per each line of build console output with + the variable payload set to the complete JSON payload to be + sent including message, timestamp, build url, etc. + The line text in particular will be present under the "message" + key as an array with a single string element. +

+

+ Once script completes execution, the payload variable will be read out + of current Binding and passed down to the Logstash backend to be persisted. If + payload is set to null by the script, then nothing will be + persisted. +

+

+ The script within a build will be executed always using the same Binding. + This means that variables can be saved between script invocations. +

+

+ At the end of the build a payload == null will be submitted. You can use this to + output any messages that you have cached for grouping or other purposes. +

+

+ Example script to filter out some console messages by pattern: + 


+    if (payload && payload["message"][0] =~ /my needless pattern/) {
+      payload = null
+    }
+
+

diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html new file mode 100644 index 00000000..d0e26e2b --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html @@ -0,0 +1,6 @@ +
+

+ Send individual log lines to Logstash. You can optionally filter and modify them + via groovy script in from advanced configuration. +

+
diff --git a/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java b/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java index 13c4774f..e52e08fd 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java @@ -15,6 +15,7 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.InOrder; import org.mockito.junit.MockitoJUnitRunner; @SuppressWarnings("resource") @@ -122,4 +123,17 @@ public void eolSuccessNoDao() throws Exception { assertEquals("Results don't match", msg, buffer.toString()); verify(mockWriter).isConnectionBroken(); } + + @Test + public void writerClosedBeforeDelegate() throws Exception { + ByteArrayOutputStream mockBuffer = Mockito.spy(buffer); + new LogstashOutputStream(mockBuffer, mockWriter).close(); + + InOrder inOrder = Mockito.inOrder(mockBuffer, mockWriter); + inOrder.verify(mockWriter).close(); + inOrder.verify(mockBuffer).close(); + + // Verify results + assertEquals("Results don't match", "", buffer.toString()); + } } diff --git a/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java b/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java index 36c54a95..b06a5983 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java @@ -20,17 +20,22 @@ import java.nio.charset.Charset; import java.util.Arrays; import java.util.Collections; -import java.util.GregorianCalendar; +import java.util.Date; import java.util.List; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.Captor; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Matchers; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -46,19 +51,25 @@ import hudson.tasks.test.AbstractTestResultAction; import jenkins.plugins.logstash.persistence.BuildData; import jenkins.plugins.logstash.persistence.LogstashIndexerDao; +import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript; +import org.jvnet.hudson.test.JenkinsRule; import net.sf.json.JSONObject; +import net.sf.json.JSONArray; @RunWith(PowerMockRunner.class) @PowerMockIgnore({"javax.crypto.*"}) @PrepareForTest(LogstashConfiguration.class) public class LogstashWriterTest { + @Rule public JenkinsRule j = new JenkinsRule(); + // Extension of the unit under test that avoids making calls to getInstance() to get the DAO singleton static LogstashWriter createLogstashWriter(final AbstractBuild testBuild, OutputStream error, final String url, final LogstashIndexerDao indexer, - final BuildData data) { - return new LogstashWriter(testBuild, error, null, testBuild.getCharset()) { + final BuildData data, + final LogstashPayloadProcessor processor) { + return new LogstashWriter(testBuild, error, null, testBuild.getCharset(), processor) { @Override LogstashIndexerDao getIndexerDao() { return indexer; @@ -83,6 +94,14 @@ String getJenkinsUrl() { }; } + static LogstashWriter createLogstashWriter(final AbstractBuild testBuild, + OutputStream error, + final String url, + final LogstashIndexerDao indexer, + final BuildData data) { + return createLogstashWriter(testBuild, error, url, indexer, data, null); + } + ByteArrayOutputStream errorBuffer; @Mock LogstashIndexerDao mockDao; @@ -112,7 +131,7 @@ public void before() throws Exception { when(mockBuild.getProject()).thenReturn(mockProject); when(mockBuild.getParent()).thenReturn(mockProject); when(mockBuild.getNumber()).thenReturn(123456); - when(mockBuild.getTimestamp()).thenReturn(new GregorianCalendar()); + when(mockBuild.getTime()).thenReturn(new Date()); when(mockBuild.getRootBuild()).thenReturn(mockBuild); when(mockBuild.getBuildVariables()).thenReturn(Collections.emptyMap()); when(mockBuild.getSensitiveBuildVariables()).thenReturn(Collections.emptySet()); @@ -132,10 +151,16 @@ public void before() throws Exception { when(mockProject.getName()).thenReturn("LogstashWriterTest"); when(mockProject.getFullName()).thenReturn("parent/LogstashWriterTest"); - when(mockDao.buildPayload(any(BuildData.class), anyString(), anyListOf(String.class))) - .thenReturn(JSONObject.fromObject("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}")); - + .thenAnswer(new Answer() { + @Override + public JSONObject answer(InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + JSONObject json = JSONObject.fromObject("{\"data\":{},\"message\": null,\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + json.element("message", args[2]); + return json; + } + }); Mockito.doNothing().when(mockDao).push(anyString()); when(mockDao.getDescription()).thenReturn("localhost:8080"); @@ -170,7 +195,7 @@ public void constructorSuccess() throws Exception { verify(mockBuild).getAction(AbstractTestResultAction.class); verify(mockBuild).getExecutor(); verify(mockBuild, times(2)).getNumber(); - verify(mockBuild).getTimestamp(); + verify(mockBuild).getTime(); verify(mockBuild, times(4)).getRootBuild(); verify(mockBuild).getBuildVariables(); verify(mockBuild).getSensitiveBuildVariables(); @@ -266,14 +291,65 @@ public void writeBuildLogSuccess() throws Exception { // Verify results // No error output assertEquals("Results don't match", "", errorBuffer.toString()); - verify(mockBuild).getLog(3); + String lines = JSONArray.fromObject(mockBuild.getLog(3)).toString(); + verify(mockBuild, times(2)).getLog(3); verify(mockBuild).getCharset(); verify(mockDao).buildPayload(eq(mockBuildData), eq("http://my-jenkins-url"), anyListOf(String.class)); - verify(mockDao).push("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + verify(mockDao).push("{\"data\":{},\"message\":" + lines + ",\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); verify(mockBuildData).updateResult(); } + @Test + public void writeProcessedSuccess() throws Exception { + String goodMsg = "test message"; + String ignoredMsg = "ignored input"; + String scriptString = + "if (payload) {\n" + + " if (payload['message'][0] =~ /" + ignoredMsg + "/) {\n" + + " payload = null\n" + + " } else {\n" + + " console.println('l');\n" + + " }\n" + + " lastPayload = payload\n" + + "} else {\n" + + " console.println('test build console message')\n" + + " payload = lastPayload\n" + + "}"; + + SecureGroovyScript script = new SecureGroovyScript(scriptString, true, null); + script.configuringWithNonKeyItem(); + LogstashScriptProcessor processor = new LogstashScriptProcessor(script, errorBuffer); + LogstashWriter writer = createLogstashWriter(mockBuild, errorBuffer, "http://my-jenkins-url", mockDao, mockBuildData, processor); + errorBuffer.reset(); + + // Unit under test + writer.write(goodMsg); + writer.write(ignoredMsg); + writer.write(goodMsg); + writer.close(); + + // Verify results + // buffer contains 2 lines logged by the script, then standard tear down message and finally test message at close + assertEquals("Results don't match", "l\nl\nTearing down Script Log Processor..\ntest build console message\n", errorBuffer.toString()); + + InOrder inOrder = Mockito.inOrder(mockDao); + + // first message is generated and pushed to DAO + inOrder.verify(mockDao).buildPayload(Matchers.eq(mockBuildData), Matchers.eq("http://my-jenkins-url"), Matchers.anyListOf(String.class)); + inOrder.verify(mockDao).push("{\"data\":{},\"message\":[\"" + goodMsg + "\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + // now message only generated but filtered out by script thus not pushed to DAO + inOrder.verify(mockDao, times(2)).buildPayload(Matchers.eq(mockBuildData), Matchers.eq("http://my-jenkins-url"), Matchers.anyListOf(String.class)); + // the message at close time is generated by the script so no call to DAO for that + inOrder.verify(mockDao, times(2)).push("{\"data\":{},\"message\":[\"" + goodMsg + "\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + + // when creating the mock LogstashWriter by `createLogstashWriter` + verify(mockBuild).getCharset(); + + // building payload + verify(mockBuildData, times(3)).updateResult(); + } + @Test public void writeSuccessConnectionBroken() throws Exception { Mockito.doNothing().doThrow(new IOException("BOOM!")).doNothing().when(mockDao).push(anyString()); @@ -337,11 +413,12 @@ public void writeBuildLogGetLogError() throws Exception { List expectedErrorLines = Arrays.asList( "[logstash-plugin]: Unable to serialize log data.", "java.io.IOException: Unable to read log file"); - verify(mockDao).push("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); verify(mockDao).buildPayload(eq(mockBuildData), eq("http://my-jenkins-url"), logLinesCaptor.capture()); verify(mockBuildData).updateResult(); List actualLogLines = logLinesCaptor.getValue(); + String linesJSON = JSONArray.fromObject(actualLogLines).toString(); + verify(mockDao).push("{\"data\":{},\"message\":" + linesJSON + ",\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); assertThat("The exception was not sent to Logstash", actualLogLines.get(0), containsString(expectedErrorLines.get(0))); assertThat("The exception was not sent to Logstash", actualLogLines.get(1), containsString(expectedErrorLines.get(1))); diff --git a/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java b/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java index 2a9b8e60..9a0151e6 100644 --- a/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java +++ b/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java @@ -9,7 +9,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; -import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -87,7 +86,7 @@ public void before() throws Exception { when(mockBuild.getDescription()).thenReturn("Mock project for testing BuildData"); when(mockBuild.getParent()).thenReturn(mockProject); when(mockBuild.getNumber()).thenReturn(123456); - when(mockBuild.getTimestamp()).thenReturn(new GregorianCalendar()); + when(mockBuild.getTime()).thenReturn(new Date()); when(mockBuild.getRootBuild()).thenReturn(mockBuild); when(mockBuild.getBuildVariables()).thenReturn(Collections.emptyMap()); when(mockBuild.getSensitiveBuildVariables()).thenReturn(Collections.emptySet()); @@ -143,7 +142,7 @@ private void verifyMocks() throws Exception verify(mockBuild).getAction(AbstractTestResultAction.class); verify(mockBuild).getExecutor(); verify(mockBuild).getNumber(); - verify(mockBuild).getTimestamp(); + verify(mockBuild).getTime(); verify(mockBuild, times(4)).getRootBuild(); verify(mockBuild).getBuildVariables(); verify(mockBuild).getSensitiveBuildVariables();