Merge pull request #151 from qq254963746/develop

add mapdb FailStore and other upgrade

README.md

@@ -295,7 +295,7 @@ public class LTSSpringConfig implements ApplicationContextAware {
 |job.max.retry.times|可选|10|JobTracker|addConfig("job.max.retry.times", "xx")|任务的最大重试次数|
 |lts.monitor.url|可选|无|JobTracker,TaskTracker|addConfig("lts.monitor.url", "xx")|监控中心地址，也就是LTS-Admin地址，如 http://localhost:8081|
 |stop.working|可选|false|TaskTracker|addConfig("stop.working", "true")|主要用于当TaskTracker与JobTracker出现网络隔离的时候，超过一定时间隔离之后，TaskTracker自动停止当前正在运行的任务|
-|job.fail.store|可选|leveldb|JobClient,TaskTracker|addConfig("job.fail.store", "leveldb")|可选值:leveldb(默认), rocksdb, berkeleydb, FailStore实现|
+|job.fail.store|可选|leveldb|JobClient,TaskTracker|addConfig("job.fail.store", "leveldb")|可选值:leveldb(默认), rocksdb, berkeleydb, mapdb FailStore实现, leveldb有问题的同学,可以试试mapdb|
 |lazy.job.logger|可选|false|JobTracker|addConfig("lazy.job.logger", "true")|可选值:ture,false, 是否延迟批量刷盘日志, 如果启用，采用队列的方式批量将日志刷盘(在应用关闭的时候，可能会造成日志丢失)|
 |dataPath|可选|user.home|JobClient,TaskTracker,JobTracker|setDataPath("xxxx")|FailStore文件存储路径及其它数据存储路径|
 |lts.monitor.interval|可选|1|JobClient,TaskTracker,JobTracker|addConfig("lts.monitor.interval", "2")|分钟，整数，建议1-5分钟|

lts-admin/src/main/java/com/lts/web/support/csrf/CSRFHandlerInterceptor.java

@@ -0,0 +1,43 @@
+package com.lts.web.support.csrf;
+
+import org.springframework.web.servlet.handler.HandlerInterceptorAdapter;
+import org.springframework.web.servlet.resource.DefaultServletHttpRequestHandler;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+/**
+ * A Spring MVC <code>HandlerInterceptor</code> which is responsible to enforce CSRF token validity on incoming posts
+ * requests. The interceptor should be registered with Spring MVC servlet using the following syntax:
+ * <p/>
+ * <mvc:interceptors>
+ * <bean class="com.lts.web.support.csrf.CSRFHandlerInterceptor"/>
+ * </mvc:interceptors>
+ *
+ * @author Robert HG ([email protected]) on 11/10/15.
+ */
+public class CSRFHandlerInterceptor extends HandlerInterceptorAdapter {
+
+    @Override
+    public boolean preHandle(HttpServletRequest request,
+                             HttpServletResponse response, Object handler) throws Exception {
+
+        if (handler instanceof DefaultServletHttpRequestHandler) {
+            return true;
+        }
+
+        if (request.getMethod().equalsIgnoreCase("GET")) {
+            return true;
+        } else {
+            String sessionToken = CSRFTokenManager.getToken(request.getSession());
+            String requestToken = CSRFTokenManager.getToken(request);
+            // 检查 csrf token是否正确
+            if (sessionToken.equals(requestToken)) {
+                return true;
+            } else {
+                response.sendError(HttpServletResponse.SC_FORBIDDEN, "Bad or missing CSRF value");
+                return false;
+            }
+        }
+    }
+}

lts-admin/src/main/java/com/lts/web/support/csrf/CSRFTokenManager.java

@@ -0,0 +1,40 @@
+package com.lts.web.support.csrf;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpSession;
+import java.util.UUID;
+
+/**
+ * @author Robert HG ([email protected]) on 11/10/15.
+ */
+public final class CSRFTokenManager {
+
+    static final String CSRF_PARAM_NAME = "csrfToken";
+
+    public final static String CSRF_TOKEN_FOR_SESSION_ATTR_NAME = CSRFTokenManager.class.getSimpleName() + ".token";
+
+    private CSRFTokenManager() {
+    }
+
+    public static String getToken(HttpSession session) {
+        String token = null;
+
+        synchronized (session) {
+            token = (String) session.getAttribute(CSRF_TOKEN_FOR_SESSION_ATTR_NAME);
+            if (null == token) {
+                token = UUID.randomUUID().toString();
+                session.setAttribute(CSRF_TOKEN_FOR_SESSION_ATTR_NAME, token);
+            }
+        }
+        return token;
+    }
+
+    public static String getToken(HttpServletRequest request) {
+        String token = request.getParameter(CSRF_PARAM_NAME);
+        if (token == null || "".equals(token)) {
+            token = request.getHeader(CSRF_PARAM_NAME);
+        }
+        return token;
+    }
+
+}

lts-admin/src/main/java/com/lts/web/support/csrf/CSRFTool.java

@@ -0,0 +1,16 @@
+package com.lts.web.support.csrf;
+
+import javax.servlet.http.HttpServletRequest;
+
+/**
+ * 配置在 velocity tools 中
+ *
+ * <input type="hidden" name="csrfToken" value="$csrfTool.getToken($request)"/>
+ *
+ * @author Robert HG ([email protected]) on 11/10/15.
+ */
+public class CSRFTool {
+    public static String getToken(HttpServletRequest request) {
+        return CSRFTokenManager.getToken(request.getSession());
+    }
+}

lts-admin/src/main/java/com/lts/web/support/xss/XssFilter.java

@@ -0,0 +1,28 @@
+package com.lts.web.support.xss;
+
+import javax.servlet.*;
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+
+/**
+ * @author Robert HG ([email protected]) on 11/10/15.
+ */
+public class XssFilter implements Filter {
+
+    @Override
+    public void init(FilterConfig filterConfig) throws ServletException {
+
+    }
+
+    @Override
+    public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
+
+        chain.doFilter(new XssHttpServletRequestWrapper((HttpServletRequest) request), response);
+
+    }
+
+    @Override
+    public void destroy() {
+
+    }
+}

lts-admin/src/main/java/com/lts/web/support/xss/XssHttpServletRequestWrapper.java

@@ -0,0 +1,70 @@
+package com.lts.web.support.xss;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.regex.Pattern;
+
+/**
+ * @author Robert HG ([email protected]) on 11/10/15.
+ */
+public class XssHttpServletRequestWrapper extends HttpServletRequestWrapper {
+
+    public XssHttpServletRequestWrapper(HttpServletRequest servletRequest) {
+        super(servletRequest);
+    }
+
+    public String[] getParameterValues(String parameter) {
+        String[] values = super.getParameterValues(parameter);
+        if (values == null) {
+            return null;
+        }
+        int count = values.length;
+        String[] encodedValues = new String[count];
+        for (int i = 0; i < count; i++) {
+            encodedValues[i] = cleanXSS(values[i]);
+        }
+        return encodedValues;
+    }
+
+    public String getParameter(String parameter) {
+        String value = super.getParameter(parameter);
+        if (value == null) {
+            return null;
+        }
+        return cleanXSS(value);
+    }
+
+    public String getHeader(String name) {
+        String value = super.getHeader(name);
+        if (value == null)
+            return null;
+        return cleanXSS(value);
+    }
+
+    private static final List<Pattern> PATTERNS = new CopyOnWriteArrayList<Pattern>();
+
+    static {
+        PATTERNS.add(Pattern.compile("<script>(.*?)</script>", Pattern.CASE_INSENSITIVE));
+        PATTERNS.add(Pattern.compile("src[\r\n]*=[\r\n]*\\\'(.*?)\\\'", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL));
+        PATTERNS.add(Pattern.compile("src[\r\n]*=[\r\n]*\\\"(.*?)\\\"", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL));
+        PATTERNS.add(Pattern.compile("</script>", Pattern.CASE_INSENSITIVE));
+        PATTERNS.add(Pattern.compile("<script(.*?)>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL));
+        PATTERNS.add(Pattern.compile("eval\\((.*?)\\)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL));
+        PATTERNS.add(Pattern.compile("e­xpression\\((.*?)\\)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL));
+        PATTERNS.add(Pattern.compile("javascript:", Pattern.CASE_INSENSITIVE));
+        PATTERNS.add(Pattern.compile("vbscript:", Pattern.CASE_INSENSITIVE));
+        PATTERNS.add(Pattern.compile("onload(.*?)=", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL));
+    }
+
+    private String cleanXSS(String value) {
+        if (value != null) {
+            for (Pattern pattern : PATTERNS) {
+                value = pattern.matcher(value).replaceAll("");
+            }
+        }
+        return value;
+    }
+
+}

lts-admin/src/main/webapp/WEB-INF/views/templates/jobLogger.vm

@@ -94,6 +94,7 @@
             <th>日志记录时间</th>
             <th data-hide="all">日志创建时间</th>
             <th>执行节点组</th>
+            <th data-hide="all">执行节点标识</th>
             <th data-hide="phone,tablet">提交节点组</th>
             <th>日志类型</th>
             <th>执行结果</th>
@@ -114,6 +115,7 @@
             <td>{{row.logTime | dateFormat:'yyyy-MM-dd HH:mm:ss'}}</td>
             <td>{{row.gmtCreated | dateFormat:'yyyy-MM-dd HH:mm:ss'}}</td>
             <td>{{row.taskTrackerNodeGroup}}</td>
+            <td>{{row.taskTrackerIdentity}}</td>
             <td>{{row.submitNodeGroup}}</td>
             <td>{{row.logType | format:'logTypeLabel'}}</td>
             <td>{{row.success | format:'successLabel'}}</td>
@@ -131,7 +133,7 @@
         {{/each}}
         {{if results == 0}}
         <tr>
-            <td colspan="15">暂无数据</td>
+            <td colspan="16">暂无数据</td>
         </tr>
         {{/if}}
         </tbody>

lts-core/pom.xml

@@ -115,5 +115,10 @@
             <artifactId>hessian</artifactId>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mapdb</groupId>
+            <artifactId>mapdb</artifactId>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 </project>

lts-core/src/main/java/com/lts/core/commons/io/UnsafeByteArrayInputStream.java

@@ -0,0 +1,84 @@
+package com.lts.core.commons.io;
+
+import java.io.InputStream;
+
+/**
+ * @author Robert HG ([email protected])
+ */
+public class UnsafeByteArrayInputStream extends InputStream {
+
+    protected byte buf[];
+
+    protected int pos;
+
+    protected int mark = 0;
+
+    protected int count;
+
+    public UnsafeByteArrayInputStream(byte buf[]) {
+        this.buf = buf;
+        this.pos = 0;
+        this.count = buf.length;
+    }
+
+    public UnsafeByteArrayInputStream(byte buf[], int offset, int length) {
+        this.buf = buf;
+        this.pos = offset;
+        this.count = Math.min(offset + length, buf.length);
+        this.mark = offset;
+    }
+
+    public int read() {
+        return (pos < count) ? (buf[pos++] & 0xff) : -1;
+    }
+
+    public int read(byte b[], int off, int len) {
+        if (b == null) {
+            throw new NullPointerException();
+        } else if (off < 0 || len < 0 || len > b.length - off) {
+            throw new IndexOutOfBoundsException();
+        }
+
+        if (pos >= count) {
+            return -1;
+        }
+
+        int avail = count - pos;
+        if (len > avail) {
+            len = avail;
+        }
+        if (len <= 0) {
+            return 0;
+        }
+        System.arraycopy(buf, pos, b, off, len);
+        pos += len;
+        return len;
+    }
+
+    public long skip(long n) {
+        long k = count - pos;
+        if (n < k) {
+            k = n < 0 ? 0 : n;
+        }
+
+        pos += k;
+        return k;
+    }
+
+    public int available() {
+        return count - pos;
+    }
+
+    public boolean markSupported() {
+        return true;
+    }
+
+    public void mark(int readAheadLimit) {
+        mark = pos;
+    }
+
+    public void reset() {
+        pos = mark;
+    }
+
+}