Merge pull request #779 from scottsut/master

feat: 1.0.0-beta.1 release

core/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-beta.0</version>
+        <version>1.0.0-beta.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

core/src/main/java/datart/core/base/consts/Const.java

@@ -19,6 +19,8 @@
 package datart.core.base.consts;
 
 
+import java.util.regex.Pattern;
+
 public class Const {
 
     public static final byte VIZ_PUBLISH = 2;
@@ -32,7 +34,6 @@ public class Const {
     /**
      * 正则表达式
      */
-
     public static final String REG_EMAIL = "^\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*$";
 
     public static final String REG_USER_PASSWORD = ".{6,20}";
@@ -46,12 +47,15 @@ public class Const {
     //默认的变量引用符号
     public static final String DEFAULT_VARIABLE_QUOTE = "$";
     //变量匹配符
-    public static final String VARIABLE_EXP = "\\$\\w+\\$";
+    public static final Pattern VARIABLE_PATTERN = Pattern.compile("\\$\\S+\\$");
+    //变量正则模板
+    public static final String VARIABLE_PATTERN_TEMPLATE = "\\$%s\\$";
+
     /**
      * 权限变量
      */
-    //管理员权限，拥有所有数据的权限
-    public static final String ALL_PERMISSION = "@ALL_PERMISSION@";
+//    //管理员权限，拥有所有数据的权限
+//    public static final String ALL_PERMISSION = "@ALL_PERMISSION@";
 
     /**
      * Token Key

core/src/main/java/datart/core/base/consts/PlatformSettings.java

@@ -0,0 +1,25 @@
+/*
+ * Datart
+ * <p>
+ * Copyright 2021
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package datart.core.base.consts;
+
+public enum PlatformSettings {
+
+    // download record limit
+    DOWNLOAD_RECORD_LIMIT
+
+}

core/src/main/java/datart/core/base/exception/SqlParseError.java

@@ -0,0 +1,42 @@
+/*
+ * Datart
+ * <p>
+ * Copyright 2021
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package datart.core.base.exception;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@EqualsAndHashCode(callSuper = true)
+@Data
+public class SqlParseError extends BaseException {
+
+    public SqlParseError(Throwable e) {
+        super(e);
+    }
+
+    private String sql;
+
+    private String dbType;
+
+    public String toString() {
+        return "SQL:" + sql + " \r\n" +
+                "DB: " + dbType + " \r\n" +
+                "EXCEPTION:" + getMessage();
+    }
+
+}

core/src/main/java/datart/core/base/processor/ExtendProcessor.java

@@ -0,0 +1,4 @@
+package datart.core.base.processor;
+
+public interface ExtendProcessor {
+}

core/src/main/java/datart/core/base/processor/ProcessorResponse.java

@@ -0,0 +1,27 @@
+package datart.core.base.processor;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Builder
+@Data
+public class ProcessorResponse {
+
+    private boolean success;
+    private int errCode;
+    private String message;
+
+    public static ProcessorResponse success() {
+        return ProcessorResponse.builder()
+                .success(true)
+                .build();
+    }
+
+    public static ProcessorResponse failure(int errCode,String message) {
+        return ProcessorResponse.builder()
+                .success(false)
+                .errCode(errCode)
+                .message(message)
+                .build();
+    }
+}

core/src/main/java/datart/core/common/CSVParse.java

@@ -70,8 +70,14 @@ public List<List<Object>> parse() throws IOException {
         } else {
             types = inferDataType(records.get(1));
         }
-        return records.parallelStream().map(this::extractValues)
+        List<List<Object>> values = records.parallelStream().map(this::extractValues)
                 .collect(Collectors.toList());
+        // remove utf-8-with-bom char
+        String start = values.get(0).get(0).toString();
+        if (start.charAt(0) == '\uFEFF') {
+            values.get(0).set(0, start.substring(1));
+        }
+        return values;
     }
 
     private ValueType[] inferDataType(CSVRecord record) {

core/src/main/java/datart/core/common/RequestContext.java

@@ -0,0 +1,45 @@
+/*
+ * Datart
+ * <p>
+ * Copyright 2021
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package datart.core.common;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class RequestContext {
+
+    private static final InheritableThreadLocal<Map<String, Exception>> exceptions = new InheritableThreadLocal<>();
+
+    public static void putWarning(String name, Exception exception) {
+        Map<String, Exception> exceptionMap = exceptions.get();
+        if (exceptionMap == null) {
+            exceptionMap = new ConcurrentHashMap<>();
+            exceptions.set(exceptionMap);
+        }
+        exceptionMap.put(name, exception);
+    }
+
+    public static Map<String, Exception> getWarnings() {
+        return exceptions.get();
+    }
+
+    public static void clean() {
+        exceptions.remove();
+    }
+
+}

core/src/main/java/datart/core/data/provider/Column.java

@@ -29,9 +29,15 @@ public class Column implements Serializable {
     private String name;
 
     private ValueType type;
-    
+
     private String fmt;
 
+    private String pkDatabase;
+
+    private String pkTable;
+
+    private String pkColumn;
+
     public Column(String name, ValueType type) {
         this.name = name;
         this.type = type;

core/src/main/java/datart/core/data/provider/ScriptVariable.java

@@ -41,6 +41,9 @@ public class ScriptVariable extends TypedValue {
 
     private boolean expression;
 
+    // Permission variable valid flag, which is false when executed by the organization owner
+    private boolean disabled;
+
     @Override
     public String toString() {
         if (values == null) {

core/src/main/java/datart/core/data/provider/processor/DataProviderPostProcessor.java

@@ -0,0 +1,12 @@
+package datart.core.data.provider.processor;
+
+import datart.core.base.processor.ExtendProcessor;
+import datart.core.base.processor.ProcessorResponse;
+import datart.core.data.provider.DataProviderSource;
+import datart.core.data.provider.Dataframe;
+import datart.core.data.provider.ExecuteParam;
+import datart.core.data.provider.QueryScript;
+
+public interface DataProviderPostProcessor extends ExtendProcessor {
+    ProcessorResponse postRun(Dataframe frame, DataProviderSource config, QueryScript script, ExecuteParam executeParam);
+}

core/src/main/java/datart/core/data/provider/processor/DataProviderPreProcessor.java

@@ -0,0 +1,11 @@
+package datart.core.data.provider.processor;
+
+import datart.core.base.processor.ExtendProcessor;
+import datart.core.base.processor.ProcessorResponse;
+import datart.core.data.provider.DataProviderSource;
+import datart.core.data.provider.ExecuteParam;
+import datart.core.data.provider.QueryScript;
+
+public interface DataProviderPreProcessor extends ExtendProcessor {
+    ProcessorResponse preRun(DataProviderSource config, QueryScript script, ExecuteParam executeParam);
+}

core/src/main/java/datart/core/entity/Schedule.java

@@ -1,10 +1,9 @@
 package datart.core.entity;
 
+import java.util.Date;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 
-import java.util.Date;
-
 @Data
 @EqualsAndHashCode(callSuper = true)
 public class Schedule extends BaseEntity {

core/src/main/java/datart/core/mappers/ext/OrgSettingsMapperExt.java

@@ -16,4 +16,9 @@ public interface OrgSettingsMapperExt extends OrgSettingsMapper {
     })
     List<OrgSettings> selectByOrg(String orgId);
 
+    @Select({
+            "SELECT * FROM org_settings t WHERE t.org_id = #{orgId} and `type`=#{type}"
+    })
+    OrgSettings selectByOrgAndType(String orgId, String type);
+
 }

data-providers/file-data-provider/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-beta.0</version>
+        <version>1.0.0-beta.1</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

data-providers/http-data-provider/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-beta.0</version>
+        <version>1.0.0-beta.1</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

data-providers/jdbc-data-provider/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-beta.0</version>
+        <version>1.0.0-beta.1</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

data-providers/jdbc-data-provider/src/main/java/datart/data/provider/JdbcDataProvider.java

@@ -49,6 +49,8 @@ public class JdbcDataProvider extends DataProvider {
 
     public static final String DRIVER_CLASS = "driverClass";
 
+    public static final String ENABLE_SPECIAL_SQL = "enableSpecialSQL";
+
     private static final String I18N_PREFIX = "config.template.jdbc.";
 
     /**
@@ -113,6 +115,13 @@ private JdbcProperties conv2JdbcProperties(DataProviderSource config) {
         jdbcProperties.setDriverClass(StringUtils.isBlank(driverClass) ?
                 ProviderFactory.getJdbcDriverInfo(jdbcProperties.getDbType()).getDriverClass() :
                 driverClass);
+
+        Object enableSpecialSQL = config.getProperties().get(ENABLE_SPECIAL_SQL);
+
+        if (enableSpecialSQL != null && "true".equals(enableSpecialSQL.toString())) {
+            jdbcProperties.setEnableSpecialSql(true);
+        }
+
         Object properties = config.getProperties().get("properties");
         if (properties != null) {
             if (properties instanceof Map) {

data-providers/jdbc-data-provider/src/main/java/datart/data/provider/jdbc/DataSourceFactoryDruidImpl.java

@@ -57,7 +57,7 @@ private Properties configDataSource(JdbcProperties properties) {
         }
         pro.setProperty(DruidDataSourceFactory.PROP_MAXWAIT, JdbcDataProvider.DEFAULT_MAX_WAIT.toString());
 
-        pro.setProperty("druid.mysql.usePingMethod", "false");
+        System.setProperty("druid.mysql.usePingMethod", "false");
 
         // url properties
 //        pro.setProperty(DruidDataSourceFactory.PROP_CONNECTIONPROPERTIES, "useUnicode=true;characterEncoding=utf8;characterSetResults=utf8");