diff --git a/.github/workflows/ci_build.yml b/.github/workflows/ci_build.yml index 9c703d4517b..ae6d3aefc43 100644 --- a/.github/workflows/ci_build.yml +++ b/.github/workflows/ci_build.yml @@ -107,7 +107,8 @@ jobs: - name: Build with Maven run: | - mvn --batch-mode --update-snapshots -e -V clean install -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + mvn -B -U -ntp -e -V -T 1C clean install -pl '!inlong-distribution' -am -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + mvn install -pl inlong-distribution -am -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true env: CI: false diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 21d56a8b937..2d9c157893e 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -115,12 +115,12 @@ jobs: sudo swapon /swapfile - name: Build with Maven - run: mvn --batch-mode --update-snapshots -e -V clean install -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + run: mvn -B -U -ntp -e -V clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -pl '!inlong-distribution' -am -T 1C -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 env: CI: false - name: Unit test with Maven - run: mvn --batch-mode --update-snapshots -e -V test -pl !:sort-end-to-end-tests-v1.15,!:sort-end-to-end-tests-v1.13,!:sort-end-to-end-tests-v1.18 + run: mvn -B -U -e -V test -pl !:sort-end-to-end-tests-v1.15,!:sort-end-to-end-tests-v1.13,!:sort-end-to-end-tests-v1.18 env: CI: false diff --git a/.github/workflows/ci_ut_flink13.yml b/.github/workflows/ci_ut_flink13.yml index 3889e27d913..9084e34a3aa 100644 --- a/.github/workflows/ci_ut_flink13.yml +++ b/.github/workflows/ci_ut_flink13.yml @@ -84,12 +84,12 @@ jobs: restore-keys: ${{ runner.os }}-inlong-flink13 - name: Build for Flink 1.13 with Maven - run: mvn --update-snapshots -e -V clean install -U -pl :sort-core,:sort-end-to-end-tests-v1.13 -am -Pv1.13 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + run: mvn -B -U -ntp -e -V -T 1C clean install -pl :sort-core,:sort-end-to-end-tests-v1.13 -am -Pv1.13 -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 env: CI: false - name: Unit test for Flink 1.13 with Maven - run: mvn --update-snapshots -e -V verify -pl :sort-core,:sort-end-to-end-tests-v1.13 -am -Pv1.13 + run: mvn -U -e -V verify -pl :sort-core,:sort-end-to-end-tests-v1.13 -am -Pv1.13 env: CI: false diff --git a/.github/workflows/ci_ut_flink15.yml b/.github/workflows/ci_ut_flink15.yml index d5200b64756..53b88957e0b 100644 --- a/.github/workflows/ci_ut_flink15.yml +++ b/.github/workflows/ci_ut_flink15.yml @@ -84,12 +84,12 @@ jobs: restore-keys: ${{ runner.os }}-inlong-flink15 - name: Build for Flink 1.15 with Maven - run: mvn --update-snapshots -e -V clean install -U -pl :sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + run: mvn -B -U -ntp -e -V -T 1C clean install -pl :sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15 -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 env: CI: false - name: Unit test for Flink 1.15 with Maven - run: mvn --update-snapshots -e -V verify -pl :sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15 + run: mvn -U -e -V verify -pl :sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15 env: CI: false diff --git a/.github/workflows/ci_ut_flink18.yml b/.github/workflows/ci_ut_flink18.yml index bec63302831..d86ea53bfcb 100644 --- a/.github/workflows/ci_ut_flink18.yml +++ b/.github/workflows/ci_ut_flink18.yml @@ -84,12 +84,12 @@ jobs: restore-keys: ${{ runner.os }}-inlong-flink18 - name: Build for Flink 1.18 with Maven - run: mvn --update-snapshots -e -V clean install -U -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + run: mvn -B -U -ntp -e -V -T 1C clean install -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 env: CI: false - name: Unit test for Flink 1.18 with Maven - run: mvn --update-snapshots -e -V verify -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 + run: mvn -U -e -V verify -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 env: CI: false diff --git a/.github/workflows/codeql_analysis.yml b/.github/workflows/codeql_analysis.yml index d9c8b453801..a303eeb7805 100644 --- a/.github/workflows/codeql_analysis.yml +++ b/.github/workflows/codeql_analysis.yml @@ -92,7 +92,7 @@ jobs: - name: Build with Maven run: | - mvn --batch-mode --update-snapshots -e -V clean install -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + mvn -B -U -ntp -e -V -T 1C clean install -pl '!inlong-distribution' -am -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 env: CI: false diff --git a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/CommonConstants.java b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/CommonConstants.java index 53a5bd976ce..757db41afdd 100644 --- a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/CommonConstants.java +++ b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/CommonConstants.java @@ -68,9 +68,6 @@ public class CommonConstants { public static final String PROXY_SENDER_MAX_RETRY = "proxy.sender.maxRetry"; public static final int DEFAULT_PROXY_SENDER_MAX_RETRY = 5; - public static final String PROXY_IS_FILE = "proxy.isFile"; - public static final boolean DEFAULT_IS_FILE = false; - public static final String PROXY_CLIENT_IO_THREAD_NUM = "client.iothread.num"; public static final int DEFAULT_PROXY_CLIENT_IO_THREAD_NUM = Runtime.getRuntime().availableProcessors(); diff --git a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/AgentStatusManager.java b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/AgentStatusManager.java index 307c79f1dab..7292c1a5772 100644 --- a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/AgentStatusManager.java +++ b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/AgentStatusManager.java @@ -42,7 +42,6 @@ import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.List; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import static org.apache.inlong.agent.constant.AgentConstants.AGENT_CLUSTER_NAME; @@ -168,7 +167,7 @@ private void doSendStatusMsg(DefaultMessageSender sender) { INLONG_AGENT_SYSTEM, INLONG_AGENT_STATUS, AgentUtils.getCurrentTime(), - "", 30, TimeUnit.SECONDS); + ""); if (ret != SendResult.OK) { LOGGER.error("send status failed: ret {}", ret); } diff --git a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/FileStaticManager.java b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/FileStaticManager.java index abda6a2eab4..774dee42471 100644 --- a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/FileStaticManager.java +++ b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/FileStaticManager.java @@ -34,7 +34,6 @@ import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; import static org.apache.inlong.agent.constant.AgentConstants.AGENT_CLUSTER_NAME; import static org.apache.inlong.agent.constant.AgentConstants.AGENT_CLUSTER_TAG; @@ -135,7 +134,7 @@ private void doSendStaticMsg(DefaultMessageSender sender) { INLONG_AGENT_SYSTEM, INLONG_FILE_STATIC, AgentUtils.getCurrentTime(), - "", 30, TimeUnit.SECONDS); + ""); if (ret != SendResult.OK) { LOGGER.error("send static failed: ret {}", ret); } diff --git a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/HeartbeatManager.java b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/HeartbeatManager.java index 2a16bffb0a1..ead9439114a 100644 --- a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/HeartbeatManager.java +++ b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/HeartbeatManager.java @@ -74,7 +74,6 @@ private HeartbeatManager(AgentManager agentManager) { httpManager = new HttpManager(conf); baseManagerUrl = httpManager.getBaseUrl(); reportHeartbeatUrl = buildReportHeartbeatUrl(baseManagerUrl); - createMessageSender(); } public static HeartbeatManager getInstance(AgentManager agentManager) { @@ -121,9 +120,6 @@ private Runnable heartbeatReportThread() { if (LOGGER.isDebugEnabled()) { LOGGER.debug(" {} report heartbeat to manager", heartbeatMsg); } - if (sender == null) { - createMessageSender(); - } AgentStatusManager.sendStatusMsg(sender); FileStaticManager.sendStaticMsg(sender); } catch (Throwable e) { @@ -205,6 +201,7 @@ private void createMessageSender() { proxyClientConfig.setAliveConnections(CommonConstants.DEFAULT_PROXY_ALIVE_CONNECTION_NUM); proxyClientConfig.setIoThreadNum(CommonConstants.DEFAULT_PROXY_CLIENT_IO_THREAD_NUM); proxyClientConfig.setProtocolType(ProtocolType.TCP); + proxyClientConfig.setRequestTimeoutMs(30000L); ThreadFactory SHARED_FACTORY = new DefaultThreadFactory("agent-sender-manager-heartbeat", Thread.currentThread().isDaemon()); sender = new DefaultMessageSender(proxyClientConfig, SHARED_FACTORY); diff --git a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java index cb92576c3c2..de05a9d0f72 100755 --- a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java +++ b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java @@ -80,6 +80,7 @@ public class ModuleManager extends AbstractDaemon { public static final String LOCAL_CONFIG_FILE = "modules.json"; private static final Logger LOGGER = LoggerFactory.getLogger(ModuleManager.class); public static final int MAX_MODULE_SIZE = 10; + public static final int CHECK_PROCESS_TIMES = 20; private final InstallerConfiguration conf; private final String confPath; private final BlockingQueue configQueue; @@ -345,7 +346,7 @@ private void checkModules() { } break; case INSTALLED: - if (!isProcessAllStarted(module)) { + if (!isProcessAllStarted(module, CHECK_PROCESS_TIMES)) { LOGGER.info("module {}({}) process not all started try to start", module.getId(), module.getName()); if (!startModule(module)) { @@ -495,7 +496,7 @@ private boolean startModule(ModuleConfig module) { String ret = ExcuteLinux.exeCmd(module.getStartCommand()); LOGGER.info("start module {}({}) proc[{}] return {} ", module.getId(), module.getName(), i, ret); } - if (isProcessAllStarted(module)) { + if (isProcessAllStarted(module, CHECK_PROCESS_TIMES)) { LOGGER.info("start module {}({}) success", module.getId(), module.getName()); return true; } else { @@ -517,21 +518,27 @@ private void uninstallModule(ModuleConfig module) { LOGGER.info("uninstall module {}({}) return {} ", module.getId(), module.getName(), ret); } - private boolean isProcessAllStarted(ModuleConfig module) { - String ret = ExcuteLinux.exeCmd(module.getCheckCommand()); - if (ret == null) { - LOGGER.error("get module {}({}) process num failed", module.getId(), module.getName()); - return false; - } - String[] processArray = ret.split("\n"); - int cnt = 0; - for (int i = 0; i < processArray.length; i++) { - if (processArray[i].length() > 0) { - cnt++; + private boolean isProcessAllStarted(ModuleConfig module, int times) { + for (int check = 0; check < times; check++) { + AgentUtils.silenceSleepInSeconds(1); + String ret = ExcuteLinux.exeCmd(module.getCheckCommand()); + if (ret == null) { + LOGGER.error("[{}] get module {}({}) process num failed", check, module.getId(), module.getName()); + continue; + } + String[] processArray = ret.split("\n"); + int cnt = 0; + for (int i = 0; i < processArray.length; i++) { + if (processArray[i].length() > 0) { + cnt++; + } + } + LOGGER.info("[{}] get module {}({}) process num {}", check, module.getId(), module.getName(), cnt); + if (cnt >= module.getProcessesNum()) { + return true; } } - LOGGER.info("get module {}({}) process num {}", module.getId(), module.getName(), cnt); - return cnt >= module.getProcessesNum(); + return false; } private boolean downloadModule(ModuleConfig module) { diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java index ecd2383bf03..240fc7357f5 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -138,7 +138,7 @@ private void mockFunctions() { PowerMockito.doAnswer(invocation -> { ModuleConfig module = invocation.getArgument(0); return true; - }).when(manager, "isProcessAllStarted", Mockito.any()); + }).when(manager, "isProcessAllStarted", Mockito.any(), Mockito.anyInt()); PowerMockito.doReturn(null).when(manager, "getHttpManager", Mockito.any()); } catch (Exception e) { diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java index 984baf6de63..a37a171a372 100755 --- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java +++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java @@ -87,7 +87,6 @@ public class SenderManager { private final int aliveConnectionNum; private final boolean isCompress; private final int msgType; - private final boolean isFile; private final long maxSenderTimeout; private final int maxSenderRetry; private final long retrySleepTime; @@ -133,7 +132,6 @@ public SenderManager(InstanceProfile profile, String inlongGroupId, String sourc CommonConstants.PROXY_SENDER_MAX_RETRY, CommonConstants.DEFAULT_PROXY_SENDER_MAX_RETRY); retrySleepTime = agentConf.getLong( CommonConstants.PROXY_RETRY_SLEEP, CommonConstants.DEFAULT_PROXY_RETRY_SLEEP); - isFile = profile.getBoolean(CommonConstants.PROXY_IS_FILE, CommonConstants.DEFAULT_IS_FILE); ioThreadNum = profile.getInt(CommonConstants.PROXY_CLIENT_IO_THREAD_NUM, CommonConstants.DEFAULT_PROXY_CLIENT_IO_THREAD_NUM); enableBusyWait = profile.getBoolean(CommonConstants.PROXY_CLIENT_ENABLE_BUSY_WAIT, @@ -200,7 +198,6 @@ private void createMessageSender() throws Exception { ProxyClientConfig proxyClientConfig = new ProxyClientConfig(managerAddr, inlongGroupId, authSecretId, authSecretKey); proxyClientConfig.setTotalAsyncCallbackSize(totalAsyncBufSize); - proxyClientConfig.setFile(isFile); proxyClientConfig.setAliveConnections(aliveConnectionNum); proxyClientConfig.setIoThreadNum(ioThreadNum); diff --git a/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/HttpUtils.java b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/HttpUtils.java index 34c60dc66b3..694674dbf89 100644 --- a/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/HttpUtils.java +++ b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/HttpUtils.java @@ -98,8 +98,9 @@ public static String httpGet(String component, String url, String secretId, Stri return responseStr; } } - } catch (Exception e) { - LOGGER.error("Send get request has exception", e); + } catch (Throwable e) { + LOGGER.error("Http request url = {}, secretId = {}, secretKey = {}, component = {} has exception!", url, + secretId, secretKey, component, e); } return null; } diff --git a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/send/SenderManager.java b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/send/SenderManager.java index b795b7aa981..97f2baf33cc 100644 --- a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/send/SenderManager.java +++ b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/send/SenderManager.java @@ -77,15 +77,16 @@ public void closeSocket() { public boolean checkSocket() { if (socket.isClosed() || !socket.isConnected()) { + InetSocketAddress inetSocketAddress = null; try { - InetSocketAddress inetSocketAddress = ProxyManager.getInstance().getInetSocketAddress(); + inetSocketAddress = ProxyManager.getInstance().getInetSocketAddress(); if (inetSocketAddress == null) { LOGGER.error("Audit proxy address is null!"); return false; } reconnect(inetSocketAddress, auditConfig.getSocketTimeout()); } catch (IOException exception) { - LOGGER.error("Connect to audit proxy {} has exception!", socket.getInetAddress(), exception); + LOGGER.error("Connect to audit proxy {} has exception!", inetSocketAddress, exception); return false; } } diff --git a/inlong-audit/audit-service/pom.xml b/inlong-audit/audit-service/pom.xml index e73e77b70a3..f04dcfe680a 100644 --- a/inlong-audit/audit-service/pom.xml +++ b/inlong-audit/audit-service/pom.xml @@ -228,7 +228,7 @@ maven-shade-plugin true - true + false *:* diff --git a/inlong-audit/audit-store/pom.xml b/inlong-audit/audit-store/pom.xml index 60d69a3e191..74e46ff3176 100644 --- a/inlong-audit/audit-store/pom.xml +++ b/inlong-audit/audit-store/pom.xml @@ -366,7 +366,7 @@ maven-shade-plugin true - true + false *:* diff --git a/inlong-common/src/main/java/org/apache/inlong/common/msg/AttributeConstants.java b/inlong-common/src/main/java/org/apache/inlong/common/msg/AttributeConstants.java index ceaeef3054e..3402fc64555 100644 --- a/inlong-common/src/main/java/org/apache/inlong/common/msg/AttributeConstants.java +++ b/inlong-common/src/main/java/org/apache/inlong/common/msg/AttributeConstants.java @@ -21,6 +21,7 @@ public interface AttributeConstants { String SEPARATOR = "&"; String KEY_VALUE_SEPARATOR = "="; + String LINE_FEED_SEP = "\n"; /** * group id diff --git a/inlong-dashboard/package-lock.json b/inlong-dashboard/package-lock.json index efa9d0a4132..599ba438118 100644 --- a/inlong-dashboard/package-lock.json +++ b/inlong-dashboard/package-lock.json @@ -13290,6 +13290,11 @@ "whatwg-fetch": "^3.6.2" } }, + "react-csv": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/react-csv/-/react-csv-2.2.2.tgz", + "integrity": "sha512-RG5hOcZKZFigIGE8LxIEV/OgS1vigFQT4EkaHeKgyuCbUAu9Nbd/1RYq++bJcJJ9VOqO/n9TZRADsXNDR4VEpw==" + }, "react-dev-utils": { "version": "12.0.1", "resolved": "https://registry.npmjs.org/react-dev-utils/-/react-dev-utils-12.0.1.tgz", diff --git a/inlong-dashboard/package.json b/inlong-dashboard/package.json index c287ad670b4..7c61492967e 100644 --- a/inlong-dashboard/package.json +++ b/inlong-dashboard/package.json @@ -14,6 +14,7 @@ "nprogress": "^0.2.0", "path-to-regexp": "^6.2.0", "react": "17.0.1", + "react-csv": "^2.2.2", "react-dom": "17.0.1", "react-i18next": "^11.10.0", "react-redux": "^7.2.0", diff --git a/inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts b/inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts index 970bd181a7e..b89a77ada9b 100644 --- a/inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts +++ b/inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts @@ -96,6 +96,14 @@ export class SourceDefaultInfo implements DataWithBackend, RenderRow, RenderList @I18n('meta.Sources.Type') sourceType: string; + @FieldDecorator({ + type: 'select', + hidden: true, + }) + @IngestionField() + @I18n('meta.Sources.File.ClusterName') + readonly clusterTag: string; + @FieldDecorator({ type: 'input', rules: [ diff --git a/inlong-dashboard/src/plugins/sources/defaults/File.ts b/inlong-dashboard/src/plugins/sources/defaults/File.ts index 81ce7e99d10..5cbfa61b0b4 100644 --- a/inlong-dashboard/src/plugins/sources/defaults/File.ts +++ b/inlong-dashboard/src/plugins/sources/defaults/File.ts @@ -49,6 +49,7 @@ export default class PulsarSource data: { keyword, type: 'AGENT', + clusterTag: values.clusterTag, pageNum: 1, pageSize: 10, }, @@ -138,7 +139,13 @@ export default class PulsarSource @FieldDecorator({ type: 'input', tooltip: i18n.t('meta.Sources.File.FilePathHelp'), - rules: [{ required: true }], + rules: [ + { required: true }, + { + pattern: /^\S*$/, + message: i18n.t('meta.Sources.File.FilePathPatternHelp'), + }, + ], props: values => ({ disabled: Boolean(values.id), }), diff --git a/inlong-dashboard/src/plugins/sources/defaults/Kafka.ts b/inlong-dashboard/src/plugins/sources/defaults/Kafka.ts index 1c11b35ab46..7d56aaeb187 100644 --- a/inlong-dashboard/src/plugins/sources/defaults/Kafka.ts +++ b/inlong-dashboard/src/plugins/sources/defaults/Kafka.ts @@ -48,6 +48,7 @@ export default class KafkaSource data: { keyword, type: 'AGENT', + clusterTag: values.clusterTag, pageNum: 1, pageSize: 10, }, diff --git a/inlong-dashboard/src/plugins/sources/defaults/Mongodb.ts b/inlong-dashboard/src/plugins/sources/defaults/Mongodb.ts index 3f2b614ad2c..dae65463166 100644 --- a/inlong-dashboard/src/plugins/sources/defaults/Mongodb.ts +++ b/inlong-dashboard/src/plugins/sources/defaults/Mongodb.ts @@ -48,6 +48,7 @@ export default class MongodbSource data: { keyword, type: 'AGENT', + clusterTag: values.clusterTag, pageNum: 1, pageSize: 10, }, diff --git a/inlong-dashboard/src/plugins/sources/defaults/PostgreSQL.ts b/inlong-dashboard/src/plugins/sources/defaults/PostgreSQL.ts index bb42c816ab3..e3abec09c56 100644 --- a/inlong-dashboard/src/plugins/sources/defaults/PostgreSQL.ts +++ b/inlong-dashboard/src/plugins/sources/defaults/PostgreSQL.ts @@ -48,6 +48,7 @@ export default class PostgreSQLSource data: { keyword, type: 'AGENT', + clusterTag: values.clusterTag, pageNum: 1, pageSize: 10, }, diff --git a/inlong-dashboard/src/plugins/sources/defaults/Pulsar.ts b/inlong-dashboard/src/plugins/sources/defaults/Pulsar.ts index 3f3f0bf9c12..3a1a45156a0 100644 --- a/inlong-dashboard/src/plugins/sources/defaults/Pulsar.ts +++ b/inlong-dashboard/src/plugins/sources/defaults/Pulsar.ts @@ -49,6 +49,7 @@ export default class PulsarSource data: { keyword, type: 'AGENT', + clusterTag: values.clusterTag, pageNum: 1, pageSize: 10, }, diff --git a/inlong-dashboard/src/plugins/sync/common/SyncDefaultInfo.ts b/inlong-dashboard/src/plugins/sync/common/SyncDefaultInfo.ts index 1fbf1ecfdac..7bf18927d9a 100644 --- a/inlong-dashboard/src/plugins/sync/common/SyncDefaultInfo.ts +++ b/inlong-dashboard/src/plugins/sync/common/SyncDefaultInfo.ts @@ -73,7 +73,7 @@ export class SyncDefaultInfo implements DataWithBackend, RenderRow, RenderList { }) @ColumnDecorator() @I18n('meta.Synchronize.GroupOwners') - inCharges: string; + inCharges: string[]; @FieldDecorator({ type: 'radio', @@ -120,22 +120,51 @@ export class SyncDefaultInfo implements DataWithBackend, RenderRow, RenderList { @FieldDecorator({ type: 'radio', - initialValue: 0, + initialValue: 'Quartz', visible: values => values.inlongGroupMode === 2, rules: [{ required: true }], props: { options: [ { - label: i18n.t('meta.Synchronize.Conventional'), - value: 0, + label: i18n.t('meta.Synchronize.ScheduleQuartz'), + value: 'Quartz', }, { - label: i18n.t('meta.Synchronize.Crontab'), - value: 1, + label: i18n.t('meta.Synchronize.ScheduleAirflow'), + value: 'Airflow', + }, + { + label: i18n.t('meta.Synchronize.DolphinScheduler'), + value: 'DolphinScheduler', }, ], }, }) + @I18n('meta.Synchronize.ScheduleEngine') + scheduleEngine: string; + + @FieldDecorator({ + type: 'radio', + initialValue: 0, + visible: values => values.inlongGroupMode === 2, + + rules: [{ required: true }], + props: values => { + return { + disabled: values.scheduleEngine === 'DolphinScheduler', + options: [ + { + label: i18n.t('meta.Synchronize.Conventional'), + value: 0, + }, + { + label: i18n.t('meta.Synchronize.Crontab'), + value: 1, + }, + ], + }; + }, + }) @I18n('meta.Synchronize.ScheduleType') scheduleType: Number; @FieldDecorator({ diff --git a/inlong-dashboard/src/ui/locales/cn.json b/inlong-dashboard/src/ui/locales/cn.json index 23f83e31aa6..535063ee005 100644 --- a/inlong-dashboard/src/ui/locales/cn.json +++ b/inlong-dashboard/src/ui/locales/cn.json @@ -6,6 +6,7 @@ "basic.ConnectionSuccess": "连接成功", "basic.Save": "保存", "basic.Cancel": "取消", + "basic.Confirm": "确定", "basic.Create": "新建", "basic.Delete": "删除", "basic.DeleteConfirm": "确认删除吗?", @@ -33,6 +34,7 @@ "meta.Sources.File.ClusterName": "集群名称", "meta.Sources.File.FilePath": "⽂件路径", "meta.Sources.File.FilePathHelp": "必须是绝对路径，支持正则表达式，多个时以逗号分隔，如：/data/.*log", + "meta.Sources.File.FilePathPatternHelp": "文件路径不能包含空格", "meta.Sources.File.FileIpHelp": "请选择文件采集节点 IP，若 IP 不固定请选择 All", "meta.Sources.File.IpRule": "请输入正确的 IP 地址", "meta.Sources.File.VersionRule": "请输入正确的版本号", @@ -431,6 +433,10 @@ "meta.Synchronize.OfflineSync": "离线同步", "meta.Synchronize.SchedulingRules": "调度规则", "meta.Synchronize.ScheduleType": "调度类型", + "meta.Synchronize.ScheduleEngine": "调度引擎", + "meta.Synchronize.ScheduleQuartz": "Quartz", + "meta.Synchronize.ScheduleAirflow": "Airflow", + "meta.Synchronize.DolphinScheduler": "DolphinScheduler", "meta.Synchronize.Conventional": "常规", "meta.Synchronize.Crontab": "Crontab", "meta.Synchronize.DelayTime": "延迟时间", @@ -711,6 +717,7 @@ "pages.GroupDetail.Audit.Sink": "数据目标", "pages.GroupDetail.Audit.Total": "总计", "pages.GroupDetail.Audit.DatepickerRule": "超出可选范围", + "pages.GroupDetail.Audit.ExportCSV": "导出 CSV 文件", "pages.GroupDetail.Delay.QueryDate": "查询日期", "pages.GroupDetail.Delay.AverageTitle": "平均传输时延 (ms)", "pages.GroupDetail.Delay.RealTimeTitle": "传输时延 (ms)", @@ -815,6 +822,7 @@ "pages.Clusters.Name": "集群名称", "pages.Clusters.Tag": "集群标签", "pages.Clusters.InCharges": "责任人", + "pages.Clusters.Node.Description": "节点描述", "pages.Clusters.Description": "集群描述", "pages.Clusters.TestConnection": "测试连接", "pages.Clusters.Node.Name": "节点", @@ -847,15 +855,29 @@ "pages.Clusters.Node.SSHKey": "SSH 密钥", "pages.Clusters.Node.SSHPort": "SSH 端口", "pages.Clusters.Node.SSHKeyHelper": "请将公钥上传至 Agent 节点的 ~/.ssh/authorized_keys 文件中", + "pages.Clusters.Node.Version": "版本", "pages.Clusters.Node.Status": "状态", "pages.Clusters.Node.Status.Normal": "正常", "pages.Clusters.Node.Status.Timeout": "心跳超时", + "pages.Clusters.Node.Status.INSTALLING": "安装中", + "pages.Clusters.Node.Status.INSTALLFAILED": "安装失败", + "pages.Clusters.Node.Status.INSTALLSUCCESS": "安装成功", "pages.Clusters.Node.LastModifier": "最后操作", "pages.Clusters.Node.Creator": "创建人", "pages.Clusters.Node.Create": "新建节点", "pages.Clusters.Node.IpRule": "请输入正确的IP地址", "pages.Clusters.Node.PortRule": "请输入正确的端口", "pages.Clusters.Node.ProtocolTypeRule": "请输入正确的协议类型", + "pages.Clusters.Node.BatchUpdate": "批量操作", + "pages.Clusters.Node.BatchNum": "分批数", + "pages.Clusters.Node.Interval": "间隔", + "pages.Clusters.Node.Minute": "分钟", + "pages.Clusters.Node.OperationStatusQuery": "操作状态查询", + "pages.Clusters.Node.OperationType": "操作类型", + "pages.Clusters.Node.UpgradeAgentAndInstaller": "更新 Agent 和 Installer", + "pages.Clusters.Node.UpgradeAgent": "更新 Agent", + "pages.Clusters.Node.UpgradeInstaller": "更新 Installer", + "pages.Clusters.Node.RestartAgent": "重启 Agent", "pages.Clusters.Node.Online": "在线", "pages.Clusters.Pulsar.PulsarTenant": "默认租户", "pages.Clusters.Pulsar.TokenPlaceholder": "如果群集配置了令牌，则为必需", @@ -961,6 +983,7 @@ "pages.ModuleAudit.Id": "ID 查询", "pages.ModuleAudit.Metric": "指标查询", "pages.ModuleAudit.config.Ip": "机器 IP", + "pages.ModuleAudit.config.SubValue": "差异值", "pages.ModuleAudit.config.BenchmarkIndicator": "基准指标", "pages.ModuleAudit.config.ComparativeIndicators": "对比指标", "pages.ModuleAudit.config.InlongGroupId": "数据流组ID", diff --git a/inlong-dashboard/src/ui/locales/en.json b/inlong-dashboard/src/ui/locales/en.json index 89f3af4be20..b74f54e58e1 100644 --- a/inlong-dashboard/src/ui/locales/en.json +++ b/inlong-dashboard/src/ui/locales/en.json @@ -6,6 +6,7 @@ "basic.ConnectionSuccess": "Connection success", "basic.Save": "Save", "basic.Cancel": "Cancel", + "basic.Confirm": "Confirm", "basic.Create": "Create", "basic.Delete": "Delete", "basic.DeleteConfirm": "Are you sure to delete?", @@ -33,6 +34,7 @@ "meta.Sources.File.ClusterName": "Cluster name", "meta.Sources.File.FilePath": "File path", "meta.Sources.File.FilePathHelp": "Must be an absolute path and support regular expressions, such as: /data/.*log", + "meta.Sources.File.FilePathPatternHelp": "File path cannot contain spaces", "meta.Sources.File.FileIpHelp": "Please select the file collection node IP. If the IP is not fixed, please select All", "meta.Sources.File.IpRule": "Please enter the IP address correctly", "meta.Sources.File.VersionRule":"Please enter the correct version number", @@ -431,6 +433,10 @@ "meta.Synchronize.OfflineSync": "Offline Sync", "meta.Synchronize.SchedulingRules": "Scheduling Rules", "meta.Synchronize.ScheduleType": "Schedule Type", + "meta.Synchronize.ScheduleEngine": "Schedule Engine", + "meta.Synchronize.ScheduleQuartz": "Quartz", + "meta.Synchronize.ScheduleAirflow": "Airflow", + "meta.Synchronize.DolphinScheduler": "DolphinScheduler", "meta.Synchronize.Conventional": "Conventional", "meta.Synchronize.Crontab": "Crontab", "meta.Synchronize.DelayTime": "Delay Time", @@ -711,6 +717,7 @@ "pages.GroupDetail.Metric.Item": "Metric item", "pages.GroupDetail.Audit.Total": "Total", "pages.GroupDetail.Audit.DatepickerRule": "Out of selectable time range", + "pages.GroupDetail.Audit.ExportCSV": "Export CSV File", "pages.GroupDetail.Delay.QueryDate": "Query date", "pages.GroupDetail.Delay.AverageTitle": "Average transmission delay (ms)", "pages.GroupDetail.Delay.RealTimeTitle": "Transmission delay (ms)", @@ -815,6 +822,7 @@ "pages.Clusters.Name": "Cluster Name", "pages.Clusters.Tag": "Cluster Tag", "pages.Clusters.InCharges": "Owners", + "pages.Clusters.Node.Description": "Description", "pages.Clusters.Description": "Description", "pages.Clusters.TestConnection": "Test connection", "pages.Clusters.Node.Name": "Node", @@ -847,15 +855,29 @@ "pages.Clusters.Node.Password": "SSH Password", "pages.Clusters.Node.SSHPort": "SSH Port", "pages.Clusters.Node.SSHKeyHelper": "Please upload the public key to the ~/.ssh/authorized_keys file of the Agent node", + "pages.Clusters.Node.Version":"Version", "pages.Clusters.Node.Status": "Status", "pages.Clusters.Node.Status.Normal": "Normal", "pages.Clusters.Node.Status.Timeout": "Timeout", + "pages.Clusters.Node.Status.INSTALLING": "Installing", + "pages.Clusters.Node.Status.INSTALLFAILED": "Install Failed", + "pages.Clusters.Node.Status.INSTALLSUCCESS": "Install Success", "pages.Clusters.Node.LastModifier": "Last modifier", "pages.Clusters.Node.Creator": "Creator", "pages.Clusters.Node.Create": "Create", "pages.Clusters.Node.IpRule": "Please enter the IP address correctly", "pages.Clusters.Node.PortRule": "Please enter the port address correctly", "pages.Clusters.Node.ProtocolTypeRule": "Please enter the protocol type correctly", + "pages.Clusters.Node.BatchUpdate": "Batch Operation", + "pages.Clusters.Node.BatchNum": "Batch Num", + "pages.Clusters.Node.Interval": "Interval", + "pages.Clusters.Node.Minute": "Minute", + "pages.Clusters.Node.OperationStatusQuery": "Operation Status Query", + "pages.Clusters.Node.OperationType": "Operation Type", + "pages.Clusters.Node.UpgradeAgentAndInstaller": "Upgrade Agent and Installer", + "pages.Clusters.Node.UpgradeAgent": "Upgrade Agent", + "pages.Clusters.Node.UpgradeInstaller": "Upgrade Installer", + "pages.Clusters.Node.RestartAgent": "Restart Agent", "pages.Clusters.Node.Online": "Online", "pages.Clusters.Pulsar.PulsarTenant": "Default tenant", "pages.Clusters.Pulsar.TokenPlaceholder": "Required if the cluster is configured with Token", @@ -961,6 +983,7 @@ "pages.ModuleAudit.Id": "Query by id", "pages.ModuleAudit.Metric": "Query by Metric", "pages.ModuleAudit.config.Ip": "Machine ip", + "pages.ModuleAudit.config.SubValue": "Sub Value", "pages.ModuleAudit.config.BenchmarkIndicator": "Benchmark indicator", "pages.ModuleAudit.config.ComparativeIndicators": "Comparative indicator", "pages.ModuleAudit.config.InlongGroupId": "Inlong group id", diff --git a/inlong-dashboard/src/ui/pages/ClusterTags/TagDetailModal.tsx b/inlong-dashboard/src/ui/pages/ClusterTags/TagDetailModal.tsx index 743e3089a31..0375da77001 100644 --- a/inlong-dashboard/src/ui/pages/ClusterTags/TagDetailModal.tsx +++ b/inlong-dashboard/src/ui/pages/ClusterTags/TagDetailModal.tsx @@ -107,7 +107,6 @@ const TagDetailModal: React.FC = ({ id, ...modalProps }) => maxTagCount: 9, maxTagTextLength: 20, maxTagPlaceholder: omittedValues => { - console.log('omittedValues', omittedValues); return ( {i18n.t('miscellaneous.total')} diff --git a/inlong-dashboard/src/ui/pages/Clusters/AgentBatchUpdateModal.tsx b/inlong-dashboard/src/ui/pages/Clusters/AgentBatchUpdateModal.tsx new file mode 100644 index 00000000000..561e65898e3 --- /dev/null +++ b/inlong-dashboard/src/ui/pages/Clusters/AgentBatchUpdateModal.tsx @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React, { useMemo, useState } from 'react'; +import { Button, message, Modal } from 'antd'; +import { ModalProps } from 'antd/es/modal'; +import i18n from '@/i18n'; +import FormGenerator, { useForm } from '@/ui/components/FormGenerator'; +import dayjs, { Dayjs } from 'dayjs'; + +export interface Props extends ModalProps { + agentList?: []; + agentTotal?: number; + parentId?: number; + openStatusModal?: () => void; + getArgs?: (args) => void; +} + +const Comp: React.FC = ({ agentList, agentTotal, parentId, ...modalProps }) => { + const [form] = useForm(); + const content = () => [ + { + type: 'inputnumber', + label: i18n.t('pages.Clusters.Node.BatchNum'), + name: 'batchNum', + initialValue: 2, + rules: [ + { + required: true, + }, + ], + }, + { + type: 'inputnumber', + label: i18n.t('pages.Clusters.Node.Interval'), + name: 'interval', + initialValue: 5, + rules: [ + { + required: true, + }, + ], + suffix: i18n.t('pages.Clusters.Node.Minute'), + }, + { + type: 'radio', + label: i18n.t('pages.Clusters.Node.OperationType'), + name: 'operationType', + initialValue: 0, + rules: [{ required: true }], + props: { + style: { + display: 'flex', + flexDirection: 'column', + justifyContent: 'start', + alignItems: 'start', + }, + options: [ + { + label: i18n.t('pages.Clusters.Node.UpgradeAgentAndInstaller'), + value: 0, + }, + { + label: i18n.t('pages.Clusters.Node.UpgradeAgent'), + value: 1, + }, + { + label: i18n.t('pages.Clusters.Node.UpgradeInstaller'), + value: 2, + }, + { + label: i18n.t('pages.Clusters.Node.RestartAgent'), + value: 3, + }, + ], + }, + }, + { + type: 'select', + label: i18n.t('pages.Clusters.Node.Agent.Version'), + name: 'moduleIdList', + visible: values => values.operationType !== 2 && values.operationType !== 3, + props: { + options: { + requestAuto: true, + requestTrigger: ['onOpen'], + requestService: keyword => ({ + url: '/module/list', + method: 'POST', + data: { + keyword, + pageNum: 1, + pageSize: 9999, + }, + }), + requestParams: { + formatResult: result => + result?.list + ?.filter(item => item.type === 'AGENT') + .map(item => ({ + ...item, + label: `${item.name} ${item.version}`, + value: item.id, + })), + }, + }, + }, + rules: [ + { + required: true, + }, + ], + }, + { + type: 'select', + label: i18n.t('pages.Clusters.Node.AgentInstaller'), + name: 'installer', + visible: values => values.operationType === 0 || values.operationType === 2, + props: { + options: { + requestAuto: true, + requestTrigger: ['onOpen'], + requestService: keyword => ({ + url: '/module/list', + method: 'POST', + data: { + keyword, + pageNum: 1, + pageSize: 9999, + }, + }), + requestParams: { + formatResult: result => + result?.list + ?.filter(item => item.type === 'INSTALLER') + .map(item => ({ + ...item, + label: `${item.name} ${item.version}`, + value: item.id, + })), + }, + }, + }, + rules: [ + { + required: true, + }, + ], + }, + ]; + + const valuesToSubmitList = (agentList, values, submitList) => { + switch (values.operationType) { + case 0: + agentList.forEach(item => { + delete item.protocolType; + item.moduleIdList = [values.moduleIdList, values.installer]; + submitList.push({ + ...item, + moduleIdList: [values.moduleIdList, values.installer], + isInstall: true, + }); + }); + break; + case 1: + agentList.forEach(item => { + delete item.protocolType; + item.moduleIdList = [values.moduleIdList, item.moduleIdList[1]]; + submitList.push({ + ...item, + }); + delete item.isInstall; + }); + break; + + case 2: + agentList.forEach(item => { + delete item.protocolType; + item.isInstall = true; + item.moduleIdList = [item.moduleIdList[0], values.installer]; + submitList.push({ + ...item, + }); + }); + break; + + case 3: + agentList.forEach(item => { + delete item.protocolType; + submitList.push({ + ...item, + }); + delete item.isInstall; + }); + break; + } + }; + const batchUpdate = async (agentList, onOk: (e: React.MouseEvent) => void) => { + const values = await form.validateFields(); + const submitList = []; + + valuesToSubmitList(agentList, values, submitList); + const baseBatchSize = + Math.floor(submitList.length / values.batchNum) === 0 + ? 1 + : Math.floor(submitList.length / values.batchNum); + const remainder = submitList.length % values.batchNum; + const map = new Map(); + const batchNum = agentList.length < values.batchNum ? agentList.length : values.batchNum; + + for (let i = 1; i <= batchNum; i++) { + if (i === batchNum) { + map.set(i, submitList.slice((i - 1) * baseBatchSize, submitList.length)); + } else { + map.set(i, submitList.slice((i - 1) * baseBatchSize, i * baseBatchSize)); + } + } + const args = { + map: map, + interval: values.interval, + operationType: values.operationType, + ids: agentList.map(item => item.id), + submitDataList: submitList, + }; + modalProps.getArgs(args); + modalProps?.onOk(values); + modalProps?.openStatusModal(); + }; + + return ( + modalProps.onCancel(e)}> + {i18n.t('basic.Cancel')} + , + , + ]} + > + + + ); +}; + +export default Comp; diff --git a/inlong-dashboard/src/ui/pages/Clusters/HeartBeatModal.tsx b/inlong-dashboard/src/ui/pages/Clusters/HeartBeatModal.tsx index 50427f51829..01100f2864b 100644 --- a/inlong-dashboard/src/ui/pages/Clusters/HeartBeatModal.tsx +++ b/inlong-dashboard/src/ui/pages/Clusters/HeartBeatModal.tsx @@ -17,36 +17,41 @@ * under the License. */ -import React, { useMemo } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { Modal } from 'antd'; import { ModalProps } from 'antd/es/modal'; import { useRequest, useUpdateEffect } from '@/ui/hooks'; import i18n from '@/i18n'; import HighTable from '@/ui/components/HighTable'; import { timestampFormat } from '@/core/utils'; +import { defaultSize } from '@/configs/pagination'; export interface Props extends ModalProps { type?: string; ip?: string; } -const Comp: React.FC = ({ type, ip, ...modalProps }) => { +const Comp: React.FC = ({ ...modalProps }) => { + const [options, setOptions] = useState({ + inlongGroupId: '', + inlongStreamId: '', + pageNum: 1, + pageSize: defaultSize, + }); + const { data: heartList, run: getHeartList } = useRequest( { url: '/heartbeat/component/list', method: 'POST', data: { - component: type, - inlongGroupId: '', - inlongStreamId: '', - instance: ip, + ...options, + component: 'AGENT', + instance: modalProps.ip, }, }, { - manual: true, - onSuccess: data => { - console.log(data); - }, + refreshDeps: [options], + onSuccess: data => {}, }, ); @@ -81,11 +86,20 @@ const Comp: React.FC = ({ type, ip, ...modalProps }) => { ]; }, []); const pagination = { - pageSize: 5, - current: 1, - total: heartList?.list?.length, + pageSize: +options.pageSize, + current: +options.pageNum, + total: heartList?.total, }; - useUpdateEffect(() => { + + const onChange = ({ current: pageNum, pageSize }) => { + setOptions(prev => ({ + ...prev, + pageNum, + pageSize, + })); + }; + + useEffect(() => { if (modalProps.open) { getHeartList(); } @@ -102,8 +116,9 @@ const Comp: React.FC = ({ type, ip, ...modalProps }) => { table={{ columns: columns, rowKey: 'id', - dataSource: heartList?.list, + dataSource: heartList?.list || [], pagination, + onChange, }} /> diff --git a/inlong-dashboard/src/ui/pages/Clusters/NodeEditModal.tsx b/inlong-dashboard/src/ui/pages/Clusters/NodeEditModal.tsx index 80b7355120d..0496e820cfe 100644 --- a/inlong-dashboard/src/ui/pages/Clusters/NodeEditModal.tsx +++ b/inlong-dashboard/src/ui/pages/Clusters/NodeEditModal.tsx @@ -47,6 +47,10 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m // Only keep the first element and give the rest to the 'installer' result.installer = result?.moduleIdList.slice(1); result.moduleIdList = result?.moduleIdList.slice(0, 1); + if (result.username) { + setInstallType(true); + result.isInstall = true; + } } form.setFieldsValue(result); }, @@ -64,6 +68,14 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m if (isUpdate) { submitData.id = id; submitData.version = savedData?.version; + if (type === 'AGENT') { + if (!submitData.isInstall) { + submitData.username = ''; + submitData.password = ''; + submitData.sshPort = ''; + submitData.sshKey = ''; + } + } } if (type === 'AGENT') { if (submitData.installer !== undefined) { @@ -83,29 +95,19 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m message.success(i18n.t('basic.OperatingSuccess')); }; - const { data: agentInstaller, run: getAgentInstall } = useRequest( - () => ({ + const [agentInstaller, setAgentInstaller] = useState([]); + const getAgentInstaller = async () => { + const result = await request({ url: '/module/list', method: 'POST', data: { pageNum: 1, pageSize: 9999, }, - }), - { - manual: true, - onSuccess: result => { - const temp = result?.list - ?.filter(item => item.type === 'INSTALLER') - .map(item => ({ - ...item, - label: `${item.name} ${item.version}`, - value: item.id, - })); - form.setFieldValue('installer', temp[0].id); - }, - }, - ); + }); + setAgentInstaller(result.list?.sort((a, b) => b.modifyTime - a.modifyTime)); + return result; + }; const { data: sshKeys, run: getSSHKeys } = useRequest( () => ({ @@ -123,9 +125,10 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m const testSSHConnection = async () => { const values = await form.validateFields(); const submitData = { - ...values, - type, - parentId: savedData?.parentId || clusterId, + ip: values.ip, + sshPort: values.sshPort, + username: values.username, + password: values.password, }; await request({ url: '/cluster/node/testSSHConnection', @@ -140,18 +143,25 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m // open setInstallType(false); form.resetFields(); + getAgentInstaller(); if (id) { getData(id); - } else { - if (type === 'AGENT') { - getAgentInstall(); - } } } }, [modalProps.open]); + useEffect(() => { + form.setFieldValue('identifyType', 'password'); + if (modalProps.open && !id) { + form.setFieldValue( + 'installer', + agentInstaller?.filter(item => item.type === 'INSTALLER')?.[0]?.id, + ); + } + }, [agentInstaller]); + const content = useMemo(() => { - return [ + return Id => [ { type: 'input', label: 'IP', @@ -159,9 +169,13 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m rules: [ { pattern: rulesPattern.ip, + required: type === 'AGENT', message: i18n.t('pages.Clusters.Node.IpRule'), }, ], + props: { + disabled: !!Id, + }, }, { type: 'inputnumber', @@ -205,34 +219,20 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m name: 'moduleIdList', hidden: type !== 'AGENT', props: { - options: { - requestAuto: true, - requestTrigger: ['onOpen'], - requestService: keyword => ({ - url: '/module/list', - method: 'POST', - data: { - keyword, - pageNum: 1, - pageSize: 9999, - }, - }), - requestParams: { - formatResult: result => - result?.list - ?.filter(item => item.type === 'AGENT') - .map(item => ({ - ...item, - label: `${item.name} ${item.version}`, - value: item.id, - })), - }, - }, + filterOption: (input, option) => + (option?.label ?? '').toLowerCase().includes(input.toLowerCase()), + options: agentInstaller + ?.filter(item => item.type === 'AGENT') + .map(item => ({ + ...item, + label: `${item.name} ${item.version}`, + value: item.id, + })), }, }, { type: 'textarea', - label: i18n.t('pages.Clusters.Description'), + label: i18n.t('pages.Clusters.Node.Description'), name: 'description', props: { maxLength: 256, @@ -267,7 +267,7 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m name: 'identifyType', initialValue: 'password', hidden: type !== 'AGENT', - visible: values => values?.isInstall && form.getFieldValue('isInstall'), + visible: values => isInstall, rules: [{ required: true }], props: { onChange: ({ target: { value } }) => { @@ -293,7 +293,7 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m name: 'username', rules: [{ required: true }], hidden: type !== 'AGENT', - visible: values => values?.isInstall && form.getFieldValue('isInstall'), + visible: isInstall, }, { type: 'input', @@ -301,12 +301,10 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m name: 'password', rules: [{ required: true }], hidden: type !== 'AGENT', - visible: values => { - return ( - (values?.isInstall && values?.identifyType === 'password') || - (form.getFieldValue('isInstall') && form.getFieldValue('identifyType') === 'password') - ); - }, + visible: values => + isInstall && + (values?.identifyType === 'password' || + form.getFieldValue('identifyType') === 'password'), }, { type: 'textarea', @@ -315,7 +313,9 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m name: 'sshKey', rules: [{ required: true }], hidden: type !== 'AGENT', - visible: values => values?.isInstall && values?.identifyType === 'sshKey', + visible: values => + isInstall && + (values?.identifyType === 'sshKey' || form.getFieldValue('identifyType') === 'sshKey'), props: { readOnly: true, autoSize: true, @@ -327,7 +327,7 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m name: 'sshPort', rules: [{ required: true }], hidden: type !== 'AGENT', - visible: values => values?.isInstall && form.getFieldValue('isInstall'), + visible: values => isInstall, }, { type: 'select', @@ -336,33 +336,17 @@ const NodeEditModal: React.FC = ({ id, type, clusterId, ...m isPro: type === 'AGENT', hidden: type !== 'AGENT', props: { - options: { - requestAuto: true, - requestTrigger: ['onOpen'], - requestService: keyword => ({ - url: '/module/list', - method: 'POST', - data: { - keyword, - pageNum: 1, - pageSize: 9999, - }, - }), - requestParams: { - formatResult: result => - result?.list - ?.filter(item => item.type === 'INSTALLER') - .map(item => ({ - ...item, - label: `${item.name} ${item.version}`, - value: item.id, - })), - }, - }, + options: agentInstaller + ?.filter(item => item.type === 'INSTALLER') + .map(item => ({ + ...item, + label: `${item.name} ${item.version}`, + value: item.id, + })), }, }, ]; - }, []); + }, [isInstall, agentInstaller, modalProps.open]); return ( = ({ id, type, clusterId, ...m ), ]} > - + ); }; diff --git a/inlong-dashboard/src/ui/pages/Clusters/NodeManage.tsx b/inlong-dashboard/src/ui/pages/Clusters/NodeManage.tsx index cb85043cc56..74ab2112e53 100644 --- a/inlong-dashboard/src/ui/pages/Clusters/NodeManage.tsx +++ b/inlong-dashboard/src/ui/pages/Clusters/NodeManage.tsx @@ -17,7 +17,7 @@ * under the License. */ -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { Button, Modal, message, Dropdown, Space } from 'antd'; import i18n from '@/i18n'; import { parse } from 'qs'; @@ -28,17 +28,28 @@ import { useRequest, useLocation } from '@/ui/hooks'; import NodeEditModal from './NodeEditModal'; import request from '@/core/utils/request'; import { timestampFormat } from '@/core/utils'; -import { genStatusTag } from './status'; +import { genStatusTag, statusList } from './status'; import HeartBeatModal from '@/ui/pages/Clusters/HeartBeatModal'; import LogModal from '@/ui/pages/Clusters/LogModal'; import { DownOutlined } from '@ant-design/icons'; import { MenuProps } from 'antd/es/menu'; +import { useForm } from 'antd/es/form/Form'; +import { getModuleList, versionMap } from '@/ui/pages/Clusters/config'; +import AgentBatchUpdateModal from '@/ui/pages/Clusters/AgentBatchUpdateModal'; +import OperationLogModal from '@/ui/pages/Clusters/OperationLogModal'; const getFilterFormContent = defaultValues => [ { type: 'inputsearch', name: 'keyword', }, + { + type: 'select', + name: 'status', + props: { + options: statusList, + }, + }, ]; const Comp: React.FC = () => { @@ -47,6 +58,7 @@ const Comp: React.FC = () => { () => (parse(location.search.slice(1)) as Record) || {}, [location.search], ); + const [form] = useForm(); const [options, setOptions] = useState({ keyword: '', @@ -54,10 +66,12 @@ const Comp: React.FC = () => { pageNum: 1, type, parentId: +clusterId, + status: '', }); const [nodeEditModal, setNodeEditModal] = useState>({ open: false, + agentInstallerList: [], }); const [logModal, setLogModal] = useState>({ open: false, @@ -65,6 +79,12 @@ const Comp: React.FC = () => { const [heartModal, setHeartModal] = useState>({ open: false, }); + const [operationLogModal, setOperationLogModal] = useState>({ + open: false, + }); + const [agentBatchUpdateModal, setAgentBatchUpdateModal] = useState>({ + open: false, + }); const { data, @@ -109,13 +129,21 @@ const Comp: React.FC = () => { onOk: async () => { record.agentRestartTime = record?.agentRestartTime + 1; delete record.isInstall; - await request({ - url: `/cluster/node/update`, - method: 'POST', - data: record, - }); + try { + const response = await request({ + url: `/cluster/node/update`, + method: 'POST', + data: record, + }); + if (response.success) { + message.success(i18n.t('basic.OperatingSuccess')); + } else { + Modal.destroyAll(); + } + } catch (e) { + Modal.destroyAll(); + } await getList(); - message.success(i18n.t('basic.OperatingSuccess')); }, }); }, @@ -126,28 +154,39 @@ const Comp: React.FC = () => { Modal.confirm({ title: i18n.t('pages.Cluster.Node.InstallTitle'), onOk: async () => { - await request({ - url: `/cluster/node/update`, - method: 'POST', - data: { - ...record, - isInstall: true, - }, - }); + try { + const response = await request({ + url: `/cluster/node/update`, + method: 'POST', + data: { + ...record, + isInstall: true, + }, + }); + if (response.success) { + message.success(i18n.t('basic.OperatingSuccess')); + } else { + Modal.destroyAll(); + } + } catch (e) { + Modal.destroyAll(); + } await getList(); - message.success(i18n.t('basic.OperatingSuccess')); }, }); }, [getList], ); - const onLog = ({ id }) => { setLogModal({ open: true, id }); }; const openHeartModal = ({ type, ip }) => { setHeartModal({ open: true, type: type, ip: ip }); }; + const openOperationLogModal = ({ ip }) => { + setOperationLogModal({ open: true, ip: ip }); + }; + const onDelete = useCallback( ({ id }) => { Modal.confirm({ @@ -208,6 +247,7 @@ const Comp: React.FC = () => { default: break; } + return result; }, }, ); @@ -232,6 +272,10 @@ const Comp: React.FC = () => { label: , key: '4', }, + { + label: , + key: '5', + }, ]; const handleMenuClick = (key, record) => { switch (key) { @@ -250,10 +294,120 @@ const Comp: React.FC = () => { case '4': openHeartModal(record); break; + case '5': + openOperationLogModal(record); + break; default: break; } }; + const agentInstallerList = useRef([]); + const [agentVersionObj, setAgentVersionObj] = useState({}); + useEffect(() => { + (() => { + getModuleList().then(res => { + agentInstallerList.current = res?.list; + setAgentVersionObj(versionMap(res?.list)); + }); + })(); + }, [type]); + const onOpenAgentModal = () => { + setAgentStatusModal({ open: true }); + }; + const [nodeList, setNodeList] = useState([]); + const statusPagination = { + total: nodeList?.length, + }; + + const [isSmall, setIsSmall] = useState(window.innerWidth < 1600); + + useEffect(() => { + const handleResize = () => { + setIsSmall(window.innerWidth < 1600); + }; + + window.addEventListener('resize', handleResize); + return () => { + window.removeEventListener('resize', handleResize); + }; + }, []); + + const getOperationMenu = useMemo(() => { + return isSmall + ? [ + { + title: i18n.t('basic.Operating'), + dataIndex: 'action', + key: 'operation', + width: isSmall ? 200 : 400, + render: (text, record) => ( + <> + + + {type === 'AGENT' && ( + handleMenuClick(key, record) }}> + e.preventDefault()}> + + {i18n.t('pages.Cluster.Node.More')} + + + + + )} + + ), + }, + ] + : [ + { + title: i18n.t('basic.Operating'), + dataIndex: 'action', + key: 'operation', + width: isSmall ? 200 : 400, + render: (text, record) => ( + <> + + + + + + + + + + ), + }, + ]; + }, [isSmall]); const columns = useMemo(() => { return [ { @@ -278,6 +432,14 @@ const Comp: React.FC = () => { dataIndex: 'status', render: text => genStatusTag(text), }, + { + title: i18n.t('pages.Clusters.Node.Agent.Version'), + dataIndex: 'moduleIdList', + render: (text, record) => { + const index = text.slice(0, 1)[0]; + return agentVersionObj[index]; + }, + }, { title: i18n.t('pages.Clusters.Node.Creator'), dataIndex: 'creator', @@ -298,35 +460,138 @@ const Comp: React.FC = () => { ), }, - { - title: i18n.t('basic.Operating'), - dataIndex: 'action', - key: 'operation', - width: 200, - render: (text, record) => ( - <> - - - {type === 'AGENT' && ( - handleMenuClick(key, record) }}> - e.preventDefault()}> - - {i18n.t('pages.Cluster.Node.More')} - - - - - )} - - ), - }, - ]; - }, [onDelete]); + ].concat(getOperationMenu); + }, [onDelete, onInstall, onUnload, type, agentVersionObj, isSmall]); + + const [disabled, setDisabled] = useState(true); + const finalStatus = useRef(false); + const [selectedRowKeys, setSelectedRowKeys] = useState([]); + const [statusAgentList, setStatusAgentList] = useState([]); + const [batchUpdateArgs, setBatchUpdateArgs] = useState({ + map: [], + interval: 0, + ids: [], + operationType: 0, + submitDataList: [], + }); + const getBatchUpdateArgs = args => { + setBatchUpdateArgs(args); + }; + const rowSelection = { + selectedRowKeys, + onChange: (selectedRowKeys, selectedRows) => { + setSelectedRowKeys(selectedRowKeys); + setDisabled(selectedRowKeys.length <= 0); + setNodeList(selectedRows); + }, + }; + const onUpdate = async submitData => + await request({ + url: '/cluster/node/update', + method: 'POST', + data: submitData, + }); + const getAgentNode = async id => + await request({ + url: `/cluster/node/get/${id}`, + method: 'GET', + }); + const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); + const batchUpdate = async () => { + const { map, interval, submitDataList } = batchUpdateArgs; + setStatusAgentList(submitDataList); + const entries = Array.from(map.entries()); + for (let i = 0; i < entries.length; i++) { + const [key, value] = entries[i]; + for (let j = 0; j < value.length; j++) { + const agentNode = await getAgentNode(value[j]?.id); + if (i === entries.length - 1 && j === value.length - 1) { + onUpdate({ + ...value[j], + version: agentNode.version, + agentRestartTime: + batchUpdateArgs.operationType === 3 + ? value[j].agentRestartTime + 1 + : value[j].agentRestartTime, + }); + await delay(1000); + await closeStatusModal(); + setStatusAgentList([]); + } else { + onUpdate({ + ...value[j], + version: agentNode.version, + agentRestartTime: + batchUpdateArgs.operationType === 3 + ? value[j].agentRestartTime + 1 + : value[j].agentRestartTime, + }); + } + } + if (i < entries.length - 1) { + finalStatus.current = false; + await delay(interval * 60 * 1000); + } + } + new Promise(resolve => { + finalStatus.current = true; + }); + }; + const getStatusAgentList = option => { + const { ids } = batchUpdateArgs; + request({ + url: '/cluster/node/list', + method: 'POST', + data: { ...option }, + }).then(response => { + const list = response?.list.filter(item => ids.includes(item.id)); + setStatusAgentList(list); + }); + }; + + useEffect(() => { + batchUpdate(); + }, [batchUpdateArgs]); + const [agentStatusModal, setAgentStatusModal] = useState({ + open: false, + }); + + useEffect(() => { + const internalId = setInterval(() => { + if (agentStatusModal.open) { + getStatusAgentList({ + pageSize: 99999, + pageNum: 1, + type: 'AGENT', + parentId: +clusterId, + }); + } + }, 10000); + return () => { + clearInterval(internalId); + }; + }, [agentStatusModal.open]); + const closeStatusModal = async () => { + const completeList = statusAgentList.filter(item => item.status !== 2); + if (completeList.length === statusAgentList.length && finalStatus.current) { + setStatusAgentList([]); + } + setNodeList([]); + setSelectedRowKeys([]); + setDisabled(true); + await getList(); + setAgentStatusModal({ open: false }); + }; + const tableProps = { rowSelection: {} }; + if (type === 'AGENT') { + tableProps.rowSelection = { + type: 'checkbox', + ...rowSelection, + }; + } else { + delete tableProps.rowSelection; + } return ( { setNodeEditModal({ open: true })}> - {i18n.t('pages.Clusters.Node.Create')} - + <> + {type === 'AGENT' && ( + + )} + {statusAgentList.length > 0 && ( + + )} + + } table={{ + ...tableProps, columns: type === 'AGENT' ? columns.filter( item => item.dataIndex !== 'enabledOnline' && item.dataIndex !== 'port' && - item.dataIndex !== 'protocolType', + item.dataIndex !== 'protocolType' && + item.dataIndex !== 'modifier', ) - : columns, + : columns.filter( + item => item.dataIndex !== 'moduleIdList' && item.dataIndex !== 'installer', + ), rowKey: 'id', dataSource: data?.list, pagination, @@ -388,12 +693,54 @@ const Comp: React.FC = () => { { - await getList(); - setHeartModal({ open: false }); - }} onCancel={() => setHeartModal({ open: false })} /> + { + setAgentBatchUpdateModal({ open: false }); + }} + onCancel={() => setAgentBatchUpdateModal({ open: false })} + /> + + { + setOperationLogModal({ open: false }); + }} + onCancel={() => setOperationLogModal({ open: false })} + > + { + await closeStatusModal(); + }} + > + {i18n.t('pages.GroupDetail.Stream.Closed')} + , + ]} + > + + item.dataIndex !== 'action' && + item.dataIndex !== 'enabledOnline' && + item.dataIndex !== 'port' && + item.dataIndex !== 'protocolType', + ), + rowKey: 'id', + dataSource: statusAgentList, + pagination: statusPagination, + }} + /> + ); }; diff --git a/inlong-dashboard/src/ui/pages/Clusters/OperationLogModal.tsx b/inlong-dashboard/src/ui/pages/Clusters/OperationLogModal.tsx new file mode 100644 index 00000000000..d829dc557ad --- /dev/null +++ b/inlong-dashboard/src/ui/pages/Clusters/OperationLogModal.tsx @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React, { useEffect, useMemo, useState } from 'react'; +import { Button, message, Modal } from 'antd'; +import i18n from '@/i18n'; +import { ModalProps } from 'antd/es/modal'; +import { defaultSize } from '@/configs/pagination'; +import { useRequest } from '@/ui/hooks'; +import HighTable from '@/ui/components/HighTable'; +import { getFormContent, getTableColumns } from '@/ui/pages/Clusters/config'; +export interface Props extends ModalProps { + ip?: string; + operationType?: string; +} + +const Comp: React.FC = ({ ...modalProps }) => { + const [options, setOptions] = useState({ + pageSize: defaultSize, + pageNum: 1, + }); + + const { data: sourceData, run } = useRequest( + { + url: '/operationLog/list', + method: 'POST', + data: { + ...options, + ip: modalProps.ip, + operationTarget: 'CLUSTER_NODE', + }, + }, + { + manual: true, + }, + ); + + const pagination = { + pageSize: options.pageSize, + current: options.pageNum, + total: sourceData?.total, + }; + const onChange = ({ current: pageNum, pageSize }) => { + setOptions(prev => ({ + ...prev, + pageNum, + pageSize, + })); + }; + + const onFilter = allValues => { + setOptions(prev => ({ + ...prev, + ...allValues, + pageNum: 1, + })); + }; + + useEffect(() => { + if (modalProps.open) { + run(); + } + }, [modalProps.open, options]); + + return ( + + + + ); +}; + +export default Comp; diff --git a/inlong-dashboard/src/ui/pages/Clusters/config.tsx b/inlong-dashboard/src/ui/pages/Clusters/config.tsx new file mode 100644 index 00000000000..82c1689cdcb --- /dev/null +++ b/inlong-dashboard/src/ui/pages/Clusters/config.tsx @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import request from '@/core/utils/request'; +import i18n from '@/i18n'; +import { Tooltip } from 'antd'; +import { dateFormat } from '@/core/utils'; +import React from 'react'; + +export const getModuleList = async () => { + return await request({ + url: '/module/list', + method: 'POST', + data: { + pageNum: 1, + pageSize: 9999, + }, + }); +}; + +export const versionMap = data => { + return data + ?.filter(item => item.type === 'AGENT') + .map(item => ({ + ...item, + label: `${item.name} ${item.version}`, + value: item.id, + })) + .reduce( + (acc, cur) => ({ + ...acc, + [cur.value]: cur.label, + }), + {}, + ); +}; + +export const installerMap = () => { + return getModuleList().then(res => + res?.list + ?.filter(item => item.type === 'INSTALLER') + .map(item => ({ + ...item, + label: `${item.name} ${item.version}`, + value: item.id, + })) + .reduce( + (acc, cur) => ({ + ...acc, + [cur.value]: cur.label, + }), + {}, + ), + ); +}; + +const typeList = [ + { + label: 'Create', + value: 'CREATE', + }, + { + label: 'Update', + value: 'UPDATE', + }, + { + label: 'Delete', + value: 'DELETE', + }, + { + label: 'Get', + value: 'GET', + }, +]; + +export const getFormContent = () => [ + { + type: 'select', + label: i18n.t('pages.GroupDetail.OperationLog.OperationType'), + name: 'operationType', + props: { + allowClear: true, + dropdownMatchSelectWidth: false, + options: typeList, + }, + }, +]; + +export const getTableColumns = [ + { + title: i18n.t('pages.GroupDetail.OperationLog.Table.Operator'), + dataIndex: 'operator', + }, + { + title: i18n.t('pages.GroupDetail.OperationLog.Table.OperationType'), + dataIndex: 'operationType', + render: text => typeList.find(c => c.value === text)?.label || text, + }, + { + title: i18n.t('pages.GroupDetail.OperationLog.Table.Log'), + dataIndex: 'body', + ellipsis: true, + render: body => ( + + {body} + + ), + }, + { + title: i18n.t('pages.GroupDetail.OperationLog.Table.OperationTime'), + dataIndex: 'requestTime', + render: text => dateFormat(new Date(text)), + }, +]; diff --git a/inlong-dashboard/src/ui/pages/Clusters/index.tsx b/inlong-dashboard/src/ui/pages/Clusters/index.tsx index d5ffea25ec3..a42f5052db6 100644 --- a/inlong-dashboard/src/ui/pages/Clusters/index.tsx +++ b/inlong-dashboard/src/ui/pages/Clusters/index.tsx @@ -169,7 +169,7 @@ const Comp: React.FC = () => { {i18n.t('pages.Clusters.Node.Name')} )} - {record.type !== 'DATAPROXY' && record.type !== 'AGENT' && ( + {record.type !== 'DATAPROXY' && ( diff --git a/inlong-dashboard/src/ui/pages/Clusters/status.tsx b/inlong-dashboard/src/ui/pages/Clusters/status.tsx index afeaf4e2bbf..434916a2717 100644 --- a/inlong-dashboard/src/ui/pages/Clusters/status.tsx +++ b/inlong-dashboard/src/ui/pages/Clusters/status.tsx @@ -39,6 +39,21 @@ export const statusList: StatusProp[] = [ value: 2, type: 'error', }, + { + label: i18n.t('pages.Clusters.Node.Status.INSTALLING'), + value: 3, + type: 'primary', + }, + { + label: i18n.t('pages.Clusters.Node.Status.INSTALLFAILED'), + value: 4, + type: 'error', + }, + { + label: i18n.t('pages.Clusters.Node.Status.INSTALLSUCCESS'), + value: 5, + type: 'success', + }, ]; export const statusMap = statusList.reduce( diff --git a/inlong-dashboard/src/ui/pages/ConsumeDashboard/index.tsx b/inlong-dashboard/src/ui/pages/ConsumeDashboard/index.tsx index b03d9d46870..664e0c3f764 100644 --- a/inlong-dashboard/src/ui/pages/ConsumeDashboard/index.tsx +++ b/inlong-dashboard/src/ui/pages/ConsumeDashboard/index.tsx @@ -135,6 +135,10 @@ const Comp: React.FC = () => { allowClear: true, options: lastConsumerStatusList, dropdownMatchSelectWidth: false, + showSearch: true, + filterOption: (keyword: string, option: { label: any }) => { + return (option?.label ?? '').toLowerCase().includes(keyword.toLowerCase()); + }, }, }, { @@ -146,6 +150,10 @@ const Comp: React.FC = () => { allowClear: true, options: consumes.filter(x => x.value), dropdownMatchSelectWidth: false, + showSearch: true, + filterOption: (keyword: string, option: { label: any }) => { + return (option?.label ?? '').toLowerCase().includes(keyword.toLowerCase()); + }, }, }, ], diff --git a/inlong-dashboard/src/ui/pages/GroupDashboard/index.tsx b/inlong-dashboard/src/ui/pages/GroupDashboard/index.tsx index 4e602b44ba4..10913a199de 100644 --- a/inlong-dashboard/src/ui/pages/GroupDashboard/index.tsx +++ b/inlong-dashboard/src/ui/pages/GroupDashboard/index.tsx @@ -172,6 +172,10 @@ const Comp: React.FC = () => { allowClear: true, options: statusList, dropdownMatchSelectWidth: false, + showSearch: true, + filterOption: (keyword: string, option: { label: any }) => { + return (option?.label ?? '').toLowerCase().includes(keyword.toLowerCase()); + }, }, }, { diff --git a/inlong-dashboard/src/ui/pages/GroupDataTemplate/index.tsx b/inlong-dashboard/src/ui/pages/GroupDataTemplate/index.tsx index 1a3326b179d..52851f3b637 100644 --- a/inlong-dashboard/src/ui/pages/GroupDataTemplate/index.tsx +++ b/inlong-dashboard/src/ui/pages/GroupDataTemplate/index.tsx @@ -104,7 +104,6 @@ const Comp: React.FC = () => { ); const onDelete = useCallback( record => { - console.log(record); Modal.confirm({ title: i18n.t('basic.DeleteConfirm'), onOk: async () => { diff --git a/inlong-dashboard/src/ui/pages/GroupDetail/Audit/config.tsx b/inlong-dashboard/src/ui/pages/GroupDetail/Audit/config.tsx index 10cc589d505..c9bd6f68520 100644 --- a/inlong-dashboard/src/ui/pages/GroupDetail/Audit/config.tsx +++ b/inlong-dashboard/src/ui/pages/GroupDetail/Audit/config.tsx @@ -23,6 +23,8 @@ import dayjs from 'dayjs'; import i18n from '@/i18n'; import { sinks } from '@/plugins/sinks'; import request from '@/core/utils/request'; +import { CSVLink } from 'react-csv'; +import audit from '@/ui/pages/GroupDetail/Audit/index'; export const timeStaticsDimList = [ { @@ -131,11 +133,11 @@ export const getSourceDataWithPercent = (sourceKeys, sourceMap) => { export const getDiff = (first, current) => { if (first === 0) { - return '0%'; + return first.toFixed(4) + '%'; } let result; - const diff = Math.ceil((current / first - 1) * 100); - result = diff > 0 ? '+' + diff + '%' : diff + '%'; + const diff = (current / first - 1) * 100; + result = diff > 0 ? '+' + diff.toFixed(4) + '%' : diff.toFixed(4) + '%'; return result; }; @@ -154,10 +156,18 @@ export const getSourceDataWithCommas = sourceData => { }); return sourceData; }; - let endTimeVisible = true; - -export const getFormContent = (inlongGroupId, initialValues, onSearch, onDataStreamSuccess) => [ +export const getFormContent = ( + inlongGroupId, + initialValues, + onSearch, + onDataStreamSuccess, + sourceData, + csvData, + fileName, + setInlongStreamID, + inlongStreamId, +) => [ { type: 'select', label: i18n.t('pages.ModuleAudit.config.InlongStreamId'), @@ -165,6 +175,9 @@ export const getFormContent = (inlongGroupId, initialValues, onSearch, onDataStr props: { dropdownMatchSelectWidth: false, showSearch: true, + onChange: (value, option) => { + setInlongStreamID(value); + }, options: { requestAuto: true, requestTrigger: ['onOpen', 'onSearch'], @@ -207,7 +220,7 @@ export const getFormContent = (inlongGroupId, initialValues, onSearch, onDataStr pageNum: 1, pageSize: 100, inlongGroupId, - inlongStreamId: values.inlongStreamId, + inlongStreamId: inlongStreamId, }, }), requestParams: { @@ -218,6 +231,9 @@ export const getFormContent = (inlongGroupId, initialValues, onSearch, onDataStr })) || [], }, }, + filterOption: (keyword: string, option: { label: any }) => { + return (option?.label ?? '').toLowerCase().includes(keyword.toLowerCase()); + }, }), }, { @@ -244,7 +260,6 @@ export const getFormContent = (inlongGroupId, initialValues, onSearch, onDataStr return Promise.resolve(); } const timeDiff = value - getFieldValue('startDate'); - console.log('timeDiff', value, getFieldValue('startDate'), timeDiff); if (timeDiff >= 0) { const isHourDiff = dim === 'HOUR' && timeDiff < 1000 * 60 * 60 * 24 * 3; const isDayDiff = dim === 'DAY' && timeDiff < 1000 * 60 * 60 * 24 * 7; @@ -332,6 +347,15 @@ export const getFormContent = (inlongGroupId, initialValues, onSearch, onDataStr ), }, + { + type: ( + + ), + }, ]; export const getTableColumns = (source, dim) => { diff --git a/inlong-dashboard/src/ui/pages/GroupDetail/Audit/index.tsx b/inlong-dashboard/src/ui/pages/GroupDetail/Audit/index.tsx index f4ba9fbbc43..913d583eb42 100644 --- a/inlong-dashboard/src/ui/pages/GroupDetail/Audit/index.tsx +++ b/inlong-dashboard/src/ui/pages/GroupDetail/Audit/index.tsx @@ -17,7 +17,7 @@ * under the License. */ -import React, { useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import FormGenerator, { useForm } from '@/ui/components/FormGenerator'; import HighTable from '@/ui/components/HighTable'; import { useRequest } from '@/ui/hooks'; @@ -38,14 +38,13 @@ type Props = CommonInterface; const Comp: React.FC = ({ inlongGroupId }) => { const [form] = useForm(); - const [query, setQuery] = useState({ inlongStreamId: '', startDate: +new Date(), endDate: +new Date(), timeStaticsDim: timeStaticsDimList[0].value, }); - + const [inlongStreamID, setInlongStreamID] = useState(''); const { data: sourceData = [], run } = useRequest( { url: '/audit/list', @@ -104,20 +103,68 @@ const Comp: React.FC = ({ inlongGroupId }) => { const onDataStreamSuccess = data => { const defaultDataStream = data[0]?.value; if (defaultDataStream) { + setInlongStreamID(defaultDataStream); form.setFieldsValue({ inlongStreamId: defaultDataStream }); setQuery(prev => ({ ...prev, inlongStreamId: defaultDataStream })); run(); } }; + const numToName = useCallback( + num => { + let obj = {}; + sourceData.forEach(item => { + obj = { ...obj, [item.auditId]: item.auditName }; + }); + obj = { ...obj, logTs: i18n.t('pages.GroupDetail.Audit.Time') }; + return obj[num]; + }, + [sourceData], + ); + const metricSum = useMemo(() => { + let obj = { logTs: i18n.t('pages.GroupDetail.Audit.Total') }; + sourceData.map(item => { + const sum = item.auditSet?.reduce((total, cur) => { + return total + cur.count; + }, 0); + obj = { ...obj, [item.auditId]: sum }; + }); + return obj; + }, [sourceData]); + const csvData = useMemo(() => { + const result = [...toTableData(sourceData, sourceDataMap), metricSum].map(item => { + let obj = {}; + Object.keys(item) + .reverse() + .forEach(key => { + obj = { ...obj, [numToName(key)]: item[key] }; + }); + return obj; + }); + return result; + }, [sourceData, sourceDataMap, metricSum]); + const [fileName, setFileName] = useState('audit.csv'); + useEffect(() => { + setFileName(`audit_${inlongGroupId}_${inlongStreamID}.csv`); + }, [inlongGroupId, inlongStreamID]); return ( <>
setQuery({ ...allValues, diff --git a/inlong-dashboard/src/ui/pages/GroupDetail/DataSources/DetailModal.tsx b/inlong-dashboard/src/ui/pages/GroupDetail/DataSources/DetailModal.tsx index 4f1675a3d2a..4267fc5b83b 100644 --- a/inlong-dashboard/src/ui/pages/GroupDetail/DataSources/DetailModal.tsx +++ b/inlong-dashboard/src/ui/pages/GroupDetail/DataSources/DetailModal.tsx @@ -60,6 +60,7 @@ const Comp: React.FC = ({ formatResult: result => new Entity()?.parse(result) || result, onSuccess: result => { form.setFieldsValue(result); + form.setFieldsValue({ clusterTag: result?.inlongClusterTag }); setType(result.sourceType); }, }, @@ -89,6 +90,11 @@ const Comp: React.FC = ({ message.success(t('pages.GroupDetail.Sources.SaveSuccessfully')); }; + const { run: getGroup } = useRequest(`/group/get/${inlongGroupId}`, { + onSuccess: result => { + form.setFieldsValue({ clusterTag: result?.inlongClusterTag }); + }, + }); useUpdateEffect(() => { if (modalProps.open) { // open @@ -118,7 +124,10 @@ const Comp: React.FC = ({ setType(values.sourceType)} + onValuesChange={(c, values) => { + setType(values.sourceType); + if (Object.keys(c)[0] === 'sourceType') getGroup(); + }} initialValues={id ? data : { inlongGroupId }} form={form} useMaxWidth diff --git a/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/config.tsx b/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/config.tsx index b906633e43e..05499d5beec 100644 --- a/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/config.tsx +++ b/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/config.tsx @@ -35,6 +35,10 @@ export const getFilterFormContent = (defaultValues = {} as any) => [ allowClear: true, options: statusList, dropdownMatchSelectWidth: false, + showSearch: true, + filterOption: (keyword: string, option: { label: any }) => { + return (option?.label ?? '').toLowerCase().includes(keyword.toLowerCase()); + }, }, }, ]; diff --git a/inlong-dashboard/src/ui/pages/GroupDetail/OperationLog/config.tsx b/inlong-dashboard/src/ui/pages/GroupDetail/OperationLog/config.tsx index 50d558c03dd..992f242a95b 100644 --- a/inlong-dashboard/src/ui/pages/GroupDetail/OperationLog/config.tsx +++ b/inlong-dashboard/src/ui/pages/GroupDetail/OperationLog/config.tsx @@ -87,6 +87,11 @@ export const getFormContent = inlongGroupId => [ })), }, }, + allowClear: true, + showSearch: true, + filterOption: (keyword: string, option: { label: any }) => { + return (option?.label ?? '').toLowerCase().includes(keyword.toLowerCase()); + }, }, }, { @@ -96,6 +101,11 @@ export const getFormContent = inlongGroupId => [ props: { dropdownMatchSelectWidth: false, options: targetList, + allowClear: true, + showSearch: true, + filterOption: (keyword: string, option: { label: any }) => { + return (option?.label ?? '').toLowerCase().includes(keyword.toLowerCase()); + }, }, }, { @@ -105,6 +115,11 @@ export const getFormContent = inlongGroupId => [ props: { dropdownMatchSelectWidth: false, options: typeList, + allowClear: true, + showSearch: true, + filterOption: (keyword: string, option: { label: any }) => { + return (option?.label ?? '').toLowerCase().includes(keyword.toLowerCase()); + }, }, }, ]; diff --git a/inlong-dashboard/src/ui/pages/GroupDetail/index.tsx b/inlong-dashboard/src/ui/pages/GroupDetail/index.tsx index 02961f38b4e..93a119634a5 100644 --- a/inlong-dashboard/src/ui/pages/GroupDetail/index.tsx +++ b/inlong-dashboard/src/ui/pages/GroupDetail/index.tsx @@ -64,7 +64,6 @@ const Comp: React.FC = () => { ready: !!id, refreshDeps: [id], onSuccess: result => { - console.log('res', result, getLocalStorage('tenant')?.['name']); if (getLocalStorage('tenant')?.['name'] !== result) { setLocalStorage({ name: result }); message.success(t('components.Layout.Tenant.Success')); diff --git a/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/config.tsx b/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/config.tsx index e3821a15ecf..423c0967749 100644 --- a/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/config.tsx +++ b/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/config.tsx @@ -22,8 +22,7 @@ import { Button } from 'antd'; import dayjs from 'dayjs'; import i18n from '@/i18n'; import { sinks } from '@/plugins/sinks'; -import request from '@/core/utils/request'; - +import { CSVLink } from 'react-csv'; export const timeStaticsDimList = [ { label: i18n.t('pages.GroupDetail.Audit.Min'), @@ -69,7 +68,6 @@ function getAuditLabel(auditId: number, nodeType?: string) { } export const toChartData = (source, sourceDataMap) => { - console.log(source, sourceDataMap); const xAxisData = Object.keys(sourceDataMap ? sourceDataMap : '12345'); return { legend: { @@ -158,7 +156,17 @@ export const getSourceDataWithCommas = sourceData => { let endTimeVisible = true; -export const getFormContent = (initialValues, onSearch, onDataStreamSuccess, auditData) => [ +export const getFormContent = ( + initialValues, + onSearch, + onDataStreamSuccess, + auditData, + sourceData, + csvData, + setInlongGroupId, + setInlongStreamID, + fileName, +) => [ { type: 'select', label: i18n.t('pages.ModuleAudit.config.InlongGroupId'), @@ -167,6 +175,9 @@ export const getFormContent = (initialValues, onSearch, onDataStreamSuccess, aud dropdownMatchSelectWidth: false, showSearch: true, allowClear: true, + onChange: (value, option) => { + setInlongGroupId(value); + }, options: { requestAuto: true, requestTrigger: ['onOpen', 'onSearch'], @@ -200,6 +211,9 @@ export const getFormContent = (initialValues, onSearch, onDataStreamSuccess, aud showSearch: true, allowClear: true, disabled: !Boolean(values.inlongGroupId), + onChange: (value, option) => { + setInlongStreamID(value); + }, options: { requestAuto: true, requestTrigger: ['onOpen', 'onSearch'], @@ -249,7 +263,6 @@ export const getFormContent = (initialValues, onSearch, onDataStreamSuccess, aud return Promise.resolve(); } const timeDiff = value - getFieldValue('startDate'); - console.log('timeDiff', value, getFieldValue('startDate'), timeDiff); if (timeDiff >= 0) { const isHourDiff = dim === 'HOUR' && timeDiff < 1000 * 60 * 60 * 24 * 3; const isDayDiff = dim === 'DAY' && timeDiff < 1000 * 60 * 60 * 24 * 7; @@ -325,6 +338,15 @@ export const getFormContent = (initialValues, onSearch, onDataStreamSuccess, aud ), }, + { + type: ( + + ), + }, ]; export const getTableColumns = (source, dim) => { diff --git a/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/index.tsx b/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/index.tsx index 8a4491b5e25..311d8be2d30 100644 --- a/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/index.tsx +++ b/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/index.tsx @@ -17,7 +17,7 @@ * under the License. */ -import React, { useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import FormGenerator, { useForm } from '@/ui/components/FormGenerator'; import HighTable from '@/ui/components/HighTable'; import { useRequest } from '@/ui/hooks'; @@ -45,7 +45,8 @@ const Comp: React.FC = ({ auditData }) => { endDate: +new Date(), timeStaticsDim: timeStaticsDimList[0].value, }); - + const [inlongStreamID, setInlongStreamID] = useState(''); + const [inlongGroupId, setInlongGroupId] = useState(''); const { data: sourceData = [], run } = useRequest( { url: '/audit/list', @@ -61,7 +62,17 @@ const Comp: React.FC = ({ auditData }) => { formatResult: result => result.sort((a, b) => (a.auditId - b.auditId > 0 ? 1 : -1)), }, ); - + const numToName = useCallback( + num => { + let obj = {}; + sourceData.forEach(item => { + obj = { ...obj, [item.auditId]: item.auditName }; + }); + obj = { ...obj, logTs: i18n.t('pages.GroupDetail.Audit.Time') }; + return obj[num]; + }, + [sourceData], + ); const sourceDataMap = useMemo(() => { const flatArr = sourceData .reduce( @@ -99,7 +110,29 @@ const Comp: React.FC = ({ auditData }) => { } run(); }; + const metricSum = useMemo(() => { + let obj = { logTs: i18n.t('pages.GroupDetail.Audit.Total') }; + sourceData.map(item => { + const sum = item.auditSet?.reduce((total, cur) => { + return total + cur.count; + }, 0); + obj = { ...obj, [item.auditId]: sum }; + }); + return obj; + }, [sourceData]); + const csvData = useMemo(() => { + const result = [...toTableData(sourceData, sourceDataMap), metricSum].map(item => { + let obj = {}; + Object.keys(item) + .reverse() + .forEach(key => { + obj = { ...obj, [numToName(key)]: item[key] }; + }); + return obj; + }); + return result; + }, [sourceData, sourceDataMap, metricSum]); const onDataStreamSuccess = data => { const defaultDataStream = data[0]?.value; if (defaultDataStream) { @@ -108,15 +141,28 @@ const Comp: React.FC = ({ auditData }) => { run(); } }; - + const [fileName, setFileName] = useState('metrics.csv'); + useEffect(() => { + setFileName(`metrics_${inlongGroupId}_${inlongStreamID}.csv`); + }, [inlongGroupId, inlongStreamID]); return ( <>
setQuery({ ...allValues, diff --git a/inlong-dashboard/src/ui/pages/ModuleAudit/IdModule/config.tsx b/inlong-dashboard/src/ui/pages/ModuleAudit/IdModule/config.tsx index cd868b67972..b3803906f59 100644 --- a/inlong-dashboard/src/ui/pages/ModuleAudit/IdModule/config.tsx +++ b/inlong-dashboard/src/ui/pages/ModuleAudit/IdModule/config.tsx @@ -22,7 +22,9 @@ import i18n from '@/i18n'; import request from '@/core/utils/request'; import { Button } from 'antd'; import React from 'react'; - +import { SortOrder } from 'antd/es/table/interface'; +import { range } from 'lodash'; +import { CSVLink } from 'react-csv'; export const timeStaticsDimList = [ { label: i18n.t('pages.GroupDetail.Audit.Min'), @@ -38,7 +40,20 @@ export const timeStaticsDimList = [ }, ]; +export const sumSubValue = sourceDataMap => { + if (sourceDataMap === null || sourceDataMap === undefined) { + return 0; + } + return Object.keys(sourceDataMap).reduce((acc, cur) => { + const element = sourceDataMap[cur]; + acc += element.subValue; + return acc; + }, 0); +}; export const toTableData = (source, sourceDataMap) => { + if (sourceDataMap === null || sourceDataMap === undefined) { + return []; + } return Object.keys(sourceDataMap) .reverse() .map(logTs => ({ @@ -47,7 +62,16 @@ export const toTableData = (source, sourceDataMap) => { })); }; -export const getFormContent = (initialValues, onSearch, auditData) => [ +export const getFormContent = ( + initialValues, + onSearch, + auditData, + sourceData, + csvData, + setInlongGroupId, + setInlongStreamID, + fileName, +) => [ { type: 'select', label: i18n.t('pages.ModuleAudit.config.InlongGroupId'), @@ -55,6 +79,9 @@ export const getFormContent = (initialValues, onSearch, auditData) => [ props: { dropdownMatchSelectWidth: false, showSearch: true, + onChange: (value, option) => { + setInlongGroupId(value); + }, options: { requestAuto: true, requestTrigger: ['onOpen', 'onSearch'], @@ -85,6 +112,9 @@ export const getFormContent = (initialValues, onSearch, auditData) => [ props: values => ({ dropdownMatchSelectWidth: false, showSearch: true, + onChange: (value, option) => { + setInlongStreamID(value); + }, options: { requestAuto: true, requestTrigger: ['onOpen', 'onSearch'], @@ -116,7 +146,12 @@ export const getFormContent = (initialValues, onSearch, auditData) => [ props: { allowClear: false, showTime: true, - format: 'YYYY-MM-DD HH:mm:ss', + format: 'YYYY-MM-DD HH:mm', + disabledTime: (date: dayjs.Dayjs, type, info: { from?: dayjs.Dayjs }) => { + return { + disabledSeconds: () => range(0, 60), + }; + }, }, }, { @@ -127,7 +162,12 @@ export const getFormContent = (initialValues, onSearch, auditData) => [ props: { allowClear: false, showTime: true, - format: 'YYYY-MM-DD HH:mm:ss', + format: 'YYYY-MM-DD HH:mm', + disabledTime: (date: dayjs.Dayjs, type, info: { from?: dayjs.Dayjs }) => { + return { + disabledSeconds: () => range(0, 60), + }; + }, }, }, { @@ -181,18 +221,56 @@ export const getFormContent = (initialValues, onSearch, auditData) => [ ), }, + { + type: ( + + ), + }, ]; -export const getTableColumns = source => { +const strSorter = (a, b) => { + return a?.ip.localeCompare(b?.ip); +}; +const sortOrder: SortOrder = 'descend'; + +const baseSorter = (a, b) => { + return a.base - b.base; +}; +const comparedSorter = (a, b) => { + return a.compared - b.compared; +}; +const subValueSorter = (a, b) => { + return a.subValue - b.subValue; +}; + +export const getTableColumns = (source: any) => { const data = source.map(item => ({ title: item.auditName, - dataIndex: item.auditId, + dataIndex: source[0].auditId === item.auditId ? 'base' : 'compared', + key: source[0].auditId === item.auditId ? 'base' : 'compared', + sorter: { + compare: source[0].auditId === item.auditId ? baseSorter : comparedSorter, + multiple: source[0].auditId === item.auditId ? 3 : 4, + }, render: text => text || 0, })); return [ { title: i18n.t('pages.ModuleAudit.config.Ip'), dataIndex: 'ip', + defaultSortOrder: sortOrder, + sorter: strSorter, }, - ].concat(data); + ] + .concat(data) + .concat({ + title: i18n.t('pages.ModuleAudit.config.SubValue'), + dataIndex: 'subValue', + defaultSortOrder: null, + sorter: subValueSorter, + }); }; diff --git a/inlong-dashboard/src/ui/pages/ModuleAudit/IdModule/index.tsx b/inlong-dashboard/src/ui/pages/ModuleAudit/IdModule/index.tsx index 65fb7bd0534..9deebc3d0f4 100644 --- a/inlong-dashboard/src/ui/pages/ModuleAudit/IdModule/index.tsx +++ b/inlong-dashboard/src/ui/pages/ModuleAudit/IdModule/index.tsx @@ -17,25 +17,29 @@ * under the License. */ -import React, { useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import HighTable, { useForm } from '@/ui/components/HighTable'; import { useRequest } from '@/ui/hooks'; import { timestampFormat } from '@/core/utils'; -import { getFormContent, toTableData, getTableColumns } from './config'; +import { getFormContent, toTableData, getTableColumns, sumSubValue } from './config'; +import i18n from '@/i18n'; import { AuditProps } from '@/ui/pages/ModuleAudit'; +import { Table } from 'antd'; +import dayjs from 'dayjs'; export const idModule = 'id'; const Comp: React.FC = ({ auditData }) => { const [form] = useForm(); const [query, setQuery] = useState({ - startDate: +new Date(), - endDate: +new Date(), + startDate: dayjs().startOf('hour').valueOf(), + endDate: dayjs().startOf('hour').valueOf(), auditIds: ['3', '4'], inlongGroupId: '', inlongStreamId: '', }); - + const [inlongStreamID, setInlongStreamID] = useState(''); + const [inlongGroupId, setInlongGroupId] = useState(''); const { data: sourceData = [], run } = useRequest( { url: '/audit/listAll', @@ -48,32 +52,49 @@ const Comp: React.FC = ({ auditData }) => { }, { refreshDeps: [query], - formatResult: result => result.sort((a, b) => (a.auditId - b.auditId > 0 ? 1 : -1)), + formatResult: result => { + const base = result.find(item2 => item2.auditId === query.auditIds[0].toString()); + const compared = result.find(item2 => item2.auditId === query.auditIds[1].toString()); + return [base, compared]; + }, }, ); const sourceDataMap = useMemo(() => { - const flatArr = sourceData.reduce( - (acc, cur) => - acc.concat( - cur.auditSet.map(item => ({ - ...item, - auditId: cur.auditId, - })), - ), - [], - ); - const output = flatArr.reduce((acc, cur) => { - if (!acc[cur.ip]) { - acc[cur.ip] = {}; - } + if (!sourceData) { + return {}; + } + let baseData = + sourceData[0]?.auditSet?.length > sourceData[1]?.auditSet?.length + ? sourceData[0] + : sourceData[1]; + const output = baseData?.auditSet?.reduce((acc, cur) => { acc[cur.ip] = { - ...acc[cur.ip], - [cur.auditId]: cur.count, ip: cur.ip, + base: + sourceData[0].auditId === baseData.auditId + ? cur.count + : sourceData[0].auditSet.find(item => item.ip === cur.ip) + ? sourceData[0].auditSet.find(item => item.ip === cur.ip).count + : 0, + compared: + sourceData[1].auditId === baseData.auditId + ? cur.count + : sourceData[1].auditSet.find(item => item.ip === cur.ip) + ? sourceData[1].auditSet.find(item => item.ip === cur.ip).count + : 0, }; return acc; }, {}); + if (output === undefined || output === null) { + return {}; + } + Object.keys(output).forEach(key => { + output[key] = { + ...output[key], + subValue: output[key].compared - output[key].base, + }; + }); return output; }, [sourceData]); @@ -86,28 +107,84 @@ const Comp: React.FC = ({ auditData }) => { setQuery({ ...query, ...keyword, - auditIds: - keyword.benchmark !== undefined && keyword.compared !== undefined - ? [keyword.benchmark, keyword.compared] - : ['3', '4'], + auditIds: [ + keyword.benchmark !== undefined ? keyword.benchmark : query.auditIds[0], + keyword.compared !== undefined ? keyword.compared : query.auditIds[1], + ], inlongGroupId: keyword.inlongGroupId, inlongStreamId: keyword.inlongStreamId, startDate: +keyword.startDate.$d, endDate: keyword.endDate === undefined ? +keyword.startDate.$d : +keyword.endDate.$d, }); }; + const numToName = useCallback( + num => { + let obj = {}; + obj = { + base: sourceData[0].auditName, + compared: sourceData[1].auditName, + ip: i18n.t('pages.ModuleAudit.config.Ip'), + subValue: i18n.t('pages.ModuleAudit.config.SubValue'), + }; + return obj[num]; + }, + [sourceData], + ); + const csvData = useMemo(() => { + const result = toTableData(sourceData, sourceDataMap).map(item => { + let obj = {}; + Object.keys(item) + .filter(key => key !== 'logTs') + .forEach(key => { + obj = { ...obj, [numToName(key)]: item[key] }; + }); + return obj; + }); + return result; + }, [sourceData, sourceDataMap]); + const [fileName, setFileName] = useState('metrics.csv'); + useEffect(() => { + setFileName(`id_${inlongGroupId}_${inlongStreamID}.csv`); + }, [inlongGroupId, inlongStreamID]); return ( <> ( + + + + {i18n.t('pages.GroupDetail.Audit.Total')} + + {sourceData.map((row, index) => ( + + {row.auditSet.reduce((total, item) => total + item.count, 0).toLocaleString()} + + ))} + + {sumSubValue(sourceDataMap).toLocaleString()} + + + + ), }} /> diff --git a/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/config.tsx b/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/config.tsx index 1b6dc36e886..9edc4a251ee 100644 --- a/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/config.tsx +++ b/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/config.tsx @@ -22,7 +22,9 @@ import i18n from '@/i18n'; import request from '@/core/utils/request'; import { Button } from 'antd'; import React from 'react'; - +import { CSVLink } from 'react-csv'; +import { range } from 'lodash'; +import { SortOrder } from 'antd/es/table/interface'; export const toChartData = (source, sourceDataMap) => { const xAxisData = Object.keys(sourceDataMap); return { @@ -48,6 +50,9 @@ export const toChartData = (source, sourceDataMap) => { }; export const toTableData = (source, sourceDataMap) => { + if (sourceDataMap === null || sourceDataMap === undefined) { + return []; + } return Object.keys(sourceDataMap) .reverse() .map(logTs => ({ @@ -56,10 +61,23 @@ export const toTableData = (source, sourceDataMap) => { })); }; -export const getFormContent = (initialValues, onSearch, auditData) => [ +export const getFormContent = ( + initialValues, + onSearch, + auditData, + sourceData, + csvData, + setIp, + fileName, +) => [ { type: 'input', label: i18n.t('pages.ModuleAudit.config.Ip'), + props: { + onChange: (e: any) => { + setIp(e.target.value); + }, + }, name: 'ip', }, { @@ -70,18 +88,28 @@ export const getFormContent = (initialValues, onSearch, auditData) => [ props: { allowClear: false, showTime: true, - format: 'YYYY-MM-DD HH:mm:ss', + format: 'YYYY-MM-DD HH:mm', + disabledTime: (date: dayjs.Dayjs, type, info: { from?: dayjs.Dayjs }) => { + return { + disabledSeconds: () => range(0, 60), + }; + }, }, }, { type: 'datepicker', label: i18n.t('pages.GroupDetail.Audit.EndDate'), name: 'endDate', - initialValues: dayjs(initialValues.endDate), + initialValues: dayjs().startOf('hour').valueOf(), props: { allowClear: false, showTime: true, - format: 'YYYY-MM-DD HH:mm:ss', + format: 'YYYY-MM-DD HH:mm', + disabledTime: (date: dayjs.Dayjs, type, info: { from?: dayjs.Dayjs }) => { + return { + disabledSeconds: () => range(0, 60), + }; + }, }, }, { @@ -135,22 +163,74 @@ export const getFormContent = (initialValues, onSearch, auditData) => [ ), }, + { + type: ( + + ), + }, ]; - +const baseSorter = (a, b) => { + return a.base - b.base; +}; +const comparedSorter = (a, b) => { + return a.compared - b.compared; +}; +const subValueSorter = (a, b) => { + return a.subValue - b.subValue; +}; +const groupIdStrSorter = (a, b) => { + return a?.inlongGroupId.localeCompare(b.inlongGroupId); +}; +const streamIdStrSorter = (a, b) => { + return a?.inlongStreamId.localeCompare(b?.inlongStreamId); +}; +const sortOrder: SortOrder = 'descend'; export const getTableColumns = source => { const data = source.map(item => ({ title: item.auditName, - dataIndex: item.auditId, + dataIndex: source[0].auditId === item.auditId ? 'base' : 'compared', + key: source[0].auditId === item.auditId ? 'base' : 'compared', + sorter: { + compare: source[0].auditId === item.auditId ? baseSorter : comparedSorter, + multiple: source[0].auditId === item.auditId ? 3 : 4, + }, render: text => text || 0, })); return [ { title: i18n.t('pages.ModuleAudit.config.InlongGroupId'), dataIndex: 'inlongGroupId', + key: 'inlongGroupId', + sorter: { + compare: groupIdStrSorter, + multiple: 1, + }, }, { title: i18n.t('pages.ModuleAudit.config.InlongStreamId'), dataIndex: 'inlongStreamId', + key: 'inlongStreamId', + defaultSortOrder: sortOrder, + sorter: { + compare: streamIdStrSorter, + multiple: 2, + }, }, - ].concat(data); + ] + .concat(data) + .concat([ + { + title: i18n.t('pages.ModuleAudit.config.SubValue'), + dataIndex: 'subValue', + key: 'subValue', + sorter: { + compare: subValueSorter, + multiple: 5, + }, + }, + ]); }; diff --git a/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/index.tsx b/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/index.tsx index ef24a3e7ddd..9aab3b903c6 100644 --- a/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/index.tsx +++ b/inlong-dashboard/src/ui/pages/ModuleAudit/IpModule/index.tsx @@ -17,22 +17,27 @@ * under the License. */ -import React, { useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useForm } from '@/ui/components/FormGenerator'; import HighTable from '@/ui/components/HighTable'; import { useRequest } from '@/ui/hooks'; import { timestampFormat } from '@/core/utils'; import { getFormContent, toTableData, getTableColumns } from './config'; +import i18n from '@/i18n'; import { AuditProps } from '@/ui/pages/ModuleAudit'; +import { Table } from 'antd'; +import { sumSubValue } from '@/ui/pages/ModuleAudit/IdModule/config'; +import dayjs from 'dayjs'; export const ipModule = 'ip'; const Comp: React.FC = ({ auditData }) => { const [form] = useForm(); const [query, setQuery] = useState({ - startDate: +new Date(), - endDate: +new Date(), + startDate: dayjs().startOf('hour').valueOf(), + endDate: dayjs().startOf('hour').valueOf(), auditIds: ['3', '4'], + ip: '', }); const { data: sourceData = [], run } = useRequest( @@ -47,33 +52,67 @@ const Comp: React.FC = ({ auditData }) => { }, { refreshDeps: [query], - formatResult: result => result.sort((a, b) => (a.auditId - b.auditId > 0 ? 1 : -1)), + formatResult: result => { + const base = result.find(item2 => item2.auditId === query.auditIds[0].toString()); + const compared = result.find(item2 => item2.auditId === query.auditIds[1].toString()); + return [base, compared]; + }, }, ); const sourceDataMap = useMemo(() => { - const flatArr = sourceData.reduce( - (acc, cur) => - acc.concat( - cur.auditSet.map(item => ({ - ...item, - auditId: cur.auditId, - })), - ), - [], - ); - const output = flatArr.reduce((acc, cur) => { - if (!acc[cur.inlongStreamId]) { - acc[cur.inlongStreamId] = {}; - } - acc[cur.inlongStreamId] = { - ...acc[cur.inlongStreamId], - [cur.auditId]: cur.count, + if (!sourceData) { + return {}; + } + let baseData = + sourceData[0]?.auditSet?.length > sourceData[1]?.auditSet?.length + ? sourceData[0] + : sourceData[1]; + const output = baseData?.auditSet?.reduce((acc, cur) => { + acc[cur.inlongGroupId + cur.inlongStreamId] = { inlongGroupId: cur.inlongGroupId, inlongStreamId: cur.inlongStreamId, + base: + sourceData[0].auditId === baseData.auditId + ? cur.count + : sourceData[0].auditSet.find(item => { + return ( + item.inlongGroupId + item.inlongStreamId === + cur.inlongGroupId + cur.inlongStreamId + ); + }) + ? sourceData[0].auditSet.find( + item => + item.inlongGroupId + item.inlongStreamId === + cur.inlongGroupId + cur.inlongStreamId, + ).count + : 0, + compared: + sourceData[1].auditId === baseData.auditId + ? cur.count + : sourceData[1].auditSet.find( + item => + item.inlongGroupId + item.inlongStreamId === + cur.inlongGroupId + cur.inlongStreamId, + ) + ? sourceData[1].auditSet.find( + item => + item.inlongGroupId + item.inlongStreamId === + cur.inlongGroupId + cur.inlongStreamId, + ).count + : 0, }; return acc; }, {}); + if (output === undefined || output === null) { + return {}; + } + Object.keys(output).forEach(key => { + output[key] = { + ...output[key], + subValue: output[key].compared - output[key].base, + }; + }); return output; }, [sourceData]); @@ -87,26 +126,82 @@ const Comp: React.FC = ({ auditData }) => { ...query, ...keyword, ip: keyword.ip, - auditIds: - keyword.benchmark !== undefined && keyword.compared !== undefined - ? [keyword.benchmark, keyword.compared] - : ['3', '4'], + auditIds: [ + keyword.benchmark !== undefined ? keyword.benchmark : query.auditIds[0], + keyword.compared !== undefined ? keyword.compared : query.auditIds[1], + ], startDate: +keyword.startDate.$d, endDate: keyword.endDate === undefined ? +keyword.startDate.$d : +keyword.endDate.$d, }); }; + const numToName = useCallback( + num => { + let obj = { + inlongGroupId: i18n.t('pages.ModuleAudit.config.InlongGroupId'), + inlongStreamId: i18n.t('pages.ModuleAudit.config.InlongStreamId'), + subValue: i18n.t('pages.ModuleAudit.config.SubValue'), + base: sourceData[0].auditName, + compared: sourceData[1].auditName, + }; + return obj[num]; + }, + [sourceData], + ); + const csvData = useMemo(() => { + if (!sourceData) { + return {}; + } + const result = toTableData(sourceData, sourceDataMap).map(item => { + let obj = {}; + Object.keys(item) + .filter(key => key !== 'logTs') + .forEach(key => { + obj = { ...obj, [numToName(key)]: item[key] }; + }); + return obj; + }); + return result; + }, [sourceData, sourceDataMap]); + const [ip, setIp] = useState(''); + const [fileName, setFileName] = useState('ip.csv'); + useEffect(() => { + setFileName('ip_' + ip + '.csv'); + setQuery({ + ...query, + ip, + }); + }, [ip]); return ( <> ( + + + + {i18n.t('pages.GroupDetail.Audit.Total')} + + + {sourceData.map((row, index) => ( + + {row.auditSet.reduce((total, item) => total + item.count, 0).toLocaleString()} + + ))} + + {sumSubValue(sourceDataMap).toLocaleString()} + + + + ), }} /> diff --git a/inlong-dashboard/src/ui/pages/SynchronizeDashboard/index.tsx b/inlong-dashboard/src/ui/pages/SynchronizeDashboard/index.tsx index b096f9957f6..33fc2202c6f 100644 --- a/inlong-dashboard/src/ui/pages/SynchronizeDashboard/index.tsx +++ b/inlong-dashboard/src/ui/pages/SynchronizeDashboard/index.tsx @@ -170,6 +170,10 @@ const Comp: React.FC = () => { allowClear: true, options: statusList, dropdownMatchSelectWidth: false, + showSearch: true, + filterOption: (keyword: string, option: { label: any }) => { + return (option?.label ?? '').toLowerCase().includes(keyword.toLowerCase()); + }, }, }, { diff --git a/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/config.tsx b/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/config.tsx index 78aac3df652..9d227ecdd16 100644 --- a/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/config.tsx +++ b/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/config.tsx @@ -85,6 +85,7 @@ export const useFormContent = ({ mqType, editing, isCreate, isUpdate }) => { return ( !defaultGroupKeysI18nMap[formName] || [ + 'scheduleEngine', 'scheduleType', 'time', 'crontabExpression', @@ -149,6 +150,7 @@ function transType(editing: boolean, conf) { 'ttl', 'retentionTime', 'retentionSize', + 'scheduleEngine', 'scheduleType', 'scheduleUnit', 'scheduleInterval', diff --git a/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/index.tsx b/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/index.tsx index e6325dcf5dd..f5ed2767617 100644 --- a/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/index.tsx +++ b/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/index.tsx @@ -138,11 +138,10 @@ const Comp = ({ inlongGroupId, inlongStreamId, readonly, isCreate }: Props, ref) const onOk = async () => { const values = await form.validateFields(); - let submitData = { ...values, version: data?.version, - inCharges: values.inCharges?.join(','), + inCharges: values.inCharges instanceof Array ? values.inCharges?.join(',') : values.inCharges, }; if (values.inlongGroupMode === 2) { submitData = { @@ -228,7 +227,13 @@ const Comp = ({ inlongGroupId, inlongStreamId, readonly, isCreate }: Props, ref) form={form} content={formContent} initialValues={data} - onValuesChange={(c, values) => setMqType(values.mqType)} + onValuesChange={(c, values) => { + setMqType(values.mqType); + if (c.scheduleEngine === 'DolphinScheduler') { + form.setFieldsValue({ scheduleType: 1 }); + values.scheduleType = 1; + } + }} useMaxWidth={1400} col={14} labelWrap diff --git a/inlong-distribution/pom.xml b/inlong-distribution/pom.xml index 95c54adf749..473080c3230 100644 --- a/inlong-distribution/pom.xml +++ b/inlong-distribution/pom.xml @@ -33,32 +33,6 @@ ${project.parent.basedir} - - - - org.codehaus.mojo - exec-maven-plugin - ${exec.maven.version} - - bash - - +x - ${basedir}/script/backup_module_dependencies.sh - - - - - uncompress - - exec - - package - - - - - - flink-all-version @@ -103,6 +77,27 @@ + + org.codehaus.mojo + exec-maven-plugin + ${exec.maven.version} + + bash + + +x + ${basedir}/script/backup_module_dependencies.sh + + + + + uncompress + + exec + + package + + + @@ -144,6 +139,27 @@ + + org.codehaus.mojo + exec-maven-plugin + ${exec.maven.version} + + bash + + +x + ${basedir}/script/backup_module_dependencies.sh + + + + + uncompress + + exec + + package + + + @@ -185,6 +201,27 @@ + + org.codehaus.mojo + exec-maven-plugin + ${exec.maven.version} + + bash + + +x + ${basedir}/script/backup_module_dependencies.sh + + + + + uncompress + + exec + + package + + + @@ -226,6 +263,27 @@ + + org.codehaus.mojo + exec-maven-plugin + ${exec.maven.version} + + bash + + +x + ${basedir}/script/backup_module_dependencies.sh + + + + + uncompress + + exec + + package + + + diff --git a/inlong-distribution/script/backup_module_dependencies.sh b/inlong-distribution/script/backup_module_dependencies.sh index 880730d7644..6278ce3c2a3 100644 --- a/inlong-distribution/script/backup_module_dependencies.sh +++ b/inlong-distribution/script/backup_module_dependencies.sh @@ -34,11 +34,13 @@ filelist() { } # Get the current version for InLong -find_gz_file=`ls -l ./target/*bin.tar.gz |awk '{print $9}'` +prefix="apache-inlong-" +suffix="-bin.tar.gz" +find_gz_file=$(ls -l ./target/*bin.tar.gz |awk '{print $9}') gz_file=$(basename "$find_gz_file") -name_length=`expr length $gz_file` -version_length=$(expr $name_length \- 15 \- 10) -project_version=`expr substr $gz_file 15 $version_length` +version_with_suffix="${gz_file#$prefix}" +project_version="${version_with_suffix%$suffix}" +echo "The current version for InLong is: $project_version" projectpath="./target/apache-inlong-${project_version}-bin/apache-inlong-${project_version}" # Generate the file "dependencys.txt" diff --git a/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java index 6d301703fcc..a9798c91f1f 100644 --- a/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java +++ b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java @@ -33,6 +33,8 @@ public class ScheduleEntity implements Serializable { private String inlongGroupId; // schedule type, support [normal, crontab], 0 for normal and 1 for crontab private Integer scheduleType; + // schedule engine type, support [Quartz, Airflow, DolphinScheduler] + private String scheduleEngine; // time unit for offline task schedule interval, support [month, week, day, hour, minute, oneround] // Y=year, M=month, W=week, D=day, H=hour, I=minute, O=oneround private String scheduleUnit; diff --git a/inlong-manager/manager-dao/src/main/resources/mappers/ScheduleEntityMapper.xml b/inlong-manager/manager-dao/src/main/resources/mappers/ScheduleEntityMapper.xml index d719aa89884..33d25ad78a5 100644 --- a/inlong-manager/manager-dao/src/main/resources/mappers/ScheduleEntityMapper.xml +++ b/inlong-manager/manager-dao/src/main/resources/mappers/ScheduleEntityMapper.xml @@ -22,6 +22,7 @@ + @@ -42,25 +43,25 @@ - id, inlong_group_id, schedule_type, schedule_unit, schedule_interval, start_time, + id, inlong_group_id, schedule_type, schedule_engine, schedule_unit, schedule_interval, start_time, end_time, delay_time, self_depend, task_parallelism, crontab_expression, status, previous_status, is_deleted, creator, modifier, create_time, modify_time, version - insert into schedule_config (id, inlong_group_id, schedule_type, schedule_unit, - schedule_interval, start_time, end_time, delay_time, - self_depend, task_parallelism, crontab_expression, + insert into schedule_config (id, inlong_group_id, schedule_type, schedule_engine, + schedule_unit, schedule_interval, start_time, end_time, + delay_time, self_depend, task_parallelism, crontab_expression, status, previous_status, creator, modifier) values (#{id, jdbcType=INTEGER}, #{inlongGroupId, jdbcType=VARCHAR}, - #{scheduleType, jdbcType=INTEGER}, #{scheduleUnit, jdbcType=VARCHAR}, - #{scheduleInterval, jdbcType=INTEGER}, #{startTime, jdbcType=TIMESTAMP}, - #{endTime, jdbcType=TIMESTAMP}, #{delayTime, jdbcType=INTEGER}, - #{selfDepend, jdbcType=INTEGER}, #{taskParallelism, jdbcType=INTEGER}, - #{crontabExpression, jdbcType=VARCHAR}, #{status,jdbcType=INTEGER}, - #{previousStatus,jdbcType=INTEGER}, #{creator,jdbcType=VARCHAR}, - #{modifier,jdbcType=VARCHAR}) + #{scheduleType, jdbcType=INTEGER}, #{scheduleEngine, jdbcType=VARCHAR}, + #{scheduleUnit, jdbcType=VARCHAR}, #{scheduleInterval, jdbcType=INTEGER}, + #{startTime, jdbcType=TIMESTAMP}, #{endTime, jdbcType=TIMESTAMP}, + #{delayTime, jdbcType=INTEGER}, #{selfDepend, jdbcType=INTEGER}, + #{taskParallelism, jdbcType=INTEGER}, #{crontabExpression, jdbcType=VARCHAR}, + #{status,jdbcType=INTEGER}, #{previousStatus,jdbcType=INTEGER}, + #{creator,jdbcType=VARCHAR}, #{modifier,jdbcType=VARCHAR})