From 27dc5020ecece6ffae3c8580303b7aa829ad7f42 Mon Sep 17 00:00:00 2001 From: wenweihuang Date: Tue, 2 Apr 2024 11:36:52 +0800 Subject: [PATCH 01/13] [INLONG-9909][Agent] Add unit test for installer --- .../agent-installer/conf/installer.properties | 12 +- inlong-agent/agent-installer/pom.xml | 18 ++ .../agent/installer/ManagerFetcher.java | 12 +- .../inlong/agent/installer/ModuleManager.java | 20 +- .../test/java/installer/BaseTestsHelper.java | 105 +++++++++ .../java/installer/ModuleActionTypeEnum.java | 57 +++++ .../java/installer/TestModuleManager.java | 204 ++++++++++++++++++ .../src/test/resources/conf/modules.json | 1 + .../src/test/resources/log4j2.xml | 46 ++++ 9 files changed, 453 insertions(+), 22 deletions(-) create mode 100755 inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java create mode 100644 inlong-agent/agent-installer/src/test/java/installer/ModuleActionTypeEnum.java create mode 100755 inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java create mode 100644 inlong-agent/agent-installer/src/test/resources/conf/modules.json create mode 100644 inlong-agent/agent-installer/src/test/resources/log4j2.xml diff --git a/inlong-agent/agent-installer/conf/installer.properties b/inlong-agent/agent-installer/conf/installer.properties index 52b4ce071f9..85b2de5bfc5 100755 --- a/inlong-agent/agent-installer/conf/installer.properties +++ b/inlong-agent/agent-installer/conf/installer.properties @@ -18,21 +18,21 @@ ####################### # common config ####################### -local.ip=127.0.0.1 +agent.local.ip=127.0.0.1 agent.enable.oom.exit=false ############################ # manager config ############################ -manager.addr=http://127.0.0.1:8083 -manager.auth.secretId= -manager.auth.secretKey= +agent.manager.addr=http://127.0.0.1:8083 +agent.manager.auth.secretId= +agent.manager.auth.secretKey= ############################ # cluster config for automatically report and register ############################ -cluster.tag=default_cluster -cluster.name=default_agent +agent.cluster.tag=default_cluster +agent.cluster.name=default_agent ############################ # audit config diff --git a/inlong-agent/agent-installer/pom.xml b/inlong-agent/agent-installer/pom.xml index 4ce97f8344f..939df6ebe80 100644 --- a/inlong-agent/agent-installer/pom.xml +++ b/inlong-agent/agent-installer/pom.xml @@ -58,6 +58,24 @@ + + junit + junit + test + + + org.awaitility + awaitility + test + + + org.powermock + powermock-api-mockito2 + + + org.powermock + powermock-module-junit4 + diff --git a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java index ab17b6a5a93..63fbffa76eb 100644 --- a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java +++ b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java @@ -33,13 +33,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.inlong.agent.constant.AgentConstants.AGENT_CLUSTER_NAME; +import static org.apache.inlong.agent.constant.AgentConstants.AGENT_CLUSTER_TAG; +import static org.apache.inlong.agent.constant.AgentConstants.AGENT_LOCAL_IP; import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_FETCHER_INTERVAL; import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_RETURN_PARAM_DATA; import static org.apache.inlong.agent.constant.FetcherConstants.DEFAULT_AGENT_FETCHER_INTERVAL; import static org.apache.inlong.agent.constant.FetcherConstants.DEFAULT_INSTALLER_MANAGER_CONFIG_HTTP_PATH; import static org.apache.inlong.agent.constant.FetcherConstants.INSTALLER_MANAGER_CONFIG_HTTP_PATH; import static org.apache.inlong.agent.installer.ManagerResultFormatter.getResultData; -import static org.apache.inlong.agent.utils.AgentUtils.fetchLocalIp; import static org.apache.inlong.agent.utils.AgentUtils.fetchLocalUuid; /** @@ -47,8 +49,6 @@ */ public class ManagerFetcher extends AbstractDaemon implements ProfileFetcher { - public static final String CLUSTER_NAME = "cluster.name"; - public static final String CLUSTER_TAG = "cluster.tag"; private static final Logger LOGGER = LoggerFactory.getLogger(ManagerFetcher.class); private static final GsonBuilder gsonBuilder = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss"); private static final Gson GSON = gsonBuilder.create(); @@ -68,8 +68,8 @@ public ManagerFetcher(Manager manager) { httpManager = manager.getModuleManager().getHttpManager(conf); baseManagerUrl = httpManager.getBaseUrl(); staticConfigUrl = buildStaticConfigUrl(baseManagerUrl); - clusterTag = conf.get(CLUSTER_TAG); - clusterName = conf.get(CLUSTER_NAME); + clusterTag = conf.get(AGENT_CLUSTER_TAG); + clusterName = conf.get(AGENT_CLUSTER_NAME); } /** @@ -140,7 +140,7 @@ private Runnable configFetchThread() { @Override public void start() throws Exception { // when agent start, check local ip and fetch manager ip list; - localIp = fetchLocalIp(); + localIp = conf.get(AGENT_LOCAL_IP); uuid = fetchLocalUuid(); submitWorker(configFetchThread()); } diff --git a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java index 590644d6c3a..a860e9d5113 100755 --- a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java +++ b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java @@ -59,6 +59,9 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.stream.Collectors; +import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_ADDR; +import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_AUTH_SECRET_ID; +import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_AUTH_SECRET_KEY; import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_REQUEST_TIMEOUT; import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_VIP_HTTP_PREFIX_PATH; import static org.apache.inlong.agent.constant.FetcherConstants.DEFAULT_AGENT_MANAGER_REQUEST_TIMEOUT; @@ -70,9 +73,6 @@ */ public class ModuleManager extends AbstractDaemon { - public static final String MANAGER_ADDR = "manager.addr"; - public static final String MANAGER_AUTH_SECRET_ID = "manager.auth.secretId"; - public static final String MANAGER_AUTH_SECRET_KEY = "manager.auth.secretKey"; public static final int CONFIG_QUEUE_CAPACITY = 1; public static final int CORE_THREAD_SLEEP_TIME = 10000; public static final int DOWNLOAD_PACKAGE_READ_BUFF_SIZE = 1024 * 1024; @@ -97,18 +97,18 @@ public ModuleManager() { } public HttpManager getHttpManager(InstallerConfiguration conf) { - String managerAddr = conf.get(MANAGER_ADDR); + String managerAddr = conf.get(AGENT_MANAGER_ADDR); String managerHttpPrefixPath = conf.get(AGENT_MANAGER_VIP_HTTP_PREFIX_PATH, DEFAULT_AGENT_MANAGER_VIP_HTTP_PREFIX_PATH); int timeout = conf.getInt(AGENT_MANAGER_REQUEST_TIMEOUT, DEFAULT_AGENT_MANAGER_REQUEST_TIMEOUT); - String secretId = conf.get(MANAGER_AUTH_SECRET_ID); - String secretKey = conf.get(MANAGER_AUTH_SECRET_KEY); + String secretId = conf.get(AGENT_MANAGER_AUTH_SECRET_ID); + String secretKey = conf.get(AGENT_MANAGER_AUTH_SECRET_KEY); return new HttpManager(managerAddr, managerHttpPrefixPath, timeout, secretId, secretKey); } private boolean requiredKeys(InstallerConfiguration conf) { - return conf.hasKey(MANAGER_ADDR); + return conf.hasKey(AGENT_MANAGER_ADDR); } public void submitConfig(ConfigResult config) { @@ -318,12 +318,12 @@ private void deleteModule(ModuleConfig module) { private void updateModule(ModuleConfig localModule, ModuleConfig managerModule) { LOGGER.info("update module {} start", localModule.getId()); if (localModule.getPackageConfig().getMd5().equals(managerModule.getPackageConfig().getMd5())) { + LOGGER.info("package md5 no chang, will restart", localModule.getId()); + restartModule(localModule, managerModule); + } else { LOGGER.info("package md5 changed, will reinstall", localModule.getId()); deleteModule(localModule); addModule(managerModule); - } else { - LOGGER.info("package md5 no chang, will restart", localModule.getId()); - restartModule(localModule, managerModule); } LOGGER.info("update module {} end", localModule.getId()); } diff --git a/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java b/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java new file mode 100755 index 00000000000..88430a9af50 --- /dev/null +++ b/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package installer; + +import org.apache.inlong.agent.conf.TaskProfile; +import org.apache.inlong.agent.constant.AgentConstants; +import org.apache.inlong.agent.constant.FetcherConstants; +import org.apache.inlong.agent.installer.conf.InstallerConfiguration; +import org.apache.inlong.agent.pojo.FileTask.FileTaskConfig; +import org.apache.inlong.common.enums.TaskStateEnum; +import org.apache.inlong.common.pojo.agent.DataConfig; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * common environment setting up for test cases. + */ +public class BaseTestsHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseTestsHelper.class); + private static final GsonBuilder gsonBuilder = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss"); + private static final Gson GSON = gsonBuilder.create(); + private final String className; + private Path testRootDir; + + public BaseTestsHelper(String className) { + this.className = className; + } + + public BaseTestsHelper setupAgentHome() { + testRootDir = Paths + .get("/tmp", BaseTestsHelper.class.getSimpleName(), className); + teardownAgentHome(); + boolean result = testRootDir.toFile().mkdirs(); + LOGGER.info("try to create {}, result is {}", testRootDir, result); + InstallerConfiguration.getInstallerConf().set(AgentConstants.AGENT_HOME, testRootDir.toString()); + InstallerConfiguration.getInstallerConf().set(FetcherConstants.AGENT_MANAGER_ADDR, ""); + return this; + } + + public Path getTestRootDir() { + return testRootDir; + } + + public void teardownAgentHome() { + if (testRootDir != null) { + try { + FileUtils.deleteDirectory(testRootDir.toFile()); + } catch (Exception ignored) { + LOGGER.warn("deleteDirectory error ", ignored); + } + } + } + + public TaskProfile getTaskProfile(int taskId, String pattern, boolean retry, Long startTime, Long endTime, + TaskStateEnum state, String timeZone) { + DataConfig dataConfig = getDataConfig(taskId, pattern, retry, startTime, endTime, state, timeZone); + TaskProfile profile = TaskProfile.convertToTaskProfile(dataConfig); + return profile; + } + + private DataConfig getDataConfig(int taskId, String pattern, boolean retry, Long startTime, Long endTime, + TaskStateEnum state, String timeZone) { + DataConfig dataConfig = new DataConfig(); + dataConfig.setInlongGroupId("testGroupId"); + dataConfig.setInlongStreamId("testStreamId"); + dataConfig.setDataReportType(1); + dataConfig.setTaskType(3); + dataConfig.setTaskId(taskId); + dataConfig.setTimeZone(timeZone); + dataConfig.setState(state.ordinal()); + FileTaskConfig fileTaskConfig = new FileTaskConfig(); + fileTaskConfig.setPattern(pattern); + fileTaskConfig.setTimeOffset("0h"); + fileTaskConfig.setMaxFileCount(100); + fileTaskConfig.setCycleUnit("h"); + fileTaskConfig.setRetry(retry); + fileTaskConfig.setStartTime(startTime); + fileTaskConfig.setEndTime(endTime); + dataConfig.setExtParams(GSON.toJson(fileTaskConfig)); + return dataConfig; + } +} diff --git a/inlong-agent/agent-installer/src/test/java/installer/ModuleActionTypeEnum.java b/inlong-agent/agent-installer/src/test/java/installer/ModuleActionTypeEnum.java new file mode 100644 index 00000000000..d54979a6fcb --- /dev/null +++ b/inlong-agent/agent-installer/src/test/java/installer/ModuleActionTypeEnum.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package installer; + +/** + * Enum of module action. + */ +public enum ModuleActionTypeEnum { + + DOWNLOAD(0), + INSTALL(1), + UNINSTALL(2), + START(3), + STOP(4); + + private final int type; + + ModuleActionTypeEnum(int state) { + this.type = state; + } + + public static ModuleActionTypeEnum getTaskState(int state) { + switch (state) { + case 0: + return DOWNLOAD; + case 1: + return INSTALL; + case 2: + return UNINSTALL; + case 3: + return START; + case 4: + return STOP; + default: + throw new RuntimeException("Unsupported module action " + state); + } + } + + public int getType() { + return type; + } +} diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java new file mode 100755 index 00000000000..078758cc0ae --- /dev/null +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package installer; + +import org.apache.inlong.agent.installer.ModuleManager; +import org.apache.inlong.common.pojo.agent.installer.ConfigResult; +import org.apache.inlong.common.pojo.agent.installer.ModuleConfig; +import org.apache.inlong.common.pojo.agent.installer.ModuleStateEnum; +import org.apache.inlong.common.pojo.agent.installer.PackageConfig; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(ModuleManager.class) +@PowerMockIgnore({"javax.management.*"}) +public class TestModuleManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestModuleManager.class); + private static BaseTestsHelper helper; + private static final ClassLoader LOADER = TestModuleManager.class.getClassLoader(); + private static ModuleManager manager; + private static List realActionList = new ArrayList<>(); + private static List expectedActionList = new ArrayList<>(); + private static String OLD_MD5 = "95648c83b45971dce503d5d844496cfc"; + private static String NEW_MD5 = "e573f399da60ddeff09904bb95bdc307"; + + @Data + @Builder + @AllArgsConstructor + @NoArgsConstructor + private static class Action { + + public ModuleActionTypeEnum type; + public String md5; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Action action = (Action) o; + return Objects.equals(type, action.type) && + Objects.equals(md5, action.md5); + } + } + + @BeforeClass + public static void setup() { + helper = new BaseTestsHelper(TestModuleManager.class.getName()).setupAgentHome(); + manager = PowerMockito.spy(new ModuleManager()); + } + + @AfterClass + public static void teardown() throws Exception { + manager.stop(); + } + + private void fillExpectedActionList() { + expectedActionList.add(new Action(ModuleActionTypeEnum.STOP, OLD_MD5)); + expectedActionList.add(new Action(ModuleActionTypeEnum.UNINSTALL, OLD_MD5)); + expectedActionList.add(new Action(ModuleActionTypeEnum.DOWNLOAD, NEW_MD5)); + expectedActionList.add(new Action(ModuleActionTypeEnum.INSTALL, NEW_MD5)); + expectedActionList.add(new Action(ModuleActionTypeEnum.START, NEW_MD5)); + } + + private void mockFunctions() { + try { + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + realActionList.add(new Action(ModuleActionTypeEnum.DOWNLOAD, module.getPackageConfig().getMd5())); + return true; + }).when(manager, "downloadModule", Mockito.any()); + + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + realActionList.add(new Action(ModuleActionTypeEnum.INSTALL, module.getPackageConfig().getMd5())); + return true; + }).when(manager, "installModule", Mockito.any()); + + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + realActionList.add(new Action(ModuleActionTypeEnum.UNINSTALL, module.getPackageConfig().getMd5())); + return true; + }).when(manager, "uninstallModule", Mockito.any()); + + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + realActionList.add(new Action(ModuleActionTypeEnum.START, module.getPackageConfig().getMd5())); + return true; + }).when(manager, "startModule", Mockito.any()); + + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + realActionList.add(new Action(ModuleActionTypeEnum.STOP, module.getPackageConfig().getMd5())); + return true; + }).when(manager, "stopModule", Mockito.any()); + + PowerMockito.doReturn(null).when(manager, "getHttpManager", Mockito.any()); + } catch (Exception e) { + LOGGER.error("mock downloadModule error", e); + Assert.assertTrue(false); + return; + } + } + + @Test + public void testModuleManager() { + fillExpectedActionList(); + mockFunctions(); + String confPath = LOADER.getResource("conf/").getPath(); + manager.restoreFromLocalFile(confPath); + Assert.assertTrue(manager.getModule(1).getPackageConfig().getMd5().equals(OLD_MD5)); + manager.submitConfig(getConfig()); + try { + manager.start(); + } catch (Exception e) { + LOGGER.error("start module manager error", e); + Assert.assertTrue("start module manager error", false); + } + await().atMost(3, TimeUnit.SECONDS).until(() -> manager.getModule(1) != null); + Assert.assertTrue(realActionList.size() == expectedActionList.size()); + for (Integer i = 0; i < expectedActionList.size(); i++) { + LOGGER.info("{} {}", realActionList.get(i), expectedActionList.get(i)); + Assert.assertTrue(i.toString(), realActionList.get(i).equals(expectedActionList.get(i))); + } + } + + private ConfigResult getConfig() { + List configs = new ArrayList<>(); + configs.add(getModuleConfig(1, "inlong-agent", "inlong-agent-md5-185454", "1.0", 1, + "cd ~/inlong-agent/bin;sh agent.sh start", "cd ~/inlong-agent/bin;sh agent.sh stop", + "ps aux | grep core.AgentMain | grep java | grep -v grep | awk '{print $2}'", + "cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf ~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf agent-release-1.12.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd ~/inlong-agent/bin;sh agent.sh start", + "echo empty uninstall cmd", "agent-release-1.12.0-SNAPSHOT-bin.tar.gz", + "http://11.151.252.111:8083/inlong/manager/openapi/agent/download/agent-release-1.12.0-SNAPSHOT-bin.tar.gz", + NEW_MD5)); + return ConfigResult.builder().moduleList(configs).moduleNum(1).md5("config-result-md5-193603").build(); + } + + private ModuleConfig getModuleConfig(int id, String name, String md5, String version, Integer procNum, + String startCmd, String stopCmd, String checkCmd, String installCmd, String uninstallCmd, String fileName, + String downloadUrl, + String packageMd5) { + ModuleConfig moduleConfig = new ModuleConfig(); + moduleConfig.setId(id); + moduleConfig.setName(name); + moduleConfig.setVersion(version); + moduleConfig.setMd5(md5); + moduleConfig.setProcessesNum(procNum); + moduleConfig.setStartCommand(startCmd); + moduleConfig.setStopCommand(stopCmd); + moduleConfig.setCheckCommand(checkCmd); + moduleConfig.setInstallCommand(installCmd); + moduleConfig.setUninstallCommand(uninstallCmd); + PackageConfig packageConfig = new PackageConfig(); + packageConfig.setFileName(fileName); + packageConfig.setDownloadUrl(downloadUrl); + packageConfig.setStoragePath("/tmp"); + packageConfig.setMd5(packageMd5); + moduleConfig.setPackageConfig(packageConfig); + moduleConfig.setState(ModuleStateEnum.NEW); + return moduleConfig; + } +} diff --git a/inlong-agent/agent-installer/src/test/resources/conf/modules.json b/inlong-agent/agent-installer/src/test/resources/conf/modules.json new file mode 100644 index 00000000000..1b365c587c5 --- /dev/null +++ b/inlong-agent/agent-installer/src/test/resources/conf/modules.json @@ -0,0 +1 @@ +{"md5":"b38c63451ff966a32994a867ec79d259","moduleNum":1,"moduleList":[{"id":1,"name":"inlong-agent","md5":"55175a3b2cb143f31ad3d79e081e794c","version":"1.0","processesNum":1,"startCommand":"cd ~/inlong-agent/bin;sh agent.sh start","stopCommand":"cd ~/inlong-agent/bin;sh agent.sh stop","checkCommand":"ps aux | grep core.AgentMain | grep java | grep -v grep | awk \u0027{print $2}\u0027","installCommand":"cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf ~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf apache-inlong-agent-1.12.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd ~/inlong-agent/bin;sh agent.sh start","uninstallCommand":"echo empty uninstall cmd","packageConfig":{"md5":"95648c83b45971dce503d5d844496cfc","fileName":"apache-inlong-agent-1.12.0-SNAPSHOT-bin.tar.gz","downloadUrl":"http://11.151.246.158:8083/inlong/manager/openapi/installer/download?filename\u003dapache-inlong-agent-1.12.0-SNAPSHOT-bin.tar.gz","storagePath":"/tmp"},"state":"INSTALLED"}]} \ No newline at end of file diff --git a/inlong-agent/agent-installer/src/test/resources/log4j2.xml b/inlong-agent/agent-installer/src/test/resources/log4j2.xml new file mode 100644 index 00000000000..a80b45ae24a --- /dev/null +++ b/inlong-agent/agent-installer/src/test/resources/log4j2.xml @@ -0,0 +1,46 @@ + + + + + logs/core + %d{yyyy-MM-dd HH:mm:ss.SSS} -%5p ${PID:-} [%15.15t] %-30.30C{1.}:%L %m%n + DEBUG + ${basePath}/ut-debug.log + DEBUG + + + + + + + true + + + + + + + + + + + + + \ No newline at end of file From bdfb008bb20f9b77c2ec376d5a30445f51a4957b Mon Sep 17 00:00:00 2001 From: wenweihuang Date: Tue, 2 Apr 2024 11:40:08 +0800 Subject: [PATCH 02/13] [INLONG-9909][Agent] Add unit test for installer --- .../java/org/apache/inlong/agent/installer/ModuleManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java index a860e9d5113..4e619a46d47 100755 --- a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java +++ b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java @@ -318,7 +318,7 @@ private void deleteModule(ModuleConfig module) { private void updateModule(ModuleConfig localModule, ModuleConfig managerModule) { LOGGER.info("update module {} start", localModule.getId()); if (localModule.getPackageConfig().getMd5().equals(managerModule.getPackageConfig().getMd5())) { - LOGGER.info("package md5 no chang, will restart", localModule.getId()); + LOGGER.info("package md5 no change, will restart", localModule.getId()); restartModule(localModule, managerModule); } else { LOGGER.info("package md5 changed, will reinstall", localModule.getId()); From 41fdccc13ea5f2c17f78cbf3ec5d920f507a686d Mon Sep 17 00:00:00 2001 From: wenweihuang Date: Tue, 2 Apr 2024 12:22:24 +0800 Subject: [PATCH 03/13] [INLONG-9909][Agent] Add unit test for installer --- .../src/test/java/installer/TestModuleManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java index 078758cc0ae..b42896c12eb 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -158,7 +158,7 @@ public void testModuleManager() { Assert.assertTrue("start module manager error", false); } await().atMost(3, TimeUnit.SECONDS).until(() -> manager.getModule(1) != null); - Assert.assertTrue(realActionList.size() == expectedActionList.size()); + await().atMost(10, TimeUnit.SECONDS).until(() -> realActionList.size() == expectedActionList.size()); for (Integer i = 0; i < expectedActionList.size(); i++) { LOGGER.info("{} {}", realActionList.get(i), expectedActionList.get(i)); Assert.assertTrue(i.toString(), realActionList.get(i).equals(expectedActionList.get(i))); From 551ca4f530a9a50a28de02d772374bb363cc7679 Mon Sep 17 00:00:00 2001 From: wenweihuang Date: Tue, 2 Apr 2024 12:28:55 +0800 Subject: [PATCH 04/13] [INLONG-9909][Agent] Add unit test for installer --- .../src/test/java/installer/TestModuleManager.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java index b42896c12eb..eff14c486e0 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -135,6 +135,11 @@ private void mockFunctions() { return true; }).when(manager, "stopModule", Mockito.any()); + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + return true; + }).when(manager, "isProcessAllStarted", Mockito.any()); + PowerMockito.doReturn(null).when(manager, "getHttpManager", Mockito.any()); } catch (Exception e) { LOGGER.error("mock downloadModule error", e); From a53d8a5335e2276bd206d83bd5aee2c91173b3b4 Mon Sep 17 00:00:00 2001 From: justinwwhuang Date: Tue, 2 Apr 2024 14:18:11 +0800 Subject: [PATCH 05/13] Update inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java Co-authored-by: AloysZhang --- .../java/org/apache/inlong/agent/installer/ModuleManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java index 4e619a46d47..4eeb223dad2 100755 --- a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java +++ b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java @@ -318,7 +318,7 @@ private void deleteModule(ModuleConfig module) { private void updateModule(ModuleConfig localModule, ModuleConfig managerModule) { LOGGER.info("update module {} start", localModule.getId()); if (localModule.getPackageConfig().getMd5().equals(managerModule.getPackageConfig().getMd5())) { - LOGGER.info("package md5 no change, will restart", localModule.getId()); + LOGGER.info("package md5 no change, will restart {}", localModule.getId()); restartModule(localModule, managerModule); } else { LOGGER.info("package md5 changed, will reinstall", localModule.getId()); From 4dbe9ea60fb327549f0014910926668db621d6dc Mon Sep 17 00:00:00 2001 From: justinwwhuang Date: Tue, 2 Apr 2024 14:22:00 +0800 Subject: [PATCH 06/13] Update inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java Co-authored-by: AloysZhang --- .../src/test/java/installer/BaseTestsHelper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java b/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java index 88430a9af50..175393434e5 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java +++ b/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java @@ -77,8 +77,7 @@ public void teardownAgentHome() { public TaskProfile getTaskProfile(int taskId, String pattern, boolean retry, Long startTime, Long endTime, TaskStateEnum state, String timeZone) { DataConfig dataConfig = getDataConfig(taskId, pattern, retry, startTime, endTime, state, timeZone); - TaskProfile profile = TaskProfile.convertToTaskProfile(dataConfig); - return profile; + return TaskProfile.convertToTaskProfile(dataConfig); } private DataConfig getDataConfig(int taskId, String pattern, boolean retry, Long startTime, Long endTime, From bf533409cabd97e840243d84f492edc1a41dc417 Mon Sep 17 00:00:00 2001 From: justinwwhuang Date: Tue, 2 Apr 2024 14:22:16 +0800 Subject: [PATCH 07/13] Update inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java Co-authored-by: AloysZhang --- .../src/test/java/installer/BaseTestsHelper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java b/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java index 175393434e5..445318d98bc 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java +++ b/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java @@ -40,8 +40,7 @@ public class BaseTestsHelper { private static final Logger LOGGER = LoggerFactory.getLogger(BaseTestsHelper.class); - private static final GsonBuilder gsonBuilder = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss"); - private static final Gson GSON = gsonBuilder.create(); + private static final Gson GSON = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss").create(); private final String className; private Path testRootDir; From df6594b61770af6b4e8d7054b7e74b98b432900f Mon Sep 17 00:00:00 2001 From: justinwwhuang Date: Tue, 2 Apr 2024 14:28:33 +0800 Subject: [PATCH 08/13] Update inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java Co-authored-by: AloysZhang --- .../src/test/java/installer/TestModuleManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java index eff14c486e0..08a290e9b7b 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -154,7 +154,7 @@ public void testModuleManager() { mockFunctions(); String confPath = LOADER.getResource("conf/").getPath(); manager.restoreFromLocalFile(confPath); - Assert.assertTrue(manager.getModule(1).getPackageConfig().getMd5().equals(OLD_MD5)); +Assert.assertEquals(manager.getModule(1).getPackageConfig().getMd5(), OLD_MD5); manager.submitConfig(getConfig()); try { manager.start(); From caf30455c75f5f219f977fad5b17d30ba6373d7e Mon Sep 17 00:00:00 2001 From: justinwwhuang Date: Tue, 2 Apr 2024 14:43:42 +0800 Subject: [PATCH 09/13] Update inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java Co-authored-by: AloysZhang --- .../src/test/java/installer/TestModuleManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java index 08a290e9b7b..00ad9caef36 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -143,8 +143,7 @@ private void mockFunctions() { PowerMockito.doReturn(null).when(manager, "getHttpManager", Mockito.any()); } catch (Exception e) { LOGGER.error("mock downloadModule error", e); - Assert.assertTrue(false); - return; + Assert.fail(); } } From 8fbb9c972144214f825ca5089cce14faa7243e62 Mon Sep 17 00:00:00 2001 From: justinwwhuang Date: Tue, 2 Apr 2024 14:44:01 +0800 Subject: [PATCH 10/13] Update inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java Co-authored-by: AloysZhang --- .../src/test/java/installer/TestModuleManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java index 00ad9caef36..7e71010d70b 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -159,7 +159,7 @@ public void testModuleManager() { manager.start(); } catch (Exception e) { LOGGER.error("start module manager error", e); - Assert.assertTrue("start module manager error", false); + Assert.fail("start module manager error"); } await().atMost(3, TimeUnit.SECONDS).until(() -> manager.getModule(1) != null); await().atMost(10, TimeUnit.SECONDS).until(() -> realActionList.size() == expectedActionList.size()); From 3deb499fbe17cdfc5a7685d08f188c74baf779e1 Mon Sep 17 00:00:00 2001 From: justinwwhuang Date: Tue, 2 Apr 2024 14:44:14 +0800 Subject: [PATCH 11/13] Update inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java Co-authored-by: AloysZhang --- .../src/test/java/installer/TestModuleManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java index 7e71010d70b..6eacf0e2768 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -163,9 +163,9 @@ public void testModuleManager() { } await().atMost(3, TimeUnit.SECONDS).until(() -> manager.getModule(1) != null); await().atMost(10, TimeUnit.SECONDS).until(() -> realActionList.size() == expectedActionList.size()); - for (Integer i = 0; i < expectedActionList.size(); i++) { + for (int i = 0; i < expectedActionList.size(); i++) { LOGGER.info("{} {}", realActionList.get(i), expectedActionList.get(i)); - Assert.assertTrue(i.toString(), realActionList.get(i).equals(expectedActionList.get(i))); + Assert.assertEquals(Integer.toString(i), realActionList.get(i), expectedActionList.get(i)); } } From f5e461f4d7f933502422ae263c401f3108bc3bf4 Mon Sep 17 00:00:00 2001 From: wenweihuang Date: Tue, 2 Apr 2024 15:05:07 +0800 Subject: [PATCH 12/13] [INLONG-9909][Agent] Modify based on comments --- .../inlong/agent/installer/ModuleManager.java | 4 +- .../test/java/installer/BaseTestsHelper.java | 38 +------------------ .../java/installer/TestModuleManager.java | 2 +- 3 files changed, 4 insertions(+), 40 deletions(-) diff --git a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java index 4eeb223dad2..031057bb487 100755 --- a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java +++ b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java @@ -318,10 +318,10 @@ private void deleteModule(ModuleConfig module) { private void updateModule(ModuleConfig localModule, ModuleConfig managerModule) { LOGGER.info("update module {} start", localModule.getId()); if (localModule.getPackageConfig().getMd5().equals(managerModule.getPackageConfig().getMd5())) { - LOGGER.info("package md5 no change, will restart {}", localModule.getId()); + LOGGER.info("module {} package md5 no change, will restart", localModule.getId()); restartModule(localModule, managerModule); } else { - LOGGER.info("package md5 changed, will reinstall", localModule.getId()); + LOGGER.info("module {} package md5 changed, will reinstall", localModule.getId()); deleteModule(localModule); addModule(managerModule); } diff --git a/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java b/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java index 445318d98bc..1b9d02c8c9d 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java +++ b/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java @@ -17,13 +17,9 @@ package installer; -import org.apache.inlong.agent.conf.TaskProfile; import org.apache.inlong.agent.constant.AgentConstants; import org.apache.inlong.agent.constant.FetcherConstants; import org.apache.inlong.agent.installer.conf.InstallerConfiguration; -import org.apache.inlong.agent.pojo.FileTask.FileTaskConfig; -import org.apache.inlong.common.enums.TaskStateEnum; -import org.apache.inlong.common.pojo.agent.DataConfig; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -40,7 +36,7 @@ public class BaseTestsHelper { private static final Logger LOGGER = LoggerFactory.getLogger(BaseTestsHelper.class); - private static final Gson GSON = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss").create(); + private static final Gson GSON = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss").create(); private final String className; private Path testRootDir; @@ -59,10 +55,6 @@ public BaseTestsHelper setupAgentHome() { return this; } - public Path getTestRootDir() { - return testRootDir; - } - public void teardownAgentHome() { if (testRootDir != null) { try { @@ -72,32 +64,4 @@ public void teardownAgentHome() { } } } - - public TaskProfile getTaskProfile(int taskId, String pattern, boolean retry, Long startTime, Long endTime, - TaskStateEnum state, String timeZone) { - DataConfig dataConfig = getDataConfig(taskId, pattern, retry, startTime, endTime, state, timeZone); - return TaskProfile.convertToTaskProfile(dataConfig); - } - - private DataConfig getDataConfig(int taskId, String pattern, boolean retry, Long startTime, Long endTime, - TaskStateEnum state, String timeZone) { - DataConfig dataConfig = new DataConfig(); - dataConfig.setInlongGroupId("testGroupId"); - dataConfig.setInlongStreamId("testStreamId"); - dataConfig.setDataReportType(1); - dataConfig.setTaskType(3); - dataConfig.setTaskId(taskId); - dataConfig.setTimeZone(timeZone); - dataConfig.setState(state.ordinal()); - FileTaskConfig fileTaskConfig = new FileTaskConfig(); - fileTaskConfig.setPattern(pattern); - fileTaskConfig.setTimeOffset("0h"); - fileTaskConfig.setMaxFileCount(100); - fileTaskConfig.setCycleUnit("h"); - fileTaskConfig.setRetry(retry); - fileTaskConfig.setStartTime(startTime); - fileTaskConfig.setEndTime(endTime); - dataConfig.setExtParams(GSON.toJson(fileTaskConfig)); - return dataConfig; - } } diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java index 6eacf0e2768..719727852b0 100755 --- a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -153,7 +153,7 @@ public void testModuleManager() { mockFunctions(); String confPath = LOADER.getResource("conf/").getPath(); manager.restoreFromLocalFile(confPath); -Assert.assertEquals(manager.getModule(1).getPackageConfig().getMd5(), OLD_MD5); + Assert.assertEquals(manager.getModule(1).getPackageConfig().getMd5(), OLD_MD5); manager.submitConfig(getConfig()); try { manager.start(); From d5401fd1f359bc0e50c9757680eff2af3b92bf11 Mon Sep 17 00:00:00 2001 From: wenweihuang Date: Tue, 2 Apr 2024 16:22:34 +0800 Subject: [PATCH 13/13] [INLONG-9909][Agent] Modify based on comments --- inlong-agent/agent-installer/src/test/resources/log4j2.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/inlong-agent/agent-installer/src/test/resources/log4j2.xml b/inlong-agent/agent-installer/src/test/resources/log4j2.xml index a80b45ae24a..8b051e86200 100644 --- a/inlong-agent/agent-installer/src/test/resources/log4j2.xml +++ b/inlong-agent/agent-installer/src/test/resources/log4j2.xml @@ -40,7 +40,6 @@ - \ No newline at end of file