diff --git a/inlong-agent/agent-installer/conf/installer.properties b/inlong-agent/agent-installer/conf/installer.properties index 52b4ce071f9..85b2de5bfc5 100755 --- a/inlong-agent/agent-installer/conf/installer.properties +++ b/inlong-agent/agent-installer/conf/installer.properties @@ -18,21 +18,21 @@ ####################### # common config ####################### -local.ip=127.0.0.1 +agent.local.ip=127.0.0.1 agent.enable.oom.exit=false ############################ # manager config ############################ -manager.addr=http://127.0.0.1:8083 -manager.auth.secretId= -manager.auth.secretKey= +agent.manager.addr=http://127.0.0.1:8083 +agent.manager.auth.secretId= +agent.manager.auth.secretKey= ############################ # cluster config for automatically report and register ############################ -cluster.tag=default_cluster -cluster.name=default_agent +agent.cluster.tag=default_cluster +agent.cluster.name=default_agent ############################ # audit config diff --git a/inlong-agent/agent-installer/pom.xml b/inlong-agent/agent-installer/pom.xml index 4ce97f8344f..939df6ebe80 100644 --- a/inlong-agent/agent-installer/pom.xml +++ b/inlong-agent/agent-installer/pom.xml @@ -58,6 +58,24 @@ + + junit + junit + test + + + org.awaitility + awaitility + test + + + org.powermock + powermock-api-mockito2 + + + org.powermock + powermock-module-junit4 + diff --git a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java index ab17b6a5a93..63fbffa76eb 100644 --- a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java +++ b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java @@ -33,13 +33,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.inlong.agent.constant.AgentConstants.AGENT_CLUSTER_NAME; +import static org.apache.inlong.agent.constant.AgentConstants.AGENT_CLUSTER_TAG; +import static org.apache.inlong.agent.constant.AgentConstants.AGENT_LOCAL_IP; import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_FETCHER_INTERVAL; import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_RETURN_PARAM_DATA; import static org.apache.inlong.agent.constant.FetcherConstants.DEFAULT_AGENT_FETCHER_INTERVAL; import static org.apache.inlong.agent.constant.FetcherConstants.DEFAULT_INSTALLER_MANAGER_CONFIG_HTTP_PATH; import static org.apache.inlong.agent.constant.FetcherConstants.INSTALLER_MANAGER_CONFIG_HTTP_PATH; import static org.apache.inlong.agent.installer.ManagerResultFormatter.getResultData; -import static org.apache.inlong.agent.utils.AgentUtils.fetchLocalIp; import static org.apache.inlong.agent.utils.AgentUtils.fetchLocalUuid; /** @@ -47,8 +49,6 @@ */ public class ManagerFetcher extends AbstractDaemon implements ProfileFetcher { - public static final String CLUSTER_NAME = "cluster.name"; - public static final String CLUSTER_TAG = "cluster.tag"; private static final Logger LOGGER = LoggerFactory.getLogger(ManagerFetcher.class); private static final GsonBuilder gsonBuilder = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss"); private static final Gson GSON = gsonBuilder.create(); @@ -68,8 +68,8 @@ public ManagerFetcher(Manager manager) { httpManager = manager.getModuleManager().getHttpManager(conf); baseManagerUrl = httpManager.getBaseUrl(); staticConfigUrl = buildStaticConfigUrl(baseManagerUrl); - clusterTag = conf.get(CLUSTER_TAG); - clusterName = conf.get(CLUSTER_NAME); + clusterTag = conf.get(AGENT_CLUSTER_TAG); + clusterName = conf.get(AGENT_CLUSTER_NAME); } /** @@ -140,7 +140,7 @@ private Runnable configFetchThread() { @Override public void start() throws Exception { // when agent start, check local ip and fetch manager ip list; - localIp = fetchLocalIp(); + localIp = conf.get(AGENT_LOCAL_IP); uuid = fetchLocalUuid(); submitWorker(configFetchThread()); } diff --git a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java index 590644d6c3a..031057bb487 100755 --- a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java +++ b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java @@ -59,6 +59,9 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.stream.Collectors; +import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_ADDR; +import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_AUTH_SECRET_ID; +import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_AUTH_SECRET_KEY; import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_REQUEST_TIMEOUT; import static org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_VIP_HTTP_PREFIX_PATH; import static org.apache.inlong.agent.constant.FetcherConstants.DEFAULT_AGENT_MANAGER_REQUEST_TIMEOUT; @@ -70,9 +73,6 @@ */ public class ModuleManager extends AbstractDaemon { - public static final String MANAGER_ADDR = "manager.addr"; - public static final String MANAGER_AUTH_SECRET_ID = "manager.auth.secretId"; - public static final String MANAGER_AUTH_SECRET_KEY = "manager.auth.secretKey"; public static final int CONFIG_QUEUE_CAPACITY = 1; public static final int CORE_THREAD_SLEEP_TIME = 10000; public static final int DOWNLOAD_PACKAGE_READ_BUFF_SIZE = 1024 * 1024; @@ -97,18 +97,18 @@ public ModuleManager() { } public HttpManager getHttpManager(InstallerConfiguration conf) { - String managerAddr = conf.get(MANAGER_ADDR); + String managerAddr = conf.get(AGENT_MANAGER_ADDR); String managerHttpPrefixPath = conf.get(AGENT_MANAGER_VIP_HTTP_PREFIX_PATH, DEFAULT_AGENT_MANAGER_VIP_HTTP_PREFIX_PATH); int timeout = conf.getInt(AGENT_MANAGER_REQUEST_TIMEOUT, DEFAULT_AGENT_MANAGER_REQUEST_TIMEOUT); - String secretId = conf.get(MANAGER_AUTH_SECRET_ID); - String secretKey = conf.get(MANAGER_AUTH_SECRET_KEY); + String secretId = conf.get(AGENT_MANAGER_AUTH_SECRET_ID); + String secretKey = conf.get(AGENT_MANAGER_AUTH_SECRET_KEY); return new HttpManager(managerAddr, managerHttpPrefixPath, timeout, secretId, secretKey); } private boolean requiredKeys(InstallerConfiguration conf) { - return conf.hasKey(MANAGER_ADDR); + return conf.hasKey(AGENT_MANAGER_ADDR); } public void submitConfig(ConfigResult config) { @@ -318,12 +318,12 @@ private void deleteModule(ModuleConfig module) { private void updateModule(ModuleConfig localModule, ModuleConfig managerModule) { LOGGER.info("update module {} start", localModule.getId()); if (localModule.getPackageConfig().getMd5().equals(managerModule.getPackageConfig().getMd5())) { - LOGGER.info("package md5 changed, will reinstall", localModule.getId()); + LOGGER.info("module {} package md5 no change, will restart", localModule.getId()); + restartModule(localModule, managerModule); + } else { + LOGGER.info("module {} package md5 changed, will reinstall", localModule.getId()); deleteModule(localModule); addModule(managerModule); - } else { - LOGGER.info("package md5 no chang, will restart", localModule.getId()); - restartModule(localModule, managerModule); } LOGGER.info("update module {} end", localModule.getId()); } diff --git a/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java b/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java new file mode 100755 index 00000000000..1b9d02c8c9d --- /dev/null +++ b/inlong-agent/agent-installer/src/test/java/installer/BaseTestsHelper.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package installer; + +import org.apache.inlong.agent.constant.AgentConstants; +import org.apache.inlong.agent.constant.FetcherConstants; +import org.apache.inlong.agent.installer.conf.InstallerConfiguration; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * common environment setting up for test cases. + */ +public class BaseTestsHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseTestsHelper.class); + private static final Gson GSON = new GsonBuilder().setDateFormat("yyyy-MM-dd HH:mm:ss").create(); + private final String className; + private Path testRootDir; + + public BaseTestsHelper(String className) { + this.className = className; + } + + public BaseTestsHelper setupAgentHome() { + testRootDir = Paths + .get("/tmp", BaseTestsHelper.class.getSimpleName(), className); + teardownAgentHome(); + boolean result = testRootDir.toFile().mkdirs(); + LOGGER.info("try to create {}, result is {}", testRootDir, result); + InstallerConfiguration.getInstallerConf().set(AgentConstants.AGENT_HOME, testRootDir.toString()); + InstallerConfiguration.getInstallerConf().set(FetcherConstants.AGENT_MANAGER_ADDR, ""); + return this; + } + + public void teardownAgentHome() { + if (testRootDir != null) { + try { + FileUtils.deleteDirectory(testRootDir.toFile()); + } catch (Exception ignored) { + LOGGER.warn("deleteDirectory error ", ignored); + } + } + } +} diff --git a/inlong-agent/agent-installer/src/test/java/installer/ModuleActionTypeEnum.java b/inlong-agent/agent-installer/src/test/java/installer/ModuleActionTypeEnum.java new file mode 100644 index 00000000000..d54979a6fcb --- /dev/null +++ b/inlong-agent/agent-installer/src/test/java/installer/ModuleActionTypeEnum.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package installer; + +/** + * Enum of module action. + */ +public enum ModuleActionTypeEnum { + + DOWNLOAD(0), + INSTALL(1), + UNINSTALL(2), + START(3), + STOP(4); + + private final int type; + + ModuleActionTypeEnum(int state) { + this.type = state; + } + + public static ModuleActionTypeEnum getTaskState(int state) { + switch (state) { + case 0: + return DOWNLOAD; + case 1: + return INSTALL; + case 2: + return UNINSTALL; + case 3: + return START; + case 4: + return STOP; + default: + throw new RuntimeException("Unsupported module action " + state); + } + } + + public int getType() { + return type; + } +} diff --git a/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java new file mode 100755 index 00000000000..719727852b0 --- /dev/null +++ b/inlong-agent/agent-installer/src/test/java/installer/TestModuleManager.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package installer; + +import org.apache.inlong.agent.installer.ModuleManager; +import org.apache.inlong.common.pojo.agent.installer.ConfigResult; +import org.apache.inlong.common.pojo.agent.installer.ModuleConfig; +import org.apache.inlong.common.pojo.agent.installer.ModuleStateEnum; +import org.apache.inlong.common.pojo.agent.installer.PackageConfig; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(ModuleManager.class) +@PowerMockIgnore({"javax.management.*"}) +public class TestModuleManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestModuleManager.class); + private static BaseTestsHelper helper; + private static final ClassLoader LOADER = TestModuleManager.class.getClassLoader(); + private static ModuleManager manager; + private static List realActionList = new ArrayList<>(); + private static List expectedActionList = new ArrayList<>(); + private static String OLD_MD5 = "95648c83b45971dce503d5d844496cfc"; + private static String NEW_MD5 = "e573f399da60ddeff09904bb95bdc307"; + + @Data + @Builder + @AllArgsConstructor + @NoArgsConstructor + private static class Action { + + public ModuleActionTypeEnum type; + public String md5; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Action action = (Action) o; + return Objects.equals(type, action.type) && + Objects.equals(md5, action.md5); + } + } + + @BeforeClass + public static void setup() { + helper = new BaseTestsHelper(TestModuleManager.class.getName()).setupAgentHome(); + manager = PowerMockito.spy(new ModuleManager()); + } + + @AfterClass + public static void teardown() throws Exception { + manager.stop(); + } + + private void fillExpectedActionList() { + expectedActionList.add(new Action(ModuleActionTypeEnum.STOP, OLD_MD5)); + expectedActionList.add(new Action(ModuleActionTypeEnum.UNINSTALL, OLD_MD5)); + expectedActionList.add(new Action(ModuleActionTypeEnum.DOWNLOAD, NEW_MD5)); + expectedActionList.add(new Action(ModuleActionTypeEnum.INSTALL, NEW_MD5)); + expectedActionList.add(new Action(ModuleActionTypeEnum.START, NEW_MD5)); + } + + private void mockFunctions() { + try { + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + realActionList.add(new Action(ModuleActionTypeEnum.DOWNLOAD, module.getPackageConfig().getMd5())); + return true; + }).when(manager, "downloadModule", Mockito.any()); + + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + realActionList.add(new Action(ModuleActionTypeEnum.INSTALL, module.getPackageConfig().getMd5())); + return true; + }).when(manager, "installModule", Mockito.any()); + + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + realActionList.add(new Action(ModuleActionTypeEnum.UNINSTALL, module.getPackageConfig().getMd5())); + return true; + }).when(manager, "uninstallModule", Mockito.any()); + + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + realActionList.add(new Action(ModuleActionTypeEnum.START, module.getPackageConfig().getMd5())); + return true; + }).when(manager, "startModule", Mockito.any()); + + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + realActionList.add(new Action(ModuleActionTypeEnum.STOP, module.getPackageConfig().getMd5())); + return true; + }).when(manager, "stopModule", Mockito.any()); + + PowerMockito.doAnswer(invocation -> { + ModuleConfig module = invocation.getArgument(0); + return true; + }).when(manager, "isProcessAllStarted", Mockito.any()); + + PowerMockito.doReturn(null).when(manager, "getHttpManager", Mockito.any()); + } catch (Exception e) { + LOGGER.error("mock downloadModule error", e); + Assert.fail(); + } + } + + @Test + public void testModuleManager() { + fillExpectedActionList(); + mockFunctions(); + String confPath = LOADER.getResource("conf/").getPath(); + manager.restoreFromLocalFile(confPath); + Assert.assertEquals(manager.getModule(1).getPackageConfig().getMd5(), OLD_MD5); + manager.submitConfig(getConfig()); + try { + manager.start(); + } catch (Exception e) { + LOGGER.error("start module manager error", e); + Assert.fail("start module manager error"); + } + await().atMost(3, TimeUnit.SECONDS).until(() -> manager.getModule(1) != null); + await().atMost(10, TimeUnit.SECONDS).until(() -> realActionList.size() == expectedActionList.size()); + for (int i = 0; i < expectedActionList.size(); i++) { + LOGGER.info("{} {}", realActionList.get(i), expectedActionList.get(i)); + Assert.assertEquals(Integer.toString(i), realActionList.get(i), expectedActionList.get(i)); + } + } + + private ConfigResult getConfig() { + List configs = new ArrayList<>(); + configs.add(getModuleConfig(1, "inlong-agent", "inlong-agent-md5-185454", "1.0", 1, + "cd ~/inlong-agent/bin;sh agent.sh start", "cd ~/inlong-agent/bin;sh agent.sh stop", + "ps aux | grep core.AgentMain | grep java | grep -v grep | awk '{print $2}'", + "cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf ~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf agent-release-1.12.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd ~/inlong-agent/bin;sh agent.sh start", + "echo empty uninstall cmd", "agent-release-1.12.0-SNAPSHOT-bin.tar.gz", + "http://11.151.252.111:8083/inlong/manager/openapi/agent/download/agent-release-1.12.0-SNAPSHOT-bin.tar.gz", + NEW_MD5)); + return ConfigResult.builder().moduleList(configs).moduleNum(1).md5("config-result-md5-193603").build(); + } + + private ModuleConfig getModuleConfig(int id, String name, String md5, String version, Integer procNum, + String startCmd, String stopCmd, String checkCmd, String installCmd, String uninstallCmd, String fileName, + String downloadUrl, + String packageMd5) { + ModuleConfig moduleConfig = new ModuleConfig(); + moduleConfig.setId(id); + moduleConfig.setName(name); + moduleConfig.setVersion(version); + moduleConfig.setMd5(md5); + moduleConfig.setProcessesNum(procNum); + moduleConfig.setStartCommand(startCmd); + moduleConfig.setStopCommand(stopCmd); + moduleConfig.setCheckCommand(checkCmd); + moduleConfig.setInstallCommand(installCmd); + moduleConfig.setUninstallCommand(uninstallCmd); + PackageConfig packageConfig = new PackageConfig(); + packageConfig.setFileName(fileName); + packageConfig.setDownloadUrl(downloadUrl); + packageConfig.setStoragePath("/tmp"); + packageConfig.setMd5(packageMd5); + moduleConfig.setPackageConfig(packageConfig); + moduleConfig.setState(ModuleStateEnum.NEW); + return moduleConfig; + } +} diff --git a/inlong-agent/agent-installer/src/test/resources/conf/modules.json b/inlong-agent/agent-installer/src/test/resources/conf/modules.json new file mode 100644 index 00000000000..1b365c587c5 --- /dev/null +++ b/inlong-agent/agent-installer/src/test/resources/conf/modules.json @@ -0,0 +1 @@ +{"md5":"b38c63451ff966a32994a867ec79d259","moduleNum":1,"moduleList":[{"id":1,"name":"inlong-agent","md5":"55175a3b2cb143f31ad3d79e081e794c","version":"1.0","processesNum":1,"startCommand":"cd ~/inlong-agent/bin;sh agent.sh start","stopCommand":"cd ~/inlong-agent/bin;sh agent.sh stop","checkCommand":"ps aux | grep core.AgentMain | grep java | grep -v grep | awk \u0027{print $2}\u0027","installCommand":"cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf ~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf apache-inlong-agent-1.12.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd ~/inlong-agent/bin;sh agent.sh start","uninstallCommand":"echo empty uninstall cmd","packageConfig":{"md5":"95648c83b45971dce503d5d844496cfc","fileName":"apache-inlong-agent-1.12.0-SNAPSHOT-bin.tar.gz","downloadUrl":"http://11.151.246.158:8083/inlong/manager/openapi/installer/download?filename\u003dapache-inlong-agent-1.12.0-SNAPSHOT-bin.tar.gz","storagePath":"/tmp"},"state":"INSTALLED"}]} \ No newline at end of file diff --git a/inlong-agent/agent-installer/src/test/resources/log4j2.xml b/inlong-agent/agent-installer/src/test/resources/log4j2.xml new file mode 100644 index 00000000000..8b051e86200 --- /dev/null +++ b/inlong-agent/agent-installer/src/test/resources/log4j2.xml @@ -0,0 +1,45 @@ + + + + + logs/core + %d{yyyy-MM-dd HH:mm:ss.SSS} -%5p ${PID:-} [%15.15t] %-30.30C{1.}:%L %m%n + DEBUG + ${basePath}/ut-debug.log + DEBUG + + + + + + + true + + + + + + + + + + + + \ No newline at end of file