From c6ea6a632d833138c6853d90f00f2912b68a7b33 Mon Sep 17 00:00:00 2001 From: dolcino Date: Sun, 5 Mar 2023 20:12:35 +0800 Subject: [PATCH 1/3] Change tmpdir location in case /grid folder doesn't exist --- dynoyarn-driver/src/main/resources/start-component.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dynoyarn-driver/src/main/resources/start-component.sh b/dynoyarn-driver/src/main/resources/start-component.sh index e316de5..f39e08e 100755 --- a/dynoyarn-driver/src/main/resources/start-component.sh +++ b/dynoyarn-driver/src/main/resources/start-component.sh @@ -63,7 +63,8 @@ cp *.jar $extraClasspathDir # This is where libleveldbjni is written (per-NM). Write it to a larger partition # instead of /tmp -tmpdir="/grid/a/tmp/hadoop-`whoami`" +#tmpdir="/grid/a/tmp/hadoop-`whoami`" +tmpdir="$baseDir/tmp/hadoop-`whoami`" mkdir $tmpdir # Change environment variables for the Hadoop process From f33fc6e9eafb443f4a6e78beed587e36d8fad3c1 Mon Sep 17 00:00:00 2001 From: dolcino Date: Sun, 5 Mar 2023 20:23:48 +0800 Subject: [PATCH 2/3] unset application web proxy base to avoid the /poxy/ for RM --- dynoyarn-driver/src/main/resources/start-component.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dynoyarn-driver/src/main/resources/start-component.sh b/dynoyarn-driver/src/main/resources/start-component.sh index f39e08e..ff06c9c 100755 --- a/dynoyarn-driver/src/main/resources/start-component.sh +++ b/dynoyarn-driver/src/main/resources/start-component.sh @@ -123,6 +123,7 @@ if [ "$component" = "RESOURCE_MANAGER" ]; then cp `readlink -f dynoyarn-capacity-scheduler.xml` ${baseDir}/dcs/capacity-scheduler.xml rm ${confDir}/capacity-scheduler.xml ln -s ${baseDir}/dcs/capacity-scheduler.xml ${confDir}/capacity-scheduler.xml + unset APPLICATION_WEB_PROXY_BASE else rm ${hadoopHome}/etc/hadoop/container-executor.cfg ln -s `readlink -f dynoyarn-container-executor.cfg` ${hadoopHome}/etc/hadoop/container-executor.cfg From bb1ab47eef718657e46cc24049948441c6273aef Mon Sep 17 00:00:00 2001 From: dolcino Date: Sun, 5 Mar 2023 23:14:47 +0800 Subject: [PATCH 3/3] Make driver app place simulatedFatJar for generator app. Original plan is that generator app need to upload simulatedFatJat to HDFS, and then download the simulatdFatJar from HDFS when it runs simulated container. However, dyno yarn cluster is unsecurity, the cluster use simple auth. Therefore, the simulated container which use simple auth can never download the file from HDFS which use kerberos auth. When driver app runs, it could download all necessary files, and put the file in a centain path. Meanwhile, generator app add that path into CLASSPATH. Then we can avoid above situation. --- .../dynoyarn/DriverApplicationMaster.java | 3 +++ .../com/linkedin/dynoyarn/DriverClient.java | 19 +++++++++++++++++++ .../workload/WorkloadApplicationMaster.java | 1 + .../dynoyarn/workload/WorkloadClient.java | 6 +++++- .../src/main/resources/start-component.sh | 8 ++++++++ .../simulation/SimulatedAppSubmitter.java | 17 +++++++++++++---- .../SimulatedApplicationMaster.java | 6 ++++-- 7 files changed, 53 insertions(+), 7 deletions(-) diff --git a/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/DriverApplicationMaster.java b/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/DriverApplicationMaster.java index 439e555..ec52482 100644 --- a/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/DriverApplicationMaster.java +++ b/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/DriverApplicationMaster.java @@ -605,6 +605,9 @@ private Map getContainerResources() throws IOException { Utils.localizeHDFSResource(fs, startScript, Constants.DYARN_START_SCRIPT, LocalResourceType.FILE, containerResources); Utils.localizeHDFSResource(fs, containerExecutorCfg, Constants.CONTAINER_EXECUTOR_CFG, LocalResourceType.FILE, containerResources); + //Try to get Dyno_Generate jar + Utils.localizeHDFSResource(fs, System.getenv(Constants.SIMULATED_FATJAR_NAME), Constants.SIMULATED_FATJAR, LocalResourceType.FILE, containerResources); + String hdfsClasspath = System.getenv("HDFS_CLASSPATH"); for (FileStatus status : fs.listStatus(new Path(hdfsClasspath))) { Utils.localizeHDFSResource(fs, status.getPath().toString(), status.getPath().getName(), LocalResourceType.FILE, containerResources); diff --git a/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/DriverClient.java b/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/DriverClient.java index 7ca0049..96b0b64 100644 --- a/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/DriverClient.java +++ b/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/DriverClient.java @@ -9,6 +9,8 @@ import com.linkedin.dynoyarn.common.DynoYARNConfigurationKeys; import com.linkedin.dynoyarn.common.Utils; import java.io.File; +import java.io.FileNotFoundException; +import java.io.FilenameFilter; import java.io.IOException; import java.net.URISyntaxException; import java.nio.ByteBuffer; @@ -212,6 +214,23 @@ private ContainerLaunchContext createAMContainerSpec(ApplicationId appId, ByteBu Path containerExecutorCfg = Utils.localizeLocalResource(dyarnConf, fs, CONTAINER_EXECUTOR_CFG, LocalResourceType.FILE, appResourcesPath, localResources); Utils.localizeLocalResource(dyarnConf, fs, DYNOYARN_SITE_XML, LocalResourceType.FILE, appResourcesPath, localResources); containerEnv.put(Constants.DYARN_CONF_NAME, conf.toString()); + + //Try to add dynoyarn-generator resource when launch NMs + File cwd = new File("."); + File[] files = cwd.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.startsWith("dynoyarn-generator"); + } + }); + if (files.length == 0) { + throw new FileNotFoundException("Not found dynooyarn-generator-jar"); + } + Path simulateFatJarLocation = Utils.localizeLocalResource(dyarnConf, fs, + files[0].getPath(), LocalResourceType.FILE, appResourcesPath, localResources); + + containerEnv.put(Constants.SIMULATED_FATJAR_NAME, simulateFatJarLocation.toString()); + containerEnv.put(Constants.DYARN_JAR_NAME, dyarnJar.toString()); containerEnv.put(Constants.DYARN_START_SCRIPT_NAME, startScript.toString()); containerEnv.put(Constants.CAPACITY_SCHEDULER_NAME, capacitySchedulerConfPath); diff --git a/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/workload/WorkloadApplicationMaster.java b/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/workload/WorkloadApplicationMaster.java index a454cde..b22cdd9 100644 --- a/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/workload/WorkloadApplicationMaster.java +++ b/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/workload/WorkloadApplicationMaster.java @@ -469,6 +469,7 @@ public void run() { containerShellEnv.put(Constants.SIMULATED_FATJAR_NAME, System.getenv(Constants.SIMULATED_FATJAR_NAME)); containerShellEnv.put("HDFS_CLASSPATH", System.getenv("HDFS_CLASSPATH")); containerShellEnv.put(Constants.DYARN_CONF_NAME, System.getenv(Constants.DYARN_CONF_NAME)); + containerShellEnv.put("driverAppId", System.getenv("driverAppId")); try (PrintWriter out = new PrintWriter(Constants.SPEC_FILE, "UTF-8")) { String spec = new ObjectMapper().writeValueAsString(containerAppSpecs) .replaceAll("\"", "'"); diff --git a/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/workload/WorkloadClient.java b/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/workload/WorkloadClient.java index ecf1669..91669a4 100644 --- a/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/workload/WorkloadClient.java +++ b/dynoyarn-driver/src/main/java/com/linkedin/dynoyarn/workload/WorkloadClient.java @@ -66,6 +66,8 @@ public class WorkloadClient implements AutoCloseable { private String confPath; private String workloadJarPath; + private String driverAppId; + private Path appResourcesPath; public static final String DRIVER_APP_ID_ARG = "driver_app_id"; @@ -159,7 +161,8 @@ public boolean init(String[] args) throws IOException, ParseException { if (args.length == 0) { throw new IllegalArgumentException("No args specified for client to initialize"); } - String driverAppId = cliParser.getOptionValue(DRIVER_APP_ID_ARG); + //String driverAppId = cliParser.getOptionValue(DRIVER_APP_ID_ARG); + driverAppId = cliParser.getOptionValue(DRIVER_APP_ID_ARG); workloadSpecLocation = cliParser.getOptionValue(WORKLOAD_SPEC_LOCATION_ARG); simulatedFatJarLocation = cliParser.getOptionValue(SIMULATED_FATJAR_ARG); confPath = cliParser.getOptionValue(CONF_ARG); @@ -243,6 +246,7 @@ public boolean accept(File dir, String name) { classPathEnv.append(c.trim()); } containerEnv.put("CLASSPATH", classPathEnv.toString()); + containerEnv.put("driverAppId", driverAppId); // Set logs to be readable by everyone. Set app to be modifiable only by app owner. Map acls = new HashMap<>(2); diff --git a/dynoyarn-driver/src/main/resources/start-component.sh b/dynoyarn-driver/src/main/resources/start-component.sh index ff06c9c..9b33833 100755 --- a/dynoyarn-driver/src/main/resources/start-component.sh +++ b/dynoyarn-driver/src/main/resources/start-component.sh @@ -145,6 +145,14 @@ read -r -d '' driverConfigs < containerEnv = new HashMap<>(); Map localResources = new HashMap<>(); - Utils.localizeHDFSResource(fs, System.getenv(Constants.DYARN_JAR_NAME), Constants.DYARN_JAR, LocalResourceType.FILE, localResources); - Utils.localizeHDFSResource(fs, System.getenv(Constants.SIMULATED_FATJAR_NAME), Constants.SIMULATED_FATJAR, LocalResourceType.FILE, localResources); - Utils.localizeHDFSResource(fs, System.getenv(Constants.DYARN_CONF_NAME), Constants.DYARN_CONF_NAME, LocalResourceType.FILE, localResources); + //Utils.localizeHDFSResource(fs, System.getenv(Constants.DYARN_JAR_NAME), Constants.DYARN_JAR, LocalResourceType.FILE, localResources); + //Utils.localizeHDFSResource(fs, System.getenv(Constants.SIMULATED_FATJAR_NAME), Constants.SIMULATED_FATJAR, LocalResourceType.FILE, localResources); + //Utils.localizeHDFSResource(fs, System.getenv(Constants.DYARN_CONF_NAME), Constants.DYARN_CONF_NAME, LocalResourceType.FILE, localResources); + + //In my case, I never use HDFS_CLASSPATH, therefore, I don't deal with this localizeHDFSResource String hdfsClasspath = System.getenv("HDFS_CLASSPATH"); if (hdfsClasspath != null) { for (FileStatus status : fs.listStatus(new Path(hdfsClasspath))) { @@ -278,8 +282,13 @@ private ContainerLaunchContext createAMContainerSpec(AppSpec appSpec, ByteBuffer } } + //We need to pass those parameters, since we don't upload the Dyarn_Conf + containerEnv.put("schedulerEndPoint", schedulerEndPoint); + containerEnv.put("rmEndPoint", rmEndpoint); containerEnv.put(Constants.DYARN_CONF_NAME, new Path(System.getenv(Constants.DYARN_CONF_NAME)).getName()); - containerEnv.put("CLASSPATH", "./*:$HADOOP_CONF_DIR:$HADOOP_YARN_HOME/share/hadoop/yarn/*:$HADOOP_YARN_HOME/lib/*"); + + //Add simulatedfatjar.jar in the classpath. + containerEnv.put("CLASSPATH", "./*:$HADOOP_CONF_DIR:$HADOOP_YARN_HOME/share/hadoop/yarn/*:$HADOOP_YARN_HOME/lib/*:/tmp/simulatedfatjar.jar"); containerEnv.put(Constants.IS_AM, "true"); containerEnv.put("HDFS_CLASSPATH", System.getenv("HDFS_CLASSPATH")); containerEnv.put(Constants.APP_SPEC_NAME, new ObjectMapper().writeValueAsString(appSpec) diff --git a/dynoyarn-generator/src/main/java/com/linkedin/dynoyarn/workload/simulation/SimulatedApplicationMaster.java b/dynoyarn-generator/src/main/java/com/linkedin/dynoyarn/workload/simulation/SimulatedApplicationMaster.java index 10b7a9e..05cdd1f 100644 --- a/dynoyarn-generator/src/main/java/com/linkedin/dynoyarn/workload/simulation/SimulatedApplicationMaster.java +++ b/dynoyarn-generator/src/main/java/com/linkedin/dynoyarn/workload/simulation/SimulatedApplicationMaster.java @@ -120,8 +120,10 @@ public boolean init(String[] args) throws IOException { InputStream inputStream = fs.open(new Path(clusterSpecLocation)); String out = IOUtils.toString(inputStream); ClusterInfo cluster = new ObjectMapper().readValue(out, ClusterInfo.class); - rmEndpoint = cluster.getRmHost() + ":" + cluster.getRmPort(); - amEndpoint = cluster.getRmHost() + ":" + cluster.getRmSchedulerPort(); + //rmEndpoint = cluster.getRmHost() + ":" + cluster.getRmPort(); + //amEndpoint = cluster.getRmHost() + ":" + cluster.getRmSchedulerPort(); + rmEndpoint = System.getenv("rmEndPoint"); + amEndpoint = System.getenv("schedulerEndPoint"); // should be schedulerEndPoint, amEndpoint. conf.set(YarnConfiguration.RM_ADDRESS, rmEndpoint); conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, amEndpoint); // Set NM/RM max-wait to respective intervals, so that the underlying ipc.Client only retries once.