Convert local path to cross-cluster accessible point (#10)

Convert local path to the cross-cluster accessible point so that Hive-on-Tez in tst cluster can access dw2 data
twitter-forks · Sep 5, 2018 · 5461c89 · 5461c89
1 parent 6f55b43
commit 5461c89
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 12 deletions.
diff --git a/...e/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java b/...e/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/StorageDescriptor.java
diff --git a/pom.xml b/pom.xml
@@ -1347,5 +1347,11 @@
         <module>itests</module>
       </modules>
     </profile>
+    <profile>
+      <id>twitter-tez-protobuf-version</id>
+      <properties>
+        <protobuf.version>2.5.0.t02</protobuf.version>
+      </properties>
+    </profile>
   </profiles>
 </project>
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -1078,6 +1078,13 @@ public JobConf createConfiguration(HiveConf hiveConf) throws IOException {
 
     JobConf conf = new JobConf(new TezConfiguration(hiveConf));
 
+    // Temporary fix for hadoop bug where hadoop_home is duplicated and assigned to follow params
+    conf.set("mapreduce.map.env", "");
+    conf.set("mapreduce.reduce.env", "");
+    conf.set("tez.am.launch.env", "");
+    conf.set("tez.task.launch.env", "");
+    conf.set("yarn.app.mapreduce.am.env", "");
+
     conf.set("mapred.output.committer.class", NullOutputCommitter.class.getName());
 
     conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);