DEF-2971 - Upgraded Emscripten to 1.38.12. (#105)

* DEF-2971 - Upgraded Emscripten to 1.38.12.

- Old (1.35.23) is still installed and will be used by older
  SDKs/build.yml.
- New Emscripten will still build asm.js version of engine.

server/docker-base/Dockerfile

@@ -128,26 +128,52 @@ RUN apt-get autoclean && \
   ln -s /usr/bin/python2.7 /usr/local/bin/python && \
   ln -s /usr/bin/python2.7 /usr/local/bin/python2
 
-ENV EMSCRIPTEN_SDK /opt/emsdk
-ENV EMSCRIPTEN_HOME ${EMSCRIPTEN_SDK}/bin/emsdk_portable
-ENV EMSCRIPTEN_CONFIG ${EMSCRIPTEN_HOME}/.emscripten
-# This should really be a version number in the future, but it was somehow built with "master" as the folder name
-ENV EMSCRIPTEN_VERSION master
-# Where the embedded config file ends up
-ENV EM_CONFIG ${EMSCRIPTEN_CONFIG}
 ENV EMCC_SKIP_SANITY_CHECK 1
-# Add emscripten to the path
-ENV PATH ${PATH}:${EMSCRIPTEN_HOME}:${EMSCRIPTEN_HOME}/clang/fastcomp/build_master_64/bin:${EMSCRIPTEN_HOME}/node/4.1.1_64bit/bin:${EMSCRIPTEN_HOME}/emscripten/${EMSCRIPTEN_VERSION}
 
-# The "sed" command below removes the /TEMP_DIR line from the generated config
-# We replace it with a folder of our own
+## Emscripten 1.38.12
+ENV EMSCRIPTEN_SDK_1_38_12 /opt/emsdk_1.38.12
+ENV EMSCRIPTEN_HOME_1_38_12 ${EMSCRIPTEN_SDK_1_38_12}/bin/emsdk_portable
+ENV EMSCRIPTEN_CONFIG_1_38_12 ${EMSCRIPTEN_HOME_1_38_12}/.emscripten
+ENV EMSCRIPTEN_BIN_1_38_12 ${EMSCRIPTEN_HOME_1_38_12}/emscripten/1.38.12
+
 RUN \
-  mkdir ${EMSCRIPTEN_SDK} && \
-  wget -q -O - ${S3_URL}/emsdk-1.35.23-x86_64-linux.tar.gz | tar xz -C ${EMSCRIPTEN_SDK} && \
-  ${EMSCRIPTEN_HOME}/emsdk activate --embedded && \
-  sed '/TEMP_DIR =/d' ${EMSCRIPTEN_CONFIG} && \
-  echo TEMP_DIR = \'/var/extender/builds\' >> ${EMSCRIPTEN_CONFIG} && \
-  chmod -R 755 ${EMSCRIPTEN_HOME}
+  mkdir ${EMSCRIPTEN_SDK_1_38_12} && \
+  wget -q -O - ${S3_URL}/emsdk-1.38.12-x86_64-linux.tar.gz | tar xz -C ${EMSCRIPTEN_SDK_1_38_12} && \
+  ${EMSCRIPTEN_HOME_1_38_12}/emsdk activate sdk-1.38.12-64bit --embedded && \
+  chmod -R 755 ${EMSCRIPTEN_HOME_1_38_12}
+
+# Setup a special env variable that will be prefixed to PATH if requested version is 1.38.12
+ENV EMSCRIPTEN_PATH_1_38_12 ${EMSCRIPTEN_HOME_1_38_12}:${EMSCRIPTEN_HOME_1_38_12}/clang/e1.38.12_64bit:${EMSCRIPTEN_HOME_1_38_12}/node/8.9.1_64bit/bin:${EMSCRIPTEN_BIN_1_38_12}
+
+## Emscripten 1.35.23 - Needs to be installed and activated last
+ENV EMSCRIPTEN_SDK_1_35_23 /opt/emsdk_1.35.23
+ENV EMSCRIPTEN_HOME_1_35_23 ${EMSCRIPTEN_SDK_1_35_23}/bin/emsdk_portable
+ENV EMSCRIPTEN_CONFIG_1_35_23 ${EMSCRIPTEN_HOME_1_35_23}/.emscripten
+ENV EMSCRIPTEN_BIN_1_35_23 ${EMSCRIPTEN_HOME_1_35_23}/emscripten/master
+
+# Add old 1.35.23 emscripten to the env PATH, it will be used by default since older SDKs doesn't have the emversion key in build.yml.
+# We are also setting up default variables for the config path and emscripten home for 1.35.23.
+# This can be removed once we remove 1.35.23 version support.
+ENV PATH ${PATH}:${EMSCRIPTEN_HOME_1_35_23}:${EMSCRIPTEN_HOME_1_35_23}/clang/fastcomp/build_master_64/bin:${EMSCRIPTEN_HOME_1_35_23}/node/4.1.1_64bit/bin:${EMSCRIPTEN_BIN_1_35_23}
+ENV EM_CONFIG ${EMSCRIPTEN_CONFIG_1_35_23}
+ENV EMSCRIPTEN_HOME ${EMSCRIPTEN_HOME_1_35_23}
+
+RUN \
+  mkdir ${EMSCRIPTEN_SDK_1_35_23} && \
+  wget -q -O - ${S3_URL}/emsdk-1.35.23-x86_64-linux.tar.gz | tar xz -C ${EMSCRIPTEN_SDK_1_35_23} && \
+  ${EMSCRIPTEN_HOME_1_35_23}/emsdk activate --embedded && \
+  chmod -R 755 ${EMSCRIPTEN_HOME_1_35_23}
+
+# We use the same temp directory for both versions.
+ENV EMSCRIPTEN_TEMP_DIR /var/extender/ems_temp
+RUN mkdir -p ${EMSCRIPTEN_TEMP_DIR}
+RUN chmod -R 755 ${EMSCRIPTEN_TEMP_DIR} && chown extender: ${EMSCRIPTEN_TEMP_DIR}
+# The "sed" command below removes the /TEMP_DIR line from the generated configs
+# We replace it with a folder of our own
+RUN sed '/TEMP_DIR =/d' ${EMSCRIPTEN_CONFIG_1_35_23} && \
+  echo TEMP_DIR = \'${EMSCRIPTEN_TEMP_DIR}\' >> ${EMSCRIPTEN_CONFIG_1_35_23} && \
+  sed '/TEMP_DIR =/d' ${EMSCRIPTEN_CONFIG_1_38_12} && \
+  echo TEMP_DIR = \'${EMSCRIPTEN_TEMP_DIR}\' >> ${EMSCRIPTEN_CONFIG_1_38_12}
 
 #
 # Windows

server/src/main/java/com/defold/extender/Extender.java

@@ -129,22 +129,30 @@ class Extender {
         this.appManifestContext = ExtenderUtil.getAppManifestContext(this.appManifest, platform, baseVariantManifest);
         LOGGER.info("Using context for platform: " + alternatePlatform);
 
-        // LEGACY: Make sure the Emscripten compiler doesn't pollute the environment
-        processExecutor.putEnv("EM_CACHE", buildDirectory.toString());
-
         processExecutor.setCwd(jobDirectory);
 
         if (this.platformConfig != null && this.platformConfig.env != null) {
 
             HashMap<String, Object> envContext = new HashMap<>();
             envContext.put("build_folder", buildDirectory);
 
+            // Make system env variables available for the template execution below.
+            for (Map.Entry<String, String> sysEnvEntry : System.getenv().entrySet()) {
+                envContext.put("env." + sysEnvEntry.getKey(), sysEnvEntry.getValue());
+            }
+
             Set<String> keys = this.platformConfig.env.keySet();
             for (String k : keys) {
                 String v = this.platformConfig.env.get(k);
                 v = templateExecutor.execute(v, envContext);
                 processExecutor.putEnv(k, v);
             }
+
+            // Get all "custom" env variables for this process executor and make it available
+            // for commands later on.
+            for (Map.Entry<String, String> envEntry : processExecutor.getEnv().entrySet()) {
+                this.platformConfig.context.put("env." + envEntry.getKey(), envEntry.getValue());
+            }
         }
 
         // The allowed libs/symbols are the union of the values from the different "levels": "context: allowedLibs: [...]" + "context: platforms: arm64-osx: allowedLibs: [...]"

server/src/main/java/com/defold/extender/ProcessExecutor.java

@@ -63,6 +63,10 @@ void putEnv(String key, String value) {
         env.put(key, value);
     }
 
+    Map<String, String> getEnv() {
+        return env;
+    }
+
     void setCwd(File cwd) {
         this.cwd = cwd;
     }

server/src/test/java/com/defold/extender/IntegrationTest.java

@@ -110,10 +110,10 @@ public static Collection<TestConfiguration> data() {
         DefoldVersion[] versions = {
                 // "a" is a made up sdk where we can more easily test build.yml fixes
                 new DefoldVersion("a", new Version(0, 0, 0), new String[] {"x86_64-osx", "armv7-android", "js-web", "x86_64-win32"} ),
-                new DefoldVersion("091e7e02ce492d3c4e493324b9db57d40df69e95", new Version(1, 2, 131), new String[] {"armv7-android", "armv7-ios", "arm64-ios", "x86_64-osx", "x86_64-linux", "x86_64-win32"}),
                 new DefoldVersion("7b2c2c019d6fa106f78e2e98cd3009a21d4095aa", new Version(1, 2, 133), new String[] {"armv7-android", "armv7-ios", "arm64-ios", "x86_64-osx", "x86_64-linux", "x86_64-win32", "js-web"}),
                 new DefoldVersion("b2ef3a19802728e76adf84d51d02e11d636791a3", new Version(1, 2, 134), new String[] {"armv7-android", "armv7-ios", "arm64-ios", "x86_64-osx", "x86_64-linux", "x86_64-win32", "js-web"}),
                 new DefoldVersion("1b90c9a905d634b766b467e3536458b9210ec812", new Version(1, 2, 135), new String[] {"armv7-android", "armv7-ios", "arm64-ios", "x86_64-osx", "x86_64-linux", "x86_64-win32", "js-web"}),
+                new DefoldVersion("b9bbe0c5d5ad6c165147c5461a5d64d9fc62f9f3", new Version(1, 2, 139), new String[] {"armv7-android", "armv7-ios", "arm64-ios", "x86_64-osx", "x86_64-linux", "x86_64-win32", "js-web"}),
 
                 // Use test-data/createdebugsdk.sh to package your preferred platform sdk and it ends up in the sdk/debugsdk folder
                 // Then you can write your tests without waiting for the next release