From a13b200f6f3e84e7ae69b5dd5c8dd26c2c30076a Mon Sep 17 00:00:00 2001 From: Sven Andersson Date: Wed, 17 Oct 2018 11:18:44 +0200 Subject: [PATCH] DEF-2971 - Upgraded Emscripten to 1.38.12. (#105) * DEF-2971 - Upgraded Emscripten to 1.38.12. - Old (1.35.23) is still installed and will be used by older SDKs/build.yml. - New Emscripten will still build asm.js version of engine. --- server/docker-base/Dockerfile | 60 +++++++++++++------ .../java/com/defold/extender/Extender.java | 14 ++++- .../com/defold/extender/ProcessExecutor.java | 4 ++ .../com/defold/extender/IntegrationTest.java | 2 +- 4 files changed, 59 insertions(+), 21 deletions(-) diff --git a/server/docker-base/Dockerfile b/server/docker-base/Dockerfile index aee69dc1..0044cb15 100644 --- a/server/docker-base/Dockerfile +++ b/server/docker-base/Dockerfile @@ -128,26 +128,52 @@ RUN apt-get autoclean && \ ln -s /usr/bin/python2.7 /usr/local/bin/python && \ ln -s /usr/bin/python2.7 /usr/local/bin/python2 -ENV EMSCRIPTEN_SDK /opt/emsdk -ENV EMSCRIPTEN_HOME ${EMSCRIPTEN_SDK}/bin/emsdk_portable -ENV EMSCRIPTEN_CONFIG ${EMSCRIPTEN_HOME}/.emscripten -# This should really be a version number in the future, but it was somehow built with "master" as the folder name -ENV EMSCRIPTEN_VERSION master -# Where the embedded config file ends up -ENV EM_CONFIG ${EMSCRIPTEN_CONFIG} ENV EMCC_SKIP_SANITY_CHECK 1 -# Add emscripten to the path -ENV PATH ${PATH}:${EMSCRIPTEN_HOME}:${EMSCRIPTEN_HOME}/clang/fastcomp/build_master_64/bin:${EMSCRIPTEN_HOME}/node/4.1.1_64bit/bin:${EMSCRIPTEN_HOME}/emscripten/${EMSCRIPTEN_VERSION} -# The "sed" command below removes the /TEMP_DIR line from the generated config -# We replace it with a folder of our own +## Emscripten 1.38.12 +ENV EMSCRIPTEN_SDK_1_38_12 /opt/emsdk_1.38.12 +ENV EMSCRIPTEN_HOME_1_38_12 ${EMSCRIPTEN_SDK_1_38_12}/bin/emsdk_portable +ENV EMSCRIPTEN_CONFIG_1_38_12 ${EMSCRIPTEN_HOME_1_38_12}/.emscripten +ENV EMSCRIPTEN_BIN_1_38_12 ${EMSCRIPTEN_HOME_1_38_12}/emscripten/1.38.12 + RUN \ - mkdir ${EMSCRIPTEN_SDK} && \ - wget -q -O - ${S3_URL}/emsdk-1.35.23-x86_64-linux.tar.gz | tar xz -C ${EMSCRIPTEN_SDK} && \ - ${EMSCRIPTEN_HOME}/emsdk activate --embedded && \ - sed '/TEMP_DIR =/d' ${EMSCRIPTEN_CONFIG} && \ - echo TEMP_DIR = \'/var/extender/builds\' >> ${EMSCRIPTEN_CONFIG} && \ - chmod -R 755 ${EMSCRIPTEN_HOME} + mkdir ${EMSCRIPTEN_SDK_1_38_12} && \ + wget -q -O - ${S3_URL}/emsdk-1.38.12-x86_64-linux.tar.gz | tar xz -C ${EMSCRIPTEN_SDK_1_38_12} && \ + ${EMSCRIPTEN_HOME_1_38_12}/emsdk activate sdk-1.38.12-64bit --embedded && \ + chmod -R 755 ${EMSCRIPTEN_HOME_1_38_12} + +# Setup a special env variable that will be prefixed to PATH if requested version is 1.38.12 +ENV EMSCRIPTEN_PATH_1_38_12 ${EMSCRIPTEN_HOME_1_38_12}:${EMSCRIPTEN_HOME_1_38_12}/clang/e1.38.12_64bit:${EMSCRIPTEN_HOME_1_38_12}/node/8.9.1_64bit/bin:${EMSCRIPTEN_BIN_1_38_12} + +## Emscripten 1.35.23 - Needs to be installed and activated last +ENV EMSCRIPTEN_SDK_1_35_23 /opt/emsdk_1.35.23 +ENV EMSCRIPTEN_HOME_1_35_23 ${EMSCRIPTEN_SDK_1_35_23}/bin/emsdk_portable +ENV EMSCRIPTEN_CONFIG_1_35_23 ${EMSCRIPTEN_HOME_1_35_23}/.emscripten +ENV EMSCRIPTEN_BIN_1_35_23 ${EMSCRIPTEN_HOME_1_35_23}/emscripten/master + +# Add old 1.35.23 emscripten to the env PATH, it will be used by default since older SDKs doesn't have the emversion key in build.yml. +# We are also setting up default variables for the config path and emscripten home for 1.35.23. +# This can be removed once we remove 1.35.23 version support. +ENV PATH ${PATH}:${EMSCRIPTEN_HOME_1_35_23}:${EMSCRIPTEN_HOME_1_35_23}/clang/fastcomp/build_master_64/bin:${EMSCRIPTEN_HOME_1_35_23}/node/4.1.1_64bit/bin:${EMSCRIPTEN_BIN_1_35_23} +ENV EM_CONFIG ${EMSCRIPTEN_CONFIG_1_35_23} +ENV EMSCRIPTEN_HOME ${EMSCRIPTEN_HOME_1_35_23} + +RUN \ + mkdir ${EMSCRIPTEN_SDK_1_35_23} && \ + wget -q -O - ${S3_URL}/emsdk-1.35.23-x86_64-linux.tar.gz | tar xz -C ${EMSCRIPTEN_SDK_1_35_23} && \ + ${EMSCRIPTEN_HOME_1_35_23}/emsdk activate --embedded && \ + chmod -R 755 ${EMSCRIPTEN_HOME_1_35_23} + +# We use the same temp directory for both versions. +ENV EMSCRIPTEN_TEMP_DIR /var/extender/ems_temp +RUN mkdir -p ${EMSCRIPTEN_TEMP_DIR} +RUN chmod -R 755 ${EMSCRIPTEN_TEMP_DIR} && chown extender: ${EMSCRIPTEN_TEMP_DIR} +# The "sed" command below removes the /TEMP_DIR line from the generated configs +# We replace it with a folder of our own +RUN sed '/TEMP_DIR =/d' ${EMSCRIPTEN_CONFIG_1_35_23} && \ + echo TEMP_DIR = \'${EMSCRIPTEN_TEMP_DIR}\' >> ${EMSCRIPTEN_CONFIG_1_35_23} && \ + sed '/TEMP_DIR =/d' ${EMSCRIPTEN_CONFIG_1_38_12} && \ + echo TEMP_DIR = \'${EMSCRIPTEN_TEMP_DIR}\' >> ${EMSCRIPTEN_CONFIG_1_38_12} # # Windows diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index a758df02..c39a965d 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -129,9 +129,6 @@ class Extender { this.appManifestContext = ExtenderUtil.getAppManifestContext(this.appManifest, platform, baseVariantManifest); LOGGER.info("Using context for platform: " + alternatePlatform); - // LEGACY: Make sure the Emscripten compiler doesn't pollute the environment - processExecutor.putEnv("EM_CACHE", buildDirectory.toString()); - processExecutor.setCwd(jobDirectory); if (this.platformConfig != null && this.platformConfig.env != null) { @@ -139,12 +136,23 @@ class Extender { HashMap envContext = new HashMap<>(); envContext.put("build_folder", buildDirectory); + // Make system env variables available for the template execution below. + for (Map.Entry sysEnvEntry : System.getenv().entrySet()) { + envContext.put("env." + sysEnvEntry.getKey(), sysEnvEntry.getValue()); + } + Set keys = this.platformConfig.env.keySet(); for (String k : keys) { String v = this.platformConfig.env.get(k); v = templateExecutor.execute(v, envContext); processExecutor.putEnv(k, v); } + + // Get all "custom" env variables for this process executor and make it available + // for commands later on. + for (Map.Entry envEntry : processExecutor.getEnv().entrySet()) { + this.platformConfig.context.put("env." + envEntry.getKey(), envEntry.getValue()); + } } // The allowed libs/symbols are the union of the values from the different "levels": "context: allowedLibs: [...]" + "context: platforms: arm64-osx: allowedLibs: [...]" diff --git a/server/src/main/java/com/defold/extender/ProcessExecutor.java b/server/src/main/java/com/defold/extender/ProcessExecutor.java index 6ea9523f..58f942d0 100644 --- a/server/src/main/java/com/defold/extender/ProcessExecutor.java +++ b/server/src/main/java/com/defold/extender/ProcessExecutor.java @@ -63,6 +63,10 @@ void putEnv(String key, String value) { env.put(key, value); } + Map getEnv() { + return env; + } + void setCwd(File cwd) { this.cwd = cwd; } diff --git a/server/src/test/java/com/defold/extender/IntegrationTest.java b/server/src/test/java/com/defold/extender/IntegrationTest.java index 64e79b46..fc807328 100644 --- a/server/src/test/java/com/defold/extender/IntegrationTest.java +++ b/server/src/test/java/com/defold/extender/IntegrationTest.java @@ -110,10 +110,10 @@ public static Collection data() { DefoldVersion[] versions = { // "a" is a made up sdk where we can more easily test build.yml fixes new DefoldVersion("a", new Version(0, 0, 0), new String[] {"x86_64-osx", "armv7-android", "js-web", "x86_64-win32"} ), - new DefoldVersion("091e7e02ce492d3c4e493324b9db57d40df69e95", new Version(1, 2, 131), new String[] {"armv7-android", "armv7-ios", "arm64-ios", "x86_64-osx", "x86_64-linux", "x86_64-win32"}), new DefoldVersion("7b2c2c019d6fa106f78e2e98cd3009a21d4095aa", new Version(1, 2, 133), new String[] {"armv7-android", "armv7-ios", "arm64-ios", "x86_64-osx", "x86_64-linux", "x86_64-win32", "js-web"}), new DefoldVersion("b2ef3a19802728e76adf84d51d02e11d636791a3", new Version(1, 2, 134), new String[] {"armv7-android", "armv7-ios", "arm64-ios", "x86_64-osx", "x86_64-linux", "x86_64-win32", "js-web"}), new DefoldVersion("1b90c9a905d634b766b467e3536458b9210ec812", new Version(1, 2, 135), new String[] {"armv7-android", "armv7-ios", "arm64-ios", "x86_64-osx", "x86_64-linux", "x86_64-win32", "js-web"}), + new DefoldVersion("b9bbe0c5d5ad6c165147c5461a5d64d9fc62f9f3", new Version(1, 2, 139), new String[] {"armv7-android", "armv7-ios", "arm64-ios", "x86_64-osx", "x86_64-linux", "x86_64-win32", "js-web"}), // Use test-data/createdebugsdk.sh to package your preferred platform sdk and it ends up in the sdk/debugsdk folder // Then you can write your tests without waiting for the next release