diff --git a/README.md b/README.md index 260a5ce2..1820c50c 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,21 @@ To speed things up, tests can be disabled by opening `./server/scripts/build.sh` _NOTE:_ The first time you build it will take a while. After that Docker cache will speed it up. ### Start -* Then, start a container based on that image by running: `./server/scripts/run.sh`. +* Then, start a container based on that image by running: `./server/scripts/run-local.sh`. +* The server is now available on port :9000 + + +### Darwin builds + +Since Apple discontinued opensourcing the libtapi library, we had to start building an actual mac machine. +You can test this flow locally: + +* Create the output directory: `sudo mkdir /usr/local/extender` +* Change permissions to it: `sudo chown mathiaswesterdahl:admin /usr/local/extender` +* Build the server (extender.jar): `./gradlew clean build -xtest` +* Run `./server/scripts/publish-standalone-local.sh` +* The server is now available on http://localhost:9010 + ### Stop * Just hit `Ctrl-C`. @@ -30,7 +44,7 @@ _NOTE:_ The first time you build it will take a while. After that Docker cache w #### Docker container -* When the container is running, then run `./server/scripts/debug.sh`. It connects to the container and executes bash. +* When the container is running, then run `./server/scripts/debug-local.sh`. It connects to the container and executes bash. * In detail: [Debugging](./README_DEBUGGING.md) diff --git a/README_DEBUGGING.md b/README_DEBUGGING.md index 7513204b..eac9c889 100644 --- a/README_DEBUGGING.md +++ b/README_DEBUGGING.md @@ -25,7 +25,7 @@ And now build the container again: After building your Docker container, you can login in using the script: - $ ./server/scripts/debug.sh + $ ./server/scripts/debug-local.sh Make sure you are `extender` by typing diff --git a/README_XCLOUD_SERVER.md b/README_XCLOUD_SERVER.md index 90f9d9c3..e1556be7 100644 --- a/README_XCLOUD_SERVER.md +++ b/README_XCLOUD_SERVER.md @@ -115,7 +115,7 @@ Install nginx to enable HTTPS with SSL certificates for the web application: Start nginx and make sure it starts up at boot: -`brew services start nginx` +`sudo brew services start nginx` Go to Xcloud web console and there will hopefully be a firewall popup: allow nginx to accept incoming connections. @@ -141,7 +141,7 @@ Edit the nginx configuration: ssl_certificate /usr/local/etc/ssl/aws.wildcard.defold.com.pem; ssl_certificate_key /usr/local/etc/ssl/aws.wildcard.defold.com.key; - server_name build-stage-darwin.defold.com; + server_name build-darwin-stage.defold.com; ``` 1. Pass all requests to the web application running on port 8080. Raise max file upload size to 500MB. @@ -157,6 +157,8 @@ Edit the nginx configuration: proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Port $server_port; client_max_body_size 500M; + proxy_request_buffering off; + proxy_buffering off; ``` 1. Comment out the error page. @@ -233,3 +235,35 @@ Add the following (for production): Add the following (for stage): `profile=standalone-stage` + +# Operations + +### What needs to run? + +The following services need to run: + +* NGINX load balancer & proxy +* Extender +* PF firewall + +Start nginx: + +`brew services start nginx` + +Start extender service: + +`extender start` + +Enable PF firewall: + +`sudo pfctl -ef /etc/pf.conf` + +### Where are the logs? + +* NGINX access and error logs: /usr/local/var/log/nginx/ +* Extender service logs: /usr/local/extender/logs/ + +### Xcloud services + +* [Xcloud portal with web console](https://my.flow.ch/portal/cloudserver) +* [Status page with incidents](https://status.flow.ch/) diff --git a/build.gradle b/build.gradle index 8fbdfa8e..5fc11a06 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ plugins { - id 'com.github.ben-manes.versions' version '0.20.0' + id 'com.github.ben-manes.versions' version '0.21.0' } subprojects { diff --git a/server/docker-base/Dockerfile b/server/docker-base/Dockerfile index 5d980655..3c9b8d90 100644 --- a/server/docker-base/Dockerfile +++ b/server/docker-base/Dockerfile @@ -9,9 +9,11 @@ RUN \ apt-get install -y software-properties-common && \ add-apt-repository -y ppa:webupd8team/java && \ apt-get update && \ - echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ apt-get install -y --no-install-recommends \ - oracle-java8-installer \ + wget \ + locales \ + default-jre \ + default-jdk \ gcc \ g++ \ gcc-multilib \ @@ -257,7 +259,18 @@ ENV ANDROID_SYSROOT ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/platform ENV ANDROID_BIN_PATH ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/toolchains/arm-linux-androideabi-${ANDROID_GCC_VERSION}/prebuilt/linux-x86_64/bin ENV ANDROID_SDK_BUILD_TOOLS_PATH ${ANDROID_HOME}/build-tools/${ANDROID_BUILD_TOOLS_VERSION} +ENV ANDROID_64_NDK_API_VERSION 21 +ENV ANDROID_64_GCC_VERSION 4.9 +ENV ANDROID_64_NDK_INCLUDE ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/platforms/android-${ANDROID_64_NDK_API_VERSION}/arch-arm64/usr/include +ENV ANDROID_64_STL_INCLUDE ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/sources/cxx-stl/gnu-libstdc++/${ANDROID_64_GCC_VERSION}/include +ENV ANDROID_64_STL_ARCH_INCLUDE ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/sources/cxx-stl/gnu-libstdc++/${ANDROID_64_GCC_VERSION}/libs/arm64-v8a/include +ENV ANDROID_64_STL_LIB ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/sources/cxx-stl/gnu-libstdc++/${ANDROID_64_GCC_VERSION}/libs/arm64-v8a +ENV ANDROID_64_SYSROOT ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/platforms/android-${ANDROID_64_NDK_API_VERSION}/arch-arm64 +ENV ANDROID_64_BIN_PATH ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/toolchains/aarch64-linux-android-${ANDROID_64_GCC_VERSION}/prebuilt/linux-x86_64/bin +ENV ANDROID_64_SDK_BUILD_TOOLS_PATH ${ANDROID_HOME}/build-tools/${ANDROID_BUILD_TOOLS_VERSION} + ENV PATH ${PATH}:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:${ANDROID_BIN_PATH}:${ANDROID_SDK_BUILD_TOOLS_PATH} +ENV PATH ${PATH}:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:${ANDROID_64_BIN_PATH}:${ANDROID_64_SDK_BUILD_TOOLS_PATH} ENV ANDROID_NDK_FILENAME android-ndk-r${ANDROID_NDK_VERSION}-linux-x86_64.zip ENV ANDROID_NDK_URL https://dl.google.com/android/repository/${ANDROID_NDK_FILENAME} diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 4d6773e2..fd9e8dc9 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -8,5 +8,5 @@ RUN mkdir -p /var/extender/cache/data && \ chown -R extender: /var/extender/cache USER extender -ENTRYPOINT ["java","-Xmx500m","-jar","/app.jar"] +ENTRYPOINT ["java","-Xmx1g","-XX:MaxDirectMemorySize=512m","-jar","/app.jar"] EXPOSE 9000 diff --git a/server/scripts/debug.sh b/server/scripts/debug-local.sh similarity index 85% rename from server/scripts/debug.sh rename to server/scripts/debug-local.sh index 39183560..6074756a 100755 --- a/server/scripts/debug.sh +++ b/server/scripts/debug-local.sh @@ -3,7 +3,7 @@ CONTAINER=$(docker ps -f name=extender -q) if [ -z "$CONTAINER" ]; then - echo Container \"extender\" is not running. Please start it first \(by running run.sh\)! + echo Container \"extender\" is not running. Please start it first \(by running run-local.sh\)! else docker exec -uextender -it $CONTAINER /bin/bash fi \ No newline at end of file diff --git a/server/scripts/debug_defoldsdk.py b/server/scripts/debug_defoldsdk.py index 3829e396..f94c067c 100755 --- a/server/scripts/debug_defoldsdk.py +++ b/server/scripts/debug_defoldsdk.py @@ -76,7 +76,7 @@ def Usage(): print("export DYNAMO_HOME=%s" % sdk_path) print("Starting server") - os.system("./server/scripts/run.sh") + os.system("./server/scripts/run-local.sh") diff --git a/server/scripts/local_server_install.sh b/server/scripts/local_server_install.sh deleted file mode 100755 index 5f1a91b0..00000000 --- a/server/scripts/local_server_install.sh +++ /dev/null @@ -1,51 +0,0 @@ -#! /usr/bin/env bash - -EXTENDER_DIR=./extender -if [ ! -e $EXTENDER_DIR ]; then - mkdir $EXTENDER_DIR - echo "Created" $EXTENDER_DIR -fi - -EXTENDER_CACHE_DIR=${EXTENDER_DIR}/cache/data -if [ ! -e $EXTENDER_CACHE_DIR ]; then - mkdir -p $EXTENDER_CACHE_DIR - echo "Created" $EXTENDER_CACHE_DIR -fi - -# Download packages -SDK_DIR=${EXTENDER_DIR}/platformsdk -if [ ! -e $SDK_DIR ]; then - mkdir -p $SDK_DIR - echo "Created" $SDK_DIR -fi - -S3_URL=https://s3-eu-west-1.amazonaws.com/defold-packages - -function download() { - local package_name=$1 - - if [ ! -e ${SDK_DIR}/${package_name} ]; then - mkdir _tmpdir - - echo "Downloading" ${package_name}.tar.gz - wget -q -O - ${S3_URL}/${package_name}.tar.gz | tar xz -C _tmpdir - - # the folder inside the package is something like "iPhoneOS.sdk" - local folder=`(cd _tmpdir && ls)` - echo "Found folder" $folder - mv _tmpdir/${folder} ${SDK_DIR}/${package_name} - rmdir _tmpdir - - echo "Installed" ${SDK_DIR}/${package_name} - else - echo "Package" ${SDK_DIR}/${package_name} "already installed" - fi -} - -# These are packaged with the version omitted, but it's needed in order to avoid bugs (e.g. the layout change issue) -download iPhoneOS11.2.sdk -download iPhoneOS12.1.sdk -download iPhoneSimulator12.1.sdk -download MacOSX10.13.sdk -download XcodeDefault10.1.xctoolchain - diff --git a/server/scripts/local_server_run.sh b/server/scripts/local_server_run.sh deleted file mode 100755 index 06e0ab37..00000000 --- a/server/scripts/local_server_run.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -# Run with PLATFORMSDK_DIR=/Users/mathiaswesterdahl/work/extender/extender/platformsdk ./local_server_run.sh - -if [ "$PLATFORMSDK_DIR" == "" ]; then - PWD=`pwd` - PLATFORMSDK_DIR=`find $PWD -name "platformsdk"` - echo "Found platformsdk:" $PLATFORMSDK_DIR -fi - -PLATFORMSDK_DIR=${PLATFORMSDK_DIR} PATH=${PLATFORMSDK_DIR}/XcodeDefault10.1.xctoolchain/usr/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin java -jar -Dspring.profiles.active=standalone,standalone-dev $PWD/server/build/docker/extender-0.1.0.jar diff --git a/server/scripts/publish-standalone-local.sh b/server/scripts/publish-standalone-local.sh new file mode 100755 index 00000000..4bf7fc81 --- /dev/null +++ b/server/scripts/publish-standalone-local.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SOURCE_DIR=${SCRIPT_DIR}/../.. + +VERSION=$(date "+%Y%m%d_%H%M") + +TARGET_DIR=/usr/local/extender +INSTALL_DIR=${TARGET_DIR}/${VERSION} + +source ${SCRIPT_DIR}/standalone/publish-standalone.sh + +mkdir -p ${TARGET_DIR} + +build_artifact ${SOURCE_DIR} +deploy_artifact ${SOURCE_DIR} ${TARGET_DIR} ${VERSION} + +bash ${SCRIPT_DIR}/run-standalone-local.sh diff --git a/server/scripts/publish-xcloud-prod.sh b/server/scripts/publish-xcloud-prod.sh index e860da6b..814888fc 100755 --- a/server/scripts/publish-xcloud-prod.sh +++ b/server/scripts/publish-xcloud-prod.sh @@ -14,4 +14,4 @@ source ${SCRIPT_DIR}/standalone/publish-standalone.sh check_uncommitted_changes ${SOURCE_DIR} build_artifact ${SOURCE_DIR} -deploy_artifact ${SOURCE_DIR} ${TARGET_HOST} ${TARGET_USER} ${TARGET_DIR} ${VERSION} +deploy_artifact ${SOURCE_DIR} ${TARGET_DIR} ${VERSION} ${TARGET_HOST} ${TARGET_USER} diff --git a/server/scripts/publish-xcloud-stage.sh b/server/scripts/publish-xcloud-stage.sh index 51c97ec1..554898f2 100755 --- a/server/scripts/publish-xcloud-stage.sh +++ b/server/scripts/publish-xcloud-stage.sh @@ -14,4 +14,4 @@ source ${SCRIPT_DIR}/standalone/publish-standalone.sh check_uncommitted_changes ${SOURCE_DIR} build_artifact ${SOURCE_DIR} -deploy_artifact ${SOURCE_DIR} ${TARGET_HOST} ${TARGET_USER} ${TARGET_DIR} ${VERSION} +deploy_artifact ${SOURCE_DIR} ${TARGET_DIR} ${VERSION} ${TARGET_HOST} ${TARGET_USER} diff --git a/server/scripts/run-local.sh b/server/scripts/run-local.sh new file mode 100755 index 00000000..cfb8d7d5 --- /dev/null +++ b/server/scripts/run-local.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [ -z "$DYNAMO_HOME" ]; then + docker run --rm --name extender -p 9000:9000 -e SPRING_PROFILES_ACTIVE=dev extender/extender; +else + docker run --rm --name extender -p 9000:9000 -e SPRING_PROFILES_ACTIVE=dev -v ${DYNAMO_HOME}:/dynamo_home -e DYNAMO_HOME=/dynamo_home extender/extender; +fi diff --git a/server/scripts/run-standalone-local.sh b/server/scripts/run-standalone-local.sh new file mode 100755 index 00000000..0a9d89f1 --- /dev/null +++ b/server/scripts/run-standalone-local.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +EXTENDER_DIR=/usr/local/extender +EXTENDER_INSTALL_DIR=${EXTENDER_DIR}/current + +# The SDK path is used by Defold SDK build.yml +export PLATFORMSDK_DIR=${EXTENDER_DIR}/platformsdk + +# We need access to the toolchain binary path from within the application +export PATH=${PLATFORMSDK_DIR}/XcodeDefault10.1.xctoolchain/usr/bin:/usr/local/bin:${PATH} + +echo [run] java -Dspring.profiles.active=standalone-dev -jar ${EXTENDER_INSTALL_DIR}/extender.jar +java -Dspring.profiles.active=standalone-dev -jar ${EXTENDER_INSTALL_DIR}/extender.jar diff --git a/server/scripts/run.sh b/server/scripts/run.sh deleted file mode 100755 index 7625f143..00000000 --- a/server/scripts/run.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -if [ -z "$DYNAMO_HOME" ]; then - docker run --rm --name extender -p 9000:9000 $DOCKER_ARGS extender/extender; -else - docker run --rm --name extender -p 9000:9000 -v ${DYNAMO_HOME}:/dynamo_home -e DYNAMO_HOME=/dynamo_home $DOCKER_ARGS extender/extender; -fi diff --git a/server/scripts/standalone/publish-standalone.sh b/server/scripts/standalone/publish-standalone.sh index bb2df0d7..5680e89d 100755 --- a/server/scripts/standalone/publish-standalone.sh +++ b/server/scripts/standalone/publish-standalone.sh @@ -22,14 +22,23 @@ deploy_artifact() { SERVER_DIR=${SOURCE_DIR}/server ARTIFACT_DIR=${SERVER_DIR}/${ARTIFACT_PATH} - TARGET_HOST=$2 - TARGET_USER=$3 - TARGET_DIR=$4 - VERSION=$5 + TARGET_DIR=$2 + VERSION=$3 + TARGET_HOST=$4 + TARGET_USER=$5 + + if [ -z ${TARGET_HOST} ] + then + echo "[deploy] Copying artifact ${VERSION} to local directory ${TARGET_DIR}..." + cp -r ${ARTIFACT_DIR} ${TARGET_DIR}/${VERSION} + echo "[deploy] Running setup script on local machine..." + bash ${TARGET_DIR}/${VERSION}/setup.sh ${VERSION} ${TARGET_DIR} + else + echo "[deploy] Secure copying artifact ${VERSION} to target ${TARGET_USER}@${TARGET_HOST}:${TARGET_DIR}..." + scp -r ${ARTIFACT_DIR} ${TARGET_USER}@${TARGET_HOST}:${TARGET_DIR}/${VERSION} + echo "[deploy] Running setup script on target host..." + ssh ${TARGET_USER}@${TARGET_HOST} bash ${TARGET_DIR}/${VERSION}/setup.sh ${VERSION} ${TARGET_DIR} /usr/local/bin/extender + fi - echo "[deploy] Secure copying artifact ${VERSION} to ${TARGET_USER}@${TARGET_HOST}:${TARGET_DIR}..." - scp -r ${ARTIFACT_DIR} ${TARGET_USER}@${TARGET_HOST}:${TARGET_DIR}/${VERSION} - echo "[deploy] Running setup script on target host..." - ssh ${TARGET_USER}@${TARGET_HOST} bash ${TARGET_DIR}/${VERSION}/setup.sh ${VERSION} echo "[deploy] Deployment done." } diff --git a/server/scripts/standalone/service-standalone.sh b/server/scripts/standalone/service-standalone.sh index b39a50f4..42231625 100644 --- a/server/scripts/standalone/service-standalone.sh +++ b/server/scripts/standalone/service-standalone.sh @@ -4,8 +4,8 @@ SERVICE_NAME=extender EXTENDER_DIR=/usr/local/extender ENVIRONMENT_FILE=${EXTENDER_DIR}/env.properties PATH_TO_JAR=${EXTENDER_DIR}/current/extender.jar -PID_PATH_NAME="/tmp/${SERVICE_NAME}.pid" -LOG_DIRECTORY=/usr/local/var/log/extender +PID_PATH_NAME=${EXTENDER_DIR}/${SERVICE_NAME}.pid +LOG_DIRECTORY=${EXTENDER_DIR}/logs STDOUT_LOG=${LOG_DIRECTORY}/stdout.log ERROR_LOG=${LOG_DIRECTORY}/error.log @@ -16,11 +16,11 @@ export PLATFORMSDK_DIR=${EXTENDER_DIR}/platformsdk export PATH=${PLATFORMSDK_DIR}/XcodeDefault10.1.xctoolchain/usr/bin:/usr/local/bin:${PATH} # Get the Spring profile from the environment file -if [ -f "${ENVIRONMENT_FILE}" ] +if [[ -f "${ENVIRONMENT_FILE}" ]] then while IFS='=' read -r key value do - if [ "${key}" = "profile" ]; then + if [[ "${key}" = "profile" ]]; then PROFILE=$(echo ${value}) echo "Using profile $PROFILE" fi @@ -30,32 +30,40 @@ else exit 1; fi -if [ -z ${PROFILE} ] -then - echo "Error! Environment file ${ENVIRONMENT_FILE} not found, exiting." +if [[ -z ${PROFILE} ]]; then + echo "Error! No Spring profile set in environment file ${ENVIRONMENT_FILE}, exiting." exit 1; fi start_service() { echo "${SERVICE_NAME} starting..." - if [ ! -f ${PID_PATH_NAME} ]; then - nohup java -jar ${PATH_TO_JAR} --spring.profiles.active=${PROFILE} >> ${STDOUT_LOG} 2>> ${ERROR_LOG} < /dev/null & - echo $! > ${PID_PATH_NAME} - echo "${SERVICE_NAME} started." - else - echo "${SERVICE_NAME} is already running." + + # Check if pid file already exists and if the process in the pid file is running + if [[ -f ${PID_PATH_NAME} ]]; then + PID=$(cat ${PID_PATH_NAME}); + if [[ -n "$(ps -p ${PID} -o pid=)" ]]; then + echo "Error! ${SERVICE_NAME} is already running, exiting." + exit 2 + else + echo "Warning: PID file exists but no process running, removing PID file." + rm ${PID_PATH_NAME} + fi fi + + nohup java -Xmx1g -XX:MaxDirectMemorySize=512m -jar ${PATH_TO_JAR} --spring.profiles.active=${PROFILE} >> ${STDOUT_LOG} 2>> ${ERROR_LOG} < /dev/null & + echo $! > ${PID_PATH_NAME} + echo "${SERVICE_NAME} started." } stop_service() { - if [ -f ${PID_PATH_NAME} ]; then + if [[ -f ${PID_PATH_NAME} ]]; then PID=$(cat ${PID_PATH_NAME}); echo "${SERVICE_NAME} stopping..." kill ${PID}; echo "${SERVICE_NAME} stopped." rm ${PID_PATH_NAME} else - echo "${SERVICE_NAME} is not running." + echo "Warning: ${SERVICE_NAME} is not running, no PID file." fi } diff --git a/server/scripts/standalone/setup-standalone-server.sh b/server/scripts/standalone/setup-standalone-server.sh index 80df7ca4..ba09a144 100755 --- a/server/scripts/standalone/setup-standalone-server.sh +++ b/server/scripts/standalone/setup-standalone-server.sh @@ -1,63 +1,57 @@ #!/bin/bash -if [ $# -ne 1 ]; then - printf "Usage: $(basename "$0") \n\n"; +if [[ "$#" -ne 2 ]] && [[ "$#" -ne 3 ]]; then + printf "Usage: $(basename "$0") [service script]\n\n"; exit 1; fi VERSION=$1 +EXTENDER_DIR=$2 +EXTENDER_SERVICE=$3 -EXTENDER_DIR=/usr/local/extender -EXTENDER_SERVICE=/usr/local/bin/extender -EXTENDER_INSTALL_DIR=/usr/local/extender/${VERSION} +EXTENDER_INSTALL_DIR=${EXTENDER_DIR}/${VERSION} -if [ ! -e ${EXTENDER_DIR} ]; then - echo "Error! Extender home directory ${EXTENDER_DIR} not setup, exiting.\n\n" +if [[ ! -e ${EXTENDER_DIR} ]]; then + echo "[setup] Error! Extender home directory ${EXTENDER_DIR} not setup, exiting.\n\n" exit 2 fi -# Builds -EXTENDER_BUILD_DIR=${EXTENDER_DIR}/builds -if [ ! -e ${EXTENDER_BUILD_DIR} ]; then - mkdir -p ${EXTENDER_BUILD_DIR} - echo "Created build directory at ${EXTENDER_BUILD_DIR}." -fi - # Platform SDKs PLATFORMSDK_DIR=${EXTENDER_DIR}/platformsdk -if [ ! -e ${PLATFORMSDK_DIR} ]; then +if [[ ! -e ${PLATFORMSDK_DIR} ]]; then mkdir -p ${PLATFORMSDK_DIR} - echo "Created SDK directory at ${PLATFORMSDK_DIR}." + echo "[setup] Created SDK directory at ${PLATFORMSDK_DIR}." fi # Logs -LOGS_DIR=/usr/local/var/log/extender -if [ ! -e ${LOGS_DIR} ]; then +LOGS_DIR=${EXTENDER_DIR}/logs +if [[ ! -e ${LOGS_DIR} ]]; then mkdir -p ${LOGS_DIR} - echo "Created logs directory at ${LOGS_DIR}." + echo "[setup] Created logs directory at ${LOGS_DIR}." fi S3_URL=https://s3-eu-west-1.amazonaws.com/defold-packages WGET_CMD=/usr/local/bin/wget +TMP_DOWNLOAD_DIR=/tmp/_extender_download function download_package() { local package_name=$1 - if [ ! -e ${PLATFORMSDK_DIR}/${package_name} ]; then - mkdir _tmpdir + if [[ ! -e ${PLATFORMSDK_DIR}/${package_name} ]]; then + mkdir -p ${TMP_DOWNLOAD_DIR} - echo "Downloading" ${package_name}.tar.gz - ${WGET_CMD} -q -O - ${S3_URL}/${package_name}.tar.gz | tar xz -C _tmpdir + echo "[setup] Downloading" ${package_name}.tar.gz + ${WGET_CMD} -q -O - ${S3_URL}/${package_name}.tar.gz | tar xz -C ${TMP_DOWNLOAD_DIR} # The folder inside the package is something like "iPhoneOS.sdk" - local folder=`(cd _tmpdir && ls)` - echo "Found folder" ${folder} - mv _tmpdir/${folder} ${PLATFORMSDK_DIR}/${package_name} - rmdir _tmpdir + local folder=`(cd ${TMP_DOWNLOAD_DIR} && ls)` + echo "[setup] Found folder" ${folder} + mv ${TMP_DOWNLOAD_DIR}/${folder} ${PLATFORMSDK_DIR}/${package_name} + rm -rf ${TMP_DOWNLOAD_DIR} - echo "Installed" ${PLATFORMSDK_DIR}/${package_name} + echo "[setup] Installed" ${PLATFORMSDK_DIR}/${package_name} else - echo "Package" ${PLATFORMSDK_DIR}/${package_name} "already installed" + echo "[setup] Package" ${PLATFORMSDK_DIR}/${package_name} "already installed" fi } @@ -78,8 +72,15 @@ function download_packages() { download_packages -${EXTENDER_SERVICE} stop -ln -sfn ${VERSION} /usr/local/extender/current -ln -sfn /usr/local/extender/current/service.sh /usr/local/bin/extender chmod a+x ${EXTENDER_INSTALL_DIR}/service.sh -${EXTENDER_SERVICE} start + +if [[ -e ${EXTENDER_SERVICE} ]]; then + ${EXTENDER_SERVICE} stop +fi + +ln -sfn ${VERSION} ${EXTENDER_DIR}/current + +if [[ -e ${EXTENDER_SERVICE} ]]; then + ln -sfn ${EXTENDER_DIR}/current/service.sh ${EXTENDER_SERVICE} + ${EXTENDER_SERVICE} start +fi diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index fb02d931..b2341887 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -289,6 +289,7 @@ private static Map createEmptyContext(Map origin private Map context(Map manifestContext) throws ExtenderException { Map context = new HashMap<>(config.context); + // Not needed since 1.2.153 - keep in case someone uses older build.yml if (this.platform.contains("android")) { context.put("android_ndk_path", ANDROID_NDK_PATH); context.put("android_ndk_include", ANDROID_NDK_INCLUDE_PATH); @@ -821,6 +822,10 @@ private List buildEngine() throws ExtenderException { for (File manifest : this.manifests) { ManifestConfiguration manifestConfig = Extender.loadYaml(this.jobDirectory, manifest, ManifestConfiguration.class); + if (manifestConfig == null) { + throw new ExtenderException("Missing manifest file: " + manifest.getAbsolutePath()); + } + Map manifestContext = new HashMap<>(); if (manifestConfig.platforms != null) { manifestContext = getManifestContext(platform, config, manifestConfig); diff --git a/server/src/main/java/com/defold/extender/ExtenderController.java b/server/src/main/java/com/defold/extender/ExtenderController.java index 52325970..ac0965e8 100644 --- a/server/src/main/java/com/defold/extender/ExtenderController.java +++ b/server/src/main/java/com/defold/extender/ExtenderController.java @@ -32,6 +32,7 @@ import java.io.OutputStream; import java.net.URISyntaxException; import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -110,6 +111,8 @@ public void buildEngine(HttpServletRequest request, @PathVariable("sdkVersion") String sdkVersionString) throws ExtenderException, IOException, URISyntaxException { + LOGGER.info("Starting build: sdk={}, platform={}", sdkVersionString, platform); + boolean isMultipart = ServletFileUpload.isMultipartContent(request); if (!isMultipart) { throw new ExtenderException("The request must be a multi part request"); @@ -244,11 +247,14 @@ static void receiveUpload(HttpServletRequest request, File uploadDirectory) thro if (!isRelativePath(uploadDirectory, file)) { // in case the name contains "../" throw new ExtenderException(String.format("Files must be relative to the upload package: '%s'", item.getName())); } + if (file.exists()) { + LOGGER.info("Duplicate file in received zip file: ", name); + } Files.createDirectories(file.getParentFile().toPath()); try (InputStream inputStream = item.openStream()) { - Files.copy(inputStream, file.toPath()); + Files.copy(inputStream, file.toPath(), StandardCopyOption.REPLACE_EXISTING); } count++; diff --git a/server/src/main/java/com/defold/extender/ManifestPlatformConfig.java b/server/src/main/java/com/defold/extender/ManifestPlatformConfig.java index 092a0c12..6ca071be 100644 --- a/server/src/main/java/com/defold/extender/ManifestPlatformConfig.java +++ b/server/src/main/java/com/defold/extender/ManifestPlatformConfig.java @@ -4,5 +4,6 @@ import java.util.Map; class ManifestPlatformConfig { + public Map bundle = new HashMap<>(); // used on the content pipeline side public Map context = new HashMap<>(); } diff --git a/server/src/main/java/com/defold/extender/cache/DataCacheFactory.java b/server/src/main/java/com/defold/extender/cache/DataCacheFactory.java index 868e29b9..b9b51140 100644 --- a/server/src/main/java/com/defold/extender/cache/DataCacheFactory.java +++ b/server/src/main/java/com/defold/extender/cache/DataCacheFactory.java @@ -21,9 +21,9 @@ public class DataCacheFactory { private final String bucketName; public DataCacheFactory(@Value("${extender.cache.enabled}") boolean isEnabled, - @Value("${extender.cache.type}") String storeType, - @Value("${extender.cache.local.basedir}") String baseDirectory, - @Value("${extender.cache.s3.bucket}") String bucketName) { + @Value("${extender.cache.type:}") String storeType, + @Value("${extender.cache.local.basedir:}") String baseDirectory, + @Value("${extender.cache.s3.bucket:}") String bucketName) { this.isEnabled = isEnabled; this.storeType = storeType; this.baseDirectory = baseDirectory; diff --git a/server/src/main/java/com/defold/extender/remote/RemoteBuildException.java b/server/src/main/java/com/defold/extender/remote/RemoteBuildException.java new file mode 100644 index 00000000..db766892 --- /dev/null +++ b/server/src/main/java/com/defold/extender/remote/RemoteBuildException.java @@ -0,0 +1,12 @@ +package com.defold.extender.remote; + +public class RemoteBuildException extends RuntimeException { + + public RemoteBuildException(String message, Throwable cause) { + super(message, cause); + } + + public RemoteBuildException(String message) { + super(message); + } +} diff --git a/server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java b/server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java index 032b0c4a..d2a41229 100644 --- a/server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java +++ b/server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java @@ -29,7 +29,7 @@ public class RemoteEngineBuilder { private final String remoteBuilderBaseUrl; @Autowired - public RemoteEngineBuilder(@Value("${extender.remote-builder.url}") final String remoteBuilderBaseUrl) { + public RemoteEngineBuilder(@Value("${extender.remote-builder.url:}") final String remoteBuilderBaseUrl) { this.remoteBuilderBaseUrl = remoteBuilderBaseUrl; } @@ -44,23 +44,27 @@ public byte[] build(final File projectDirectory, try { httpEntity = buildHttpEntity(projectDirectory); } catch(IllegalStateException|IOException e) { - throw new ExtenderException(e, "Failed to add files to multipart request"); + throw new RemoteBuildException("Failed to add files to multipart request", e); } try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { final HttpResponse response = sendRequest(platform, sdkVersion, httpEntity); + LOGGER.info("Remote builder response status: {}", response.getStatusLine()); + response.getEntity().writeTo(bos); final byte[] bytes = bos.toByteArray(); - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - throw new ExtenderException("Failed to build engine remotely: " + new String(bytes)); + if (isClientError(response)) { + throw new ExtenderException("Client error when building engine remotely: " + new String(bytes)); + } else if (isServerError(response)) { + throw new RemoteBuildException("Server error when building engine remotely: " + new String(bytes)); } return bytes; } catch (IOException e) { - throw new ExtenderException(e, "Failed to communicate with remote builder"); + throw new RemoteBuildException("Failed to communicate with remote builder", e); } } @@ -92,4 +96,14 @@ HttpEntity buildHttpEntity(final File projectDirectory) throws IOException { return builder.build(); } + + private boolean isClientError(final HttpResponse response) { + final int statusCode = response.getStatusLine().getStatusCode(); + return HttpStatus.SC_BAD_REQUEST <= statusCode && statusCode < HttpStatus.SC_INTERNAL_SERVER_ERROR; + } + + private boolean isServerError(final HttpResponse response) { + final int statusCode = response.getStatusLine().getStatusCode(); + return statusCode >= HttpStatus.SC_INTERNAL_SERVER_ERROR; + } } diff --git a/server/src/main/java/com/defold/extender/services/DefoldSdkService.java b/server/src/main/java/com/defold/extender/services/DefoldSdkService.java index 04699c3c..f423df72 100644 --- a/server/src/main/java/com/defold/extender/services/DefoldSdkService.java +++ b/server/src/main/java/com/defold/extender/services/DefoldSdkService.java @@ -43,8 +43,8 @@ public class DefoldSdkService { private final GaugeService gaugeService; @Autowired - DefoldSdkService(@Value("${extender.sdk-location}") String baseSdkDirectory, - @Value("${extender.sdk-cache-size}") int cacheSize, + DefoldSdkService(@Value("${extender.sdk.location}") String baseSdkDirectory, + @Value("${extender.sdk.cache-size}") int cacheSize, CounterService counterService, GaugeService gaugeService) throws IOException { this.baseSdkDirectory = new File(baseSdkDirectory).toPath(); @@ -53,6 +53,8 @@ public class DefoldSdkService { this.gaugeService = gaugeService; this.dynamoHome = System.getenv("DYNAMO_HOME") != null ? new File(System.getenv("DYNAMO_HOME")) : null; + LOGGER.info("SDK service using directory {} with cache size {}", baseSdkDirectory, cacheSize); + if (!Files.exists(this.baseSdkDirectory)) { Files.createDirectories(this.baseSdkDirectory); } diff --git a/server/src/main/resources/application-dev.yml b/server/src/main/resources/application-dev.yml new file mode 100644 index 00000000..6e660bd8 --- /dev/null +++ b/server/src/main/resources/application-dev.yml @@ -0,0 +1,13 @@ + +extender: + cache: + enabled: true + type: LOCAL + local.basedir: /var/extender/cache/data + #type: S3 + #s3.bucket: defold-extender-cache-dev + + remote-builder: + enabled: true + # This points to the host machine from inside docker + url: http://host.docker.internal:9010 diff --git a/server/src/main/resources/application-production.yml b/server/src/main/resources/application-production.yml new file mode 100644 index 00000000..3529f916 --- /dev/null +++ b/server/src/main/resources/application-production.yml @@ -0,0 +1,13 @@ + +extender: + sdk.cache-size: 5 + cache: + type: S3 + s3.bucket: defold-extender-cache-prod + remote-builder: + enabled: true + url: https://build-darwin.defold.com + +endpoints: + health: + sensitive: true diff --git a/server/src/main/resources/application-stage.yml b/server/src/main/resources/application-stage.yml new file mode 100644 index 00000000..98920706 --- /dev/null +++ b/server/src/main/resources/application-stage.yml @@ -0,0 +1,13 @@ + +extender: + sdk.cache-size: 5 + cache: + type: S3 + s3.bucket: defold-extender-cache-stage + remote-builder: + enabled: true + url: https://build-darwin-stage.defold.com + +endpoints: + health: + sensitive: true diff --git a/server/src/main/resources/application-standalone-dev.yml b/server/src/main/resources/application-standalone-dev.yml new file mode 100644 index 00000000..e41626fe --- /dev/null +++ b/server/src/main/resources/application-standalone-dev.yml @@ -0,0 +1,10 @@ + +server: + port: 9010 + +extender: + sdk.location: /usr/local/extender/sdk + cache: + enabled: false + remote-builder: + enabled: false diff --git a/server/src/main/resources/application-standalone-production.yml b/server/src/main/resources/application-standalone-production.yml new file mode 100644 index 00000000..816f28aa --- /dev/null +++ b/server/src/main/resources/application-standalone-production.yml @@ -0,0 +1,16 @@ + +server: + port: 8080 + +extender: + sdk: + location: /usr/local/extender/sdk + cache-size: 5 + cache: + enabled: false + remote-builder: + enabled: false + +endpoints: + health: + sensitive: true diff --git a/server/src/main/resources/application-standalone-stage.yml b/server/src/main/resources/application-standalone-stage.yml new file mode 100644 index 00000000..816f28aa --- /dev/null +++ b/server/src/main/resources/application-standalone-stage.yml @@ -0,0 +1,16 @@ + +server: + port: 8080 + +extender: + sdk: + location: /usr/local/extender/sdk + cache-size: 5 + cache: + enabled: false + remote-builder: + enabled: false + +endpoints: + health: + sensitive: true diff --git a/server/src/main/resources/application.yml b/server/src/main/resources/application.yml index 78ba3120..b6892afa 100644 --- a/server/src/main/resources/application.yml +++ b/server/src/main/resources/application.yml @@ -3,25 +3,18 @@ server: port: 9000 extender: - sdk-location: /var/extender/sdk - # Slightly bigger than production in order to not remove version controlled SDK - sdk-cache-size: 10 - build-location: /var/extender/builds + sdk: + location: /var/extender/sdk + # Slightly bigger than production in order to not remove version controlled SDK + cache-size: 10 server: http: idle-timeout: 600000 cache: - enabled: true - type: LOCAL + enabled: false file-size-threshold: 65536 - local: - basedir: /var/extender/cache/data - s3: - bucket: defold-extender-cache-dev remote-builder: enabled: false - # This points to the host machine from inside docker - url: http://host.docker.internal:9010 platforms: ios, osx spring: @@ -49,99 +42,3 @@ management: diskspace: path: / threshold: 524288000 #500mb - ---- - -spring.profiles: dev-s3 - -extender: - cache: - type: S3 - ---- - -spring.profiles: dev-remote-builder - -extender: - remote-builder: - enabled: true - ---- - -spring: - profiles: stage - -extender: - sdk-cache-size: 5 - cache: - type: S3 - s3: - bucket: defold-extender-cache-stage - remote-builder: - enabled: true - url: https://build-darwin-stage.defold.com - -endpoints: - health: - sensitive: true - ---- - -spring.profiles: production - -extender: - sdk-cache-size: 5 - cache: - type: S3 - s3: - bucket: defold-extender-cache-prod - remote-builder: - enabled: true - url: https://build-darwin.defold.com - -endpoints: - health: - sensitive: true - ---- - -spring.profiles: standalone - -server: - port: 8080 - -extender: - sdk-location: /usr/local/extender/tools - build-location: /usr/local/extender/builds - cache: - enabled: false - remote-builder: - enabled: false - ---- - -spring.profiles: standalone-dev -server: - port: 9010 - -extender: - sdk-location: ./extender/sdk - build-location: ./extender/builds - cache: - enabled: false - remote-builder: - enabled: false - ---- - -spring.profiles: standalone-production -spring.profiles.include: - - production - - standalone - ---- - -spring.profiles: standalone-stage -spring.profiles.include: - - stage - - standalone diff --git a/server/src/test/java/com/defold/extender/remote/RemoteEngineBuilderTest.java b/server/src/test/java/com/defold/extender/remote/RemoteEngineBuilderTest.java index e97d3e4a..328a2bc1 100644 --- a/server/src/test/java/com/defold/extender/remote/RemoteEngineBuilderTest.java +++ b/server/src/test/java/com/defold/extender/remote/RemoteEngineBuilderTest.java @@ -8,7 +8,6 @@ import org.apache.http.message.BasicHttpResponse; import org.junit.Before; import org.junit.Test; -import org.junit.Ignore; import java.io.File; import java.io.IOException; @@ -20,7 +19,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; -@Ignore public class RemoteEngineBuilderTest { private RemoteEngineBuilder remoteEngineBuilder; @@ -59,7 +57,7 @@ public void buildSuccessfully() throws IOException, ExtenderException { assertEquals(content, new String(bytes)); } - @Test(expected = ExtenderException.class) + @Test(expected = RemoteBuildException.class) @SuppressWarnings("unchecked") public void buildShouldThrowException() throws IOException, ExtenderException { final File directory = mock(File.class); diff --git a/server/src/test/java/com/defold/extender/services/DataCacheServiceTest.java b/server/src/test/java/com/defold/extender/services/DataCacheServiceTest.java index dd4d0572..73bf36ff 100644 --- a/server/src/test/java/com/defold/extender/services/DataCacheServiceTest.java +++ b/server/src/test/java/com/defold/extender/services/DataCacheServiceTest.java @@ -28,6 +28,14 @@ public class DataCacheServiceTest { private static final int fileThreshold = 5; + private DataCacheFactory createDataCacheFactoryWithS3Store() { + return new DataCacheFactory( + true, + "S3", + "", + "defold-extender-cache-dev"); + } + @Test @Ignore public void testUploadingToS3Cache() throws Exception { @@ -35,7 +43,7 @@ public void testUploadingToS3Cache() throws Exception { new CacheKeyGenerator(), new CacheInfoFileParser(), new CacheInfoFileWriter(), - new DataCacheFactory(true,"S3", "", "defold-extender-cache-dev"), + createDataCacheFactoryWithS3Store(), true, fileThreshold); @@ -50,7 +58,7 @@ public void testQueryS3Cache() throws Exception { new CacheKeyGenerator(), new CacheInfoFileParser(), new CacheInfoFileWriter(), - new DataCacheFactory(true, "S3", "", "defold-extender-cache-dev"), + createDataCacheFactoryWithS3Store(), true, fileThreshold); @@ -69,7 +77,7 @@ public void testDownloadingFromS3Cache() throws Exception { new CacheKeyGenerator(), new CacheInfoFileParser(), new CacheInfoFileWriter(), - new DataCacheFactory(true, "S3", "", "defold-extender-cache-dev"), + createDataCacheFactoryWithS3Store(), true, fileThreshold);