Merge beta

README.md

@@ -21,7 +21,21 @@ To speed things up, tests can be disabled by opening `./server/scripts/build.sh`
 _NOTE:_ The first time you build it will take a while. After that Docker cache will speed it up.
 
 ### Start
-* Then, start a container based on that image by running: `./server/scripts/run.sh`.
+* Then, start a container based on that image by running: `./server/scripts/run-local.sh`.
+* The server is now available on port :9000
+
+
+### Darwin builds
+
+Since Apple discontinued opensourcing the libtapi library, we had to start building an actual mac machine.
+You can test this flow locally:
+
+* Create the output directory: `sudo mkdir /usr/local/extender`
+* Change permissions to it: `sudo chown mathiaswesterdahl:admin /usr/local/extender`
+* Build the server (extender.jar): `./gradlew clean build -xtest`
+* Run `./server/scripts/publish-standalone-local.sh`
+* The server is now available on http://localhost:9010
+
 
 ### Stop
 * Just hit `Ctrl-C`.
@@ -30,7 +44,7 @@ _NOTE:_ The first time you build it will take a while. After that Docker cache w
 
 #### Docker container
 
-* When the container is running, then run `./server/scripts/debug.sh`. It connects to the container and executes bash.
+* When the container is running, then run `./server/scripts/debug-local.sh`. It connects to the container and executes bash.
 
 * In detail: [Debugging](./README_DEBUGGING.md)
 

README_DEBUGGING.md

@@ -25,7 +25,7 @@ And now build the container again:
 
 After building your Docker container, you can login in using the script:
 
-    $ ./server/scripts/debug.sh
+    $ ./server/scripts/debug-local.sh
 
 Make sure you are `extender` by typing
 

README_XCLOUD_SERVER.md

@@ -115,7 +115,7 @@ Install nginx to enable HTTPS with SSL certificates for the web application:
 
 Start nginx and make sure it starts up at boot:
 
-`brew services start nginx`  
+`sudo brew services start nginx`  
 
 Go to Xcloud web console and there will hopefully be a firewall popup: allow nginx to accept incoming connections.
 
@@ -141,7 +141,7 @@ Edit the nginx configuration:
            ssl_certificate /usr/local/etc/ssl/aws.wildcard.defold.com.pem;
            ssl_certificate_key /usr/local/etc/ssl/aws.wildcard.defold.com.key;
            
-           server_name  build-stage-darwin.defold.com;
+           server_name  build-darwin-stage.defold.com;
    ```
 1. Pass all requests to the web application running on port 8080. Raise max file upload size to 500MB.
 
@@ -157,6 +157,8 @@ Edit the nginx configuration:
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-Forwarded-Port $server_port;
            client_max_body_size 500M;
+           proxy_request_buffering off;
+           proxy_buffering off;
    ```
 1. Comment out the error page.
 
@@ -233,3 +235,35 @@ Add the following (for production):
 Add the following (for stage):
 
 `profile=standalone-stage`
+
+# Operations
+
+### What needs to run?
+
+The following services need to run:
+
+* NGINX load balancer & proxy
+* Extender
+* PF firewall
+
+Start nginx:
+
+`brew services start nginx`
+
+Start extender service:
+
+`extender start`
+
+Enable PF firewall:
+
+`sudo pfctl -ef /etc/pf.conf`
+
+### Where are the logs?
+
+* NGINX access and error logs: /usr/local/var/log/nginx/
+* Extender service logs: /usr/local/extender/logs/
+
+### Xcloud services
+
+* [Xcloud portal with web console](https://my.flow.ch/portal/cloudserver)
+* [Status page with incidents](https://status.flow.ch/)

build.gradle

@@ -1,6 +1,6 @@
 
 plugins {
-    id 'com.github.ben-manes.versions' version '0.20.0'
+    id 'com.github.ben-manes.versions' version '0.21.0'
 }
 
 subprojects {

server/docker-base/Dockerfile

@@ -9,9 +9,11 @@ RUN \
   apt-get install -y software-properties-common && \
   add-apt-repository -y ppa:webupd8team/java && \
   apt-get update && \
-  echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
   apt-get install -y --no-install-recommends \
-    oracle-java8-installer \
+    wget \
+    locales \
+    default-jre \
+    default-jdk \
     gcc \
     g++ \
     gcc-multilib \
@@ -257,7 +259,18 @@ ENV ANDROID_SYSROOT ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/platform
 ENV ANDROID_BIN_PATH ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/toolchains/arm-linux-androideabi-${ANDROID_GCC_VERSION}/prebuilt/linux-x86_64/bin
 ENV ANDROID_SDK_BUILD_TOOLS_PATH ${ANDROID_HOME}/build-tools/${ANDROID_BUILD_TOOLS_VERSION}
 
+ENV ANDROID_64_NDK_API_VERSION 21
+ENV ANDROID_64_GCC_VERSION 4.9
+ENV ANDROID_64_NDK_INCLUDE ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/platforms/android-${ANDROID_64_NDK_API_VERSION}/arch-arm64/usr/include
+ENV ANDROID_64_STL_INCLUDE ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/sources/cxx-stl/gnu-libstdc++/${ANDROID_64_GCC_VERSION}/include
+ENV ANDROID_64_STL_ARCH_INCLUDE ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/sources/cxx-stl/gnu-libstdc++/${ANDROID_64_GCC_VERSION}/libs/arm64-v8a/include
+ENV ANDROID_64_STL_LIB ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/sources/cxx-stl/gnu-libstdc++/${ANDROID_64_GCC_VERSION}/libs/arm64-v8a
+ENV ANDROID_64_SYSROOT ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/platforms/android-${ANDROID_64_NDK_API_VERSION}/arch-arm64
+ENV ANDROID_64_BIN_PATH ${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION}/toolchains/aarch64-linux-android-${ANDROID_64_GCC_VERSION}/prebuilt/linux-x86_64/bin
+ENV ANDROID_64_SDK_BUILD_TOOLS_PATH ${ANDROID_HOME}/build-tools/${ANDROID_BUILD_TOOLS_VERSION}
+
 ENV PATH ${PATH}:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:${ANDROID_BIN_PATH}:${ANDROID_SDK_BUILD_TOOLS_PATH}
+ENV PATH ${PATH}:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:${ANDROID_64_BIN_PATH}:${ANDROID_64_SDK_BUILD_TOOLS_PATH}
 
 ENV ANDROID_NDK_FILENAME android-ndk-r${ANDROID_NDK_VERSION}-linux-x86_64.zip
 ENV ANDROID_NDK_URL https://dl.google.com/android/repository/${ANDROID_NDK_FILENAME}

server/docker/Dockerfile

@@ -8,5 +8,5 @@ RUN mkdir -p /var/extender/cache/data && \
     chown -R extender: /var/extender/cache
 
 USER extender
-ENTRYPOINT ["java","-Xmx500m","-jar","/app.jar"]
+ENTRYPOINT ["java","-Xmx1g","-XX:MaxDirectMemorySize=512m","-jar","/app.jar"]
 EXPOSE 9000

server/scripts/debug.sh → server/scripts/debug-local.sh

@@ -3,7 +3,7 @@
 CONTAINER=$(docker ps -f name=extender -q)
 
 if [ -z "$CONTAINER" ]; then
-    echo Container \"extender\" is not running. Please start it first \(by running run.sh\)!
+    echo Container \"extender\" is not running. Please start it first \(by running run-local.sh\)!
 else
     docker exec -uextender -it $CONTAINER /bin/bash
 fi

server/scripts/debug_defoldsdk.py

@@ -76,7 +76,7 @@ def Usage():
 	print("export DYNAMO_HOME=%s" % sdk_path)
 
 	print("Starting server")
-	os.system("./server/scripts/run.sh")
+	os.system("./server/scripts/run-local.sh")
 
 
 

server/scripts/publish-standalone-local.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SOURCE_DIR=${SCRIPT_DIR}/../..
+
+VERSION=$(date "+%Y%m%d_%H%M")
+
+TARGET_DIR=/usr/local/extender
+INSTALL_DIR=${TARGET_DIR}/${VERSION}
+
+source ${SCRIPT_DIR}/standalone/publish-standalone.sh
+
+mkdir -p ${TARGET_DIR}
+
+build_artifact ${SOURCE_DIR}
+deploy_artifact ${SOURCE_DIR} ${TARGET_DIR} ${VERSION}
+
+bash ${SCRIPT_DIR}/run-standalone-local.sh

server/scripts/publish-xcloud-prod.sh

@@ -14,4 +14,4 @@ source ${SCRIPT_DIR}/standalone/publish-standalone.sh
 
 check_uncommitted_changes ${SOURCE_DIR}
 build_artifact ${SOURCE_DIR}
-deploy_artifact ${SOURCE_DIR} ${TARGET_HOST} ${TARGET_USER} ${TARGET_DIR} ${VERSION}
+deploy_artifact ${SOURCE_DIR} ${TARGET_DIR} ${VERSION} ${TARGET_HOST} ${TARGET_USER}

server/scripts/publish-xcloud-stage.sh

@@ -14,4 +14,4 @@ source ${SCRIPT_DIR}/standalone/publish-standalone.sh
 
 check_uncommitted_changes ${SOURCE_DIR}
 build_artifact ${SOURCE_DIR}
-deploy_artifact ${SOURCE_DIR} ${TARGET_HOST} ${TARGET_USER} ${TARGET_DIR} ${VERSION}
+deploy_artifact ${SOURCE_DIR} ${TARGET_DIR} ${VERSION} ${TARGET_HOST} ${TARGET_USER}

server/scripts/run-local.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [ -z "$DYNAMO_HOME" ]; then
+    docker run --rm --name extender -p 9000:9000 -e SPRING_PROFILES_ACTIVE=dev extender/extender;
+else
+    docker run --rm --name extender -p 9000:9000 -e SPRING_PROFILES_ACTIVE=dev -v ${DYNAMO_HOME}:/dynamo_home -e DYNAMO_HOME=/dynamo_home extender/extender;
+fi

server/scripts/run-standalone-local.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+EXTENDER_DIR=/usr/local/extender
+EXTENDER_INSTALL_DIR=${EXTENDER_DIR}/current
+
+# The SDK path is used by Defold SDK build.yml
+export PLATFORMSDK_DIR=${EXTENDER_DIR}/platformsdk
+
+# We need access to the toolchain binary path from within the application
+export PATH=${PLATFORMSDK_DIR}/XcodeDefault10.1.xctoolchain/usr/bin:/usr/local/bin:${PATH}
+
+echo [run] java -Dspring.profiles.active=standalone-dev -jar ${EXTENDER_INSTALL_DIR}/extender.jar
+java -Dspring.profiles.active=standalone-dev -jar ${EXTENDER_INSTALL_DIR}/extender.jar

server/scripts/standalone/publish-standalone.sh

@@ -22,14 +22,23 @@ deploy_artifact() {
     SERVER_DIR=${SOURCE_DIR}/server
     ARTIFACT_DIR=${SERVER_DIR}/${ARTIFACT_PATH}
 
-    TARGET_HOST=$2
-    TARGET_USER=$3
-    TARGET_DIR=$4
-    VERSION=$5
+    TARGET_DIR=$2
+    VERSION=$3
+    TARGET_HOST=$4
+    TARGET_USER=$5
+
+    if [ -z ${TARGET_HOST} ]
+    then
+        echo "[deploy] Copying artifact ${VERSION} to local directory ${TARGET_DIR}..."
+        cp -r ${ARTIFACT_DIR} ${TARGET_DIR}/${VERSION}
+        echo "[deploy] Running setup script on local machine..."
+        bash ${TARGET_DIR}/${VERSION}/setup.sh ${VERSION} ${TARGET_DIR}
+    else
+        echo "[deploy] Secure copying artifact ${VERSION} to target ${TARGET_USER}@${TARGET_HOST}:${TARGET_DIR}..."
+        scp -r ${ARTIFACT_DIR} ${TARGET_USER}@${TARGET_HOST}:${TARGET_DIR}/${VERSION}
+        echo "[deploy] Running setup script on target host..."
+        ssh ${TARGET_USER}@${TARGET_HOST} bash ${TARGET_DIR}/${VERSION}/setup.sh ${VERSION} ${TARGET_DIR} /usr/local/bin/extender
+    fi
 
-    echo "[deploy] Secure copying artifact ${VERSION} to ${TARGET_USER}@${TARGET_HOST}:${TARGET_DIR}..."
-    scp -r ${ARTIFACT_DIR} ${TARGET_USER}@${TARGET_HOST}:${TARGET_DIR}/${VERSION}
-    echo "[deploy] Running setup script on target host..."
-    ssh ${TARGET_USER}@${TARGET_HOST} bash ${TARGET_DIR}/${VERSION}/setup.sh ${VERSION}
     echo "[deploy] Deployment done."
 }